Implement easy Bottom Turtle HA support in the charts (#816)

* Implement easy Bottom Turtle HA support in the charts Signed-off-by: Kevin Fox <[email protected]> * Add diagram Signed-off-by: Kevin Fox <[email protected]> * Update docs Signed-off-by: Kevin Fox <[email protected]> * Some fixes and tightened defaults Signed-off-by: Kevin Fox <[email protected]> * More diagrams Signed-off-by: Kevin Fox <[email protected]> * More instructions Signed-off-by: Kevin Fox <[email protected]> * More instructions Signed-off-by: Kevin Fox <[email protected]> * More instructions Signed-off-by: Kevin Fox <[email protected]> * More instructions Signed-off-by: Kevin Fox <[email protected]> * More instructions Signed-off-by: Kevin Fox <[email protected]> * Install some bottom turtle spire bits Signed-off-by: Kevin Fox <[email protected]> * Trigger in github Signed-off-by: Kevin Fox <[email protected]> * Fix path Signed-off-by: Kevin Fox <[email protected]> * Fix path Signed-off-by: Kevin Fox <[email protected]> * Fix path Signed-off-by: Kevin Fox <[email protected]> * Fix path Signed-off-by: Kevin Fox <[email protected]> * Fix path Signed-off-by: Kevin Fox <[email protected]> * Fix path Signed-off-by: Kevin Fox <[email protected]> * Fix shell code Signed-off-by: Kevin Fox <[email protected]> * Add some more testing Signed-off-by: Kevin Fox <[email protected]> * Add some more testing Signed-off-by: Kevin Fox <[email protected]> * Add some more testing Signed-off-by: Kevin Fox <[email protected]> * Add some more debug logging Signed-off-by: Kevin Fox <[email protected]> * More logging Signed-off-by: Kevin Fox <[email protected]> * Some fixes Signed-off-by: Kevin Fox <[email protected]> * Some fixes Signed-off-by: Kevin Fox <[email protected]> * Some fixes Signed-off-by: Kevin Fox <[email protected]> * Some fixes Signed-off-by: Kevin Fox <[email protected]> * Some fixes Signed-off-by: Kevin Fox <[email protected]> * Some fixes Signed-off-by: Kevin Fox <[email protected]> * Some fixes Signed-off-by: Kevin Fox <[email protected]> * Some fixes Signed-off-by: Kevin Fox <[email protected]> * Some fixes Signed-off-by: Kevin Fox <[email protected]> * Some fixes Signed-off-by: Kevin Fox <[email protected]> * Some fixes Signed-off-by: Kevin Fox <[email protected]> * Some fixes Signed-off-by: Kevin Fox <[email protected]> * Some fixes Signed-off-by: Kevin Fox <[email protected]> * Some fixes Signed-off-by: Kevin Fox <[email protected]> * Some fixes Signed-off-by: Kevin Fox <[email protected]> * Some fixes Signed-off-by: Kevin Fox <[email protected]> * Some fixes Signed-off-by: Kevin Fox <[email protected]> * More updates Signed-off-by: Kevin Fox <[email protected]> * More updates Signed-off-by: Kevin Fox <[email protected]> * More updates Signed-off-by: Kevin Fox <[email protected]> * More updates Signed-off-by: Kevin Fox <[email protected]> * More updates Signed-off-by: Kevin Fox <[email protected]> * More updates Signed-off-by: Kevin Fox <[email protected]> * More updates Signed-off-by: Kevin Fox <[email protected]> * More updates Signed-off-by: Kevin Fox <[email protected]> * More updates Signed-off-by: Kevin Fox <[email protected]> * More updates Signed-off-by: Kevin Fox <[email protected]> * More updates Signed-off-by: Kevin Fox <[email protected]> * Add x509POP support and more testing Signed-off-by: Kevin Fox <[email protected]> * x509pop attestor support and more tests Signed-off-by: Kevin Fox <[email protected]> * More updates Signed-off-by: Kevin Fox <[email protected]> * Test some more bits Signed-off-by: Kevin Fox <[email protected]> * Test some more bits Signed-off-by: Kevin Fox <[email protected]> * Test some more bits Signed-off-by: Kevin Fox <[email protected]> * Test some more bits Signed-off-by: Kevin Fox <[email protected]> * Test some more bits Signed-off-by: Kevin Fox <[email protected]> * Test some more bits Signed-off-by: Kevin Fox <[email protected]> * Test some more bits Signed-off-by: Kevin Fox <[email protected]> * Test some more bits Signed-off-by: Kevin Fox <[email protected]> * Test some more bits Signed-off-by: Kevin Fox <[email protected]> * Test some more bits Signed-off-by: Kevin Fox <[email protected]> * Test some more bits Signed-off-by: Kevin Fox <[email protected]> * Update docs Signed-off-by: Kevin Fox <[email protected]> * Test some more bits Signed-off-by: Kevin Fox <[email protected]> * Test some more bits Signed-off-by: Kevin Fox <[email protected]> * Test some more bits Signed-off-by: Kevin Fox <[email protected]> * Test some more bits Signed-off-by: Kevin Fox <[email protected]> * Test some more bits Signed-off-by: Kevin Fox <[email protected]> * Test some more bits Signed-off-by: Kevin Fox <[email protected]> * Test some more bits Signed-off-by: Kevin Fox <[email protected]> * Test some more bits Signed-off-by: Kevin Fox <[email protected]> * Test some more bits Signed-off-by: Kevin Fox <[email protected]> * Test some more bits Signed-off-by: Kevin Fox <[email protected]> * Test some more bits Signed-off-by: Kevin Fox <[email protected]> * Test some more bits Signed-off-by: Kevin Fox <[email protected]> * Test some more bits Signed-off-by: Kevin Fox <[email protected]> * Test some more bits Signed-off-by: Kevin Fox <[email protected]> * Test some more bits Signed-off-by: Kevin Fox <[email protected]> * Test some more bits Signed-off-by: Kevin Fox <[email protected]> * Test some more bits Signed-off-by: Kevin Fox <[email protected]> * Test some more bits Signed-off-by: Kevin Fox <[email protected]> * Test some more bits Signed-off-by: Kevin Fox <[email protected]> * Test some more bits Signed-off-by: Kevin Fox <[email protected]> * Test some more bits Signed-off-by: Kevin Fox <[email protected]> * Test some more bits Signed-off-by: Kevin Fox <[email protected]> * Test some more bits Signed-off-by: Kevin Fox <[email protected]> * Test some more bits Signed-off-by: Kevin Fox <[email protected]> * Test some more bits Signed-off-by: Kevin Fox <[email protected]> * Test some more bits Signed-off-by: Kevin Fox <[email protected]> * Test some more bits Signed-off-by: Kevin Fox <[email protected]> * Test some more bits Signed-off-by: Kevin Fox <[email protected]> * Test some more bits Signed-off-by: Kevin Fox <[email protected]> * Test some more bits Signed-off-by: Kevin Fox <[email protected]> * Test some more bits Signed-off-by: Kevin Fox <[email protected]> * Test some more bits Signed-off-by: Kevin Fox <[email protected]> * Test some more bits Signed-off-by: Kevin Fox <[email protected]> * Test some more bits Signed-off-by: Kevin Fox <[email protected]> * Test some more bits Signed-off-by: Kevin Fox <[email protected]> * Test some more bits Signed-off-by: Kevin Fox <[email protected]> * Test some more bits Signed-off-by: Kevin Fox <[email protected]> * Test some more bits Signed-off-by: Kevin Fox <[email protected]> * Test some more bits Signed-off-by: Kevin Fox <[email protected]> * Test some more bits Signed-off-by: Kevin Fox <[email protected]> * Test some more bits Signed-off-by: Kevin Fox <[email protected]> * Test some more bits Signed-off-by: Kevin Fox <[email protected]> * Test some more bits Signed-off-by: Kevin Fox <[email protected]> * Test some more bits Signed-off-by: Kevin Fox <[email protected]> * Test some more bits Signed-off-by: Kevin Fox <[email protected]> * Test some more bits Signed-off-by: Kevin Fox <[email protected]> * Test some more bits Signed-off-by: Kevin Fox <[email protected]> * Test some more bits Signed-off-by: Kevin Fox <[email protected]> * Test some more bits Signed-off-by: Kevin Fox <[email protected]> * Test some more bits Signed-off-by: Kevin Fox <[email protected]> * Test some more bits Signed-off-by: Kevin Fox <[email protected]> * Test some more bits Signed-off-by: Kevin Fox <[email protected]> * Test some more bits Signed-off-by: Kevin Fox <[email protected]> * Test some more bits Signed-off-by: Kevin Fox <[email protected]> * Test some more bits Signed-off-by: Kevin Fox <[email protected]> * Test some more bits Signed-off-by: Kevin Fox <[email protected]> * Test some more bits Signed-off-by: Kevin Fox <[email protected]> * Test some more bits Signed-off-by: Kevin Fox <[email protected]> * Test some more bits Signed-off-by: Kevin Fox <[email protected]> * Test some more bits Signed-off-by: Kevin Fox <[email protected]> * Test some more bits Signed-off-by: Kevin Fox <[email protected]> * Test some more bits Signed-off-by: Kevin Fox <[email protected]> * Test some more bits Signed-off-by: Kevin Fox <[email protected]> * Test some more bits Signed-off-by: Kevin Fox <[email protected]> * Test some more bits Signed-off-by: Kevin Fox <[email protected]> * Test some more bits Signed-off-by: Kevin Fox <[email protected]> * Test some more bits Signed-off-by: Kevin Fox <[email protected]> * Test some more bits Signed-off-by: Kevin Fox <[email protected]> * Test some more bits Signed-off-by: Kevin Fox <[email protected]> * Test some more bits Signed-off-by: Kevin Fox <[email protected]> * Test some more bits Signed-off-by: Kevin Fox <[email protected]> * Test some more bits Signed-off-by: Kevin Fox <[email protected]> * Test some more bits Signed-off-by: Kevin Fox <[email protected]> * Test some more bits Signed-off-by: Kevin Fox <[email protected]> * Test some more bits Signed-off-by: Kevin Fox <[email protected]> * Test some more bits Signed-off-by: Kevin Fox <[email protected]> * Test some more bits Signed-off-by: Kevin Fox <[email protected]> * Fix pages artifact upload Signed-off-by: Kevin Fox <[email protected]> * Test some more bits Signed-off-by: Kevin Fox <[email protected]> * Test some more bits Signed-off-by: Kevin Fox <[email protected]> * Test some more bits Signed-off-by: Kevin Fox <[email protected]> * Test some more bits Signed-off-by: Kevin Fox <[email protected]> * Test some more bits Signed-off-by: Kevin Fox <[email protected]> * Test some more bits Signed-off-by: Kevin Fox <[email protected]> * Test some more bits Signed-off-by: Kevin Fox <[email protected]> * Test some more bits Signed-off-by: Kevin Fox <[email protected]> * Test some more bits Signed-off-by: Kevin Fox <[email protected]> * Initial stab at dynamic registration Signed-off-by: Kevin Fox <[email protected]> * Dynamic registration working but not integrated with test Signed-off-by: Kevin Fox <[email protected]> * Wire in dynamic registration into the test Signed-off-by: Kevin Fox <[email protected]> * Fix missing props Signed-off-by: Kevin Fox <[email protected]> * Update the svids to align Signed-off-by: Kevin Fox <[email protected]> * Update the svids to align Signed-off-by: Kevin Fox <[email protected]> * Fix service name Signed-off-by: Kevin Fox <[email protected]> * Look at data Signed-off-by: Kevin Fox <[email protected]> * Look at data Signed-off-by: Kevin Fox <[email protected]> * Look at data Signed-off-by: Kevin Fox <[email protected]> * Fix ca type Signed-off-by: Kevin Fox <[email protected]> * Test out new packages Signed-off-by: Kevin Fox <[email protected]> * Test out new packages Signed-off-by: Kevin Fox <[email protected]> * Test out new packages Signed-off-by: Kevin Fox <[email protected]> * Test out new packages Signed-off-by: Kevin Fox <[email protected]> * Test out new packages Signed-off-by: Kevin Fox <[email protected]> * Test out new packages Signed-off-by: Kevin Fox <[email protected]> * Test out new packages Signed-off-by: Kevin Fox <[email protected]> * Test out new packages Signed-off-by: Kevin Fox <[email protected]> * Test out new packages Signed-off-by: Kevin Fox <[email protected]> * Test out new packages Signed-off-by: Kevin Fox <[email protected]> * Update ports Signed-off-by: Kevin Fox <[email protected]> * Update ports Signed-off-by: Kevin Fox <[email protected]> * Work on debugging dynamic registration some more Signed-off-by: Kevin Fox <[email protected]> * Fix service account name Signed-off-by: Kevin Fox <[email protected]> * Fix service account name Signed-off-by: Kevin Fox <[email protected]> * Working... Cleanup. Signed-off-by: Kevin Fox <[email protected]> * Working... Cleanup. Signed-off-by: Kevin Fox <[email protected]> * Fix broken ssh test Signed-off-by: Kevin Fox <[email protected]> * Fix broken ssh test Signed-off-by: Kevin Fox <[email protected]> * Simplify a bit Signed-off-by: Kevin Fox <[email protected]> * Update to use the released images Signed-off-by: Kevin Fox <[email protected]> * Allow x509POP cluster name adding Signed-off-by: Kevin Fox <[email protected]> * Restrict cluster registration Signed-off-by: Kevin Fox <[email protected]> * Fix var name Signed-off-by: Kevin Fox <[email protected]> * Fix missing slash Signed-off-by: Kevin Fox <[email protected]> * Make defaults work better Signed-off-by: Kevin Fox <[email protected]> * Make defaults work better Signed-off-by: Kevin Fox <[email protected]> * Fix readme Signed-off-by: Kevin Fox <[email protected]> * updated diagram Signed-off-by: Kevin Fox <[email protected]> * Regenerate image Signed-off-by: Kevin Fox <[email protected]> * Bump spire versions Signed-off-by: Kevin Fox <[email protected]> * Fix issues identified during review Signed-off-by: Kevin Fox <[email protected]> * Update docs Signed-off-by: Kevin Fox <[email protected]> --------- Signed-off-by: Kevin Fox <[email protected]> Signed-off-by: kfox1111 <[email protected]>
2026-06-03 12:15:28 -07:00
parent a40409ec53
commit 1031167b84
38 changed files with 2274 additions and 554 deletions
@@ -0,0 +1,138 @@
+## Kubernetes Bottom Turtle HA Setup
+
+In this setup, a bottom turtle HA setup based on spire-ha-agent and then Kubernetes based access is built from the ground up.
+
+What does this mean?
+
+The bottom turtle:
+There is a pair of spire servers deployed. Trust is established between the two servers creating an HA Trust Domain without needing any 3rd party
+trust sources.
+
+A spire-ha-agent, a spire-agent@a and a spire-agent@b is run on the k8s hosts. This provides a bottom turtle trust source between the services on the os Kubernetes
+runs on.
+
+Host services can then use this trust chain to secure communications such as:
+* kubelet -> kube-apiserver
+* sshd
+* log shipper -> centeralized log processor
+* os level metrics
+* etc
+
+We will not discuss how to do that here, but need to utilize this base to establish trust inside of Kubernetes.
+
+We will bridge os to Kubernetes cluster with some configuration on the host, and deploying the helm charts to utilize and export new services on top.
+
+What do we need to do?
+
+There are two different kinds of services that need permission bridging.
+
+* SPIRE Servers
+* Downstream agents
+
+### Root Servers
+
+Setup a pair of HA root servers as described here:
+https://github.com/spiffe/bootc/tree/main/demo
+
+Root Servers, A and B:
+![Diagram](final-pi5.jpg)
+![Diagram](final-pi5.jpg)
+
+### K8s SPIRE Servers
+
+In the following diagram, we see all the parts involved from getting the K8s SPIRE Servers running on the control plane nodes.
+![Diagram](diagram-cp.png)
+
+We need to be able to use the hosts workload attestors to attest the SPIRE Servers running inside Kubernetes.
+
+To do so, we will define a workload on the root spire servers, and inject it into the spire servers inside Kubernetes.
+
+Example workload definition:
+```
+apiVersion: spire.spiffe.io/v1alpha1
+kind: ClusterStaticEntry
+metadata:
+  name: node1-k8s-spire-server
+spec:
+  parentID: spiffe://${SPIFFE_TRUST_DOMAIN}/node/node1.${SPIFFE_TRUST_DOMAIN}
+  spiffeID: spiffe://${SPIFFE_TRUST_DOMAIN}/k8s-spire-server/server-${SUBINSTANCE}
+  downstream: true
+  selectors:
+  - systemd:id:spiffe-socat-unix@k8s-spire-server-${SUBINSTANCE}.service
+  federatesWith:
+  - spire-ha
+```
+
+And on the host, we install spiffe-socat-unix via packages, and then enable the bridges:
+```
+[email protected]
+[email protected]
+```
+
+Any process that can access the unix socket will be able to become a spire downstream server. Treat this socket with great care.
+
+Consider only doing this on your control plane nodes, and restricting the spire-server to only run on the control plane nodes for extra isolation.
+
+### Downstream agents
+
+In the following diagram we show how a worker node is aranged.
+![Diagram](diagram-worker.png)
+
+We need to be able to use the hosts workload attestors to attest the SPIRE Agents running inside Kubernetes.
+
+To do so, we will define a workload on the root spire servers, and inject it into the spire agents inside Kubernetes.
+
+Example workload definition:
+```
+apiVersion: spire.spiffe.io/v1alpha1
+kind: ClusterStaticEntry
+metadata:
+  name: node1-k8s-spire-agent
+spec:
+  parentID: spiffe://${SPIFFE_TRUST_DOMAIN}/node/node1.${SPIFFE_TRUST_DOMAIN}
+  spiffeID: spiffe://${SPIFFE_TRUST_DOMAIN}/spire-exchange/node1.${SPIFFE_TRUST_DOMAIN}
+  selectors:
+  - systemd:id:spiffe-socat-unix@k8s-spire-agent-${SUBINSTANCE}.service
+```
+
+And on the host, we install spiffe-socat-unix via packages, and then enable the bridges:
+```
+[email protected]
+[email protected]
+```
+
+## Install the charts:
+
+We need to install 4 charts.
+
+* spire crds
+* side A
+* side B
+* the common infrasctructure
+
+This allows upgrading Side A or Side B completely independencly from each other, ensuring if there is a problem it will not affect production.
+
+Setup the spire-values.yaml as needed.
+
+```
+# Install the common components
+helm upgrade --install --create-namespace --namespace spire-mgmt --values "spire-values.yaml" \
+  spire oci://ghcr.io/spiffe/helm-charts/spire-nested \
+  --set tags.haAgentCommon=true \
+  --set "global.spire.namespaces.create=true" \
+  --set "global.spire.ingressControllerType=ingress-nginx" \
+  --set "spiffe-oidc-discovery-provider.ingress.enabled=true"
+
+# Install server side a
+helm upgrade --install --namespace spire-mgmt --values "spire-values.yaml" \
+  --wait spire-a oci://ghcr.io/spiffe/helm-charts/spire-nested \
+  --set tags.bottomTurtleHAA=true \
+  --set "global.spire.ingressControllerType=ingress-nginx"
+
+
+# Install server side b
+helm upgrade --install --namespace spire-mgmt --values "spire-values.yaml" \
+  --wait spire-b oci://ghcr.io/spiffe/helm-charts/spire-nested \
+  --set tags.bottomTurtleHAB=true \
+  --set "global.spire.ingressControllerType=ingress-nginx"
+```
@@ -0,0 +1,136 @@
+digraph G {
+  compound=true;
+  
+  # --- ROOT SERVERS ---
+  subgraph cluster_server1 {
+    label = "node name: n1"
+    style="filled,solid,bold";
+    color="#b3b3b3";
+    fillcolor="#f5f5f5";
+    labeljust="l"; 
+
+    spire_server_1[label=<<table border="0"><tr><td><b>SPIRE Server A</b></td></tr><tr><td align="left"><font point-size="9">NodeAttestors: tpm,x509pop</font></td></tr></table>>,shape="record",style="rounded,solid,filled,bold",color="#6c8ebf",fillcolor="#dae8fc"]
+  }
+
+  subgraph cluster_server2 {
+    label = "node name: n2"
+    style="filled,solid,bold";
+    color="#b3b3b3";
+    fillcolor="#f5f5f5";
+    labeljust="l"; 
+    spire_server_2[label=<<table border="0"><tr><td><b>SPIRE Server B</b></td></tr><tr><td align="left"><font point-size="9">NodeAttestors: tpm,x509pop</font></td></tr></table>>,shape="record",style="rounded,solid,filled,bold",color="#6c8ebf",fillcolor="#dae8fc"]
+  }
+
+  # --- NODE N3 (SPIRE SERVER A PIPELINE) ---
+  subgraph cluster_node3 {
+    label = "node name: n3"
+    style="filled,solid,bold";
+    color="#b3b3b3";
+    fillcolor="#f5f5f5";
+    labeljust="l"; 
+
+    subgraph cluster_node3_systemd {
+      label = "systemd managed"
+      style = "dashed,filled"
+      color="#939393";
+      fillcolor="#d5d5d5";
+      labeljust="l"; 
+
+      spire_agent1[label=<<table border="0"><tr><td><b>SPIRE Agent A</b></td></tr><tr><td align="left"><font point-size="9">NodeAttestor: tpm</font></td></tr><tr><td align="left"><font point-size="9">WorkloadAttestor: systemd</font></td></tr><tr><td align="left"><font point-size="9">systemd: [email protected]</font></td></tr></table>>,shape="record",style="rounded,solid,filled,bold",color="#82b366",fillcolor="#d5e8d4"]
+      spire_agent2_n3[label=<<table border="0"><tr><td><b>SPIRE Agent B</b></td></tr><tr><td align="left"><font point-size="9">NodeAttestor: tpm</font></td></tr><tr><td align="left"><font point-size="9">WorkloadAttestor: systemd</font></td></tr><tr><td align="left"><font point-size="9">systemd: [email protected]</font></td></tr></table>>,shape="record",style="rounded,solid,filled,bold",color="#82b366",fillcolor="#d5e8d4"]
+      
+      subgraph cluster_tb_n3 {
+        label=""
+        style="invis"
+        spire_ha_agent_n3[label=<<table border="0"><tr><td><b>SPIRE HA Agent</b></td></tr><tr><td align="left"><font point-size="9">systemd: [email protected]</font></td></tr></table>>,shape="record",style="rounded,solid,filled,bold",color="#82b366",fillcolor="#d5e8d4"]
+      }
+      
+      sshd1_n3[label="sshd",shape="box",style="rounded,solid,filled,bold",color="#d6b656",fillcolor="#fff2cc"]
+      kubelet1_n3[label="kubelet",shape="box",style="rounded,solid,filled,bold",color="#d6b656",fillcolor="#fff2cc"]
+      k8s_spire_server_a[label=<<table border="0"><tr><td><b>K8s SPIRE Server A Identity</b></td></tr><tr><td align="left"><font point-size="9">systemd: spiffe-socat-unix@k8s-spire-server-a</font></td></tr><tr><td align="left"><font point-size="9">tool: socat</font></td></tr></table>>,shape="box",style="rounded,solid,filled,bold",color="#d79b00",fillcolor="#ffe6cc"]
+    }
+
+    subgraph cluster_node3_k8s {
+      label = "k8s managed"
+      style = "dashed,filled"
+      color="#939393";
+      fillcolor="#d5d5d5";
+      labeljust="l"; 
+
+      k8s_upstream_csi_a[label=<<table border="0"><tr><td><b>Upstream CSI A</b></td></tr><tr><td align="left"><font point-size="9">path: /var/run/spiffe/socat/unix/k8s-spire-server-a/public/agent.sock</font></td></tr></table>>,shape="box",style="rounded,solid,filled,bold",color="#d79b00",fillcolor="#ffe6cc"]
+      spire_server_n1[label=<<table border="0"><tr><td><b>K8s SPIRE Server A</b></td></tr><tr><td align="left"><font point-size="9">NodeAttestors: x509pop</font></td></tr></table>>,shape="record",style="rounded,solid,filled,bold",color="#6c8ebf",fillcolor="#dae8fc"]
+    }
+  }
+
+  # --- NODE N4 (SPIRE SERVER B PIPELINE) ---
+  subgraph cluster_node4 {
+    label = "node name: n4"
+    style="filled,solid,bold";
+    color="#b3b3b3";
+    fillcolor="#f5f5f5";
+    labeljust="l"; 
+
+    subgraph cluster_node4_systemd {
+      label = "systemd managed"
+      style = "dashed,filled"
+      color="#939393";
+      fillcolor="#d5d5d5";
+      labeljust="l"; 
+
+      spire_agent1_n4[label=<<table border="0"><tr><td><b>SPIRE Agent A</b></td></tr><tr><td align="left"><font point-size="9">NodeAttestor: tpm</font></td></tr><tr><td align="left"><font point-size="9">WorkloadAttestor: systemd</font></td></tr><tr><td align="left"><font point-size="9">systemd: [email protected]</font></td></tr></table>>,shape="record",style="rounded,solid,filled,bold",color="#82b366",fillcolor="#d5e8d4"]
+      spire_agent2[label=<<table border="0"><tr><td><b>SPIRE Agent B</b></td></tr><tr><td align="left"><font point-size="9">NodeAttestor: tpm</font></td></tr><tr><td align="left"><font point-size="9">WorkloadAttestor: systemd</font></td></tr><tr><td align="left"><font point-size="9">systemd: [email protected]</font></td></tr></table>>,shape="record",style="rounded,solid,filled,bold",color="#82b366",fillcolor="#d5e8d4"]
+      
+      subgraph cluster_tb_n4 {
+        label=""
+        style="invis"
+        spire_ha_agent_n4[label=<<table border="0"><tr><td><b>SPIRE HA Agent</b></td></tr><tr><td align="left"><font point-size="9">systemd: [email protected]</font></td></tr></table>>,shape="record",style="rounded,solid,filled,bold",color="#82b366",fillcolor="#d5e8d4"]
+      }
+      
+      sshd1_n4[label="sshd",shape="box",style="rounded,solid,filled,bold",color="#d6b656",fillcolor="#fff2cc"]
+      kubelet1_n4[label="kubelet",shape="box",style="rounded,solid,filled,bold",color="#d6b656",fillcolor="#fff2cc"]
+      k8s_spire_server_b[label=<<table border="0"><tr><td><b>K8s SPIRE Server B Identity</b></td></tr><tr><td align="left"><font point-size="9">systemd: spiffe-socat-unix@k8s-spire-server-b</font></td></tr><tr><td align="left"><font point-size="9">tool: socat</font></td></tr></table>>,shape="box",style="rounded,solid,filled,bold",color="#d79b00",fillcolor="#ffe6cc"]
+    }
+
+    subgraph cluster_node4_k8s {
+      label = "k8s managed"
+      style = "dashed,filled"
+      color="#939393";
+      fillcolor="#d5d5d5";
+      labeljust="l"; 
+
+      k8s_upstream_csi_b[label=<<table border="0"><tr><td><b>Upstream CSI B</b></td></tr><tr><td align="left"><font point-size="9">path: /var/run/spiffe/socat/unix/k8s-spire-server-b/public/agent.sock</font></td></tr></table>>,shape="box",style="rounded,solid,filled,bold",color="#d79b00",fillcolor="#ffe6cc"]
+      spire_server_n2[label=<<table border="0"><tr><td><b>K8s SPIRE Server B</b></td></tr><tr><td align="left"><font point-size="9">NodeAttestors: x509pop</font></td></tr></table>>,shape="record",style="rounded,solid,filled,bold",color="#6c8ebf",fillcolor="#dae8fc"]
+    }
+  }
+
+  # --- NETWORKING & LINKS ---
+  
+  # Upstream Core Cross-Links
+  spire_server_1 -> spire_agent1[dir=back]
+  spire_server_1 -> spire_agent1_n4[dir=back]
+  spire_server_2 -> spire_agent2_n3[dir=back]
+  spire_server_2 -> spire_agent2[dir=back]
+
+  # Node 3 Pipelines
+  spire_agent1 -> spire_ha_agent_n3[dir=back]
+  spire_agent2_n3 -> spire_ha_agent_n3[dir=back]
+  spire_ha_agent_n3 -> sshd1_n3[dir=back]
+  spire_ha_agent_n3 -> kubelet1_n3[dir=back]
+  spire_agent1 -> k8s_spire_server_a[dir=back]
+  k8s_spire_server_a -> k8s_upstream_csi_a[dir=back]
+  k8s_upstream_csi_a -> spire_server_n1[dir=back]
+  kubelet1_n3 -> k8s_upstream_csi_a [lhead=cluster_node3_k8s, style=dotted]
+  spire_server_1 -> spire_server_n1 [dir=back]
+
+  # Node 4 Pipelines
+  spire_agent1_n4 -> spire_ha_agent_n4[dir=back]
+  spire_agent2 -> spire_ha_agent_n4[dir=back]
+  spire_ha_agent_n4 -> sshd1_n4[dir=back]
+  spire_ha_agent_n4 -> kubelet1_n4[dir=back]
+  spire_agent2 -> k8s_spire_server_b[dir=back]
+  k8s_spire_server_b -> k8s_upstream_csi_b[dir=back]
+  k8s_upstream_csi_b -> spire_server_n2[dir=back]
+  kubelet1_n4 -> k8s_upstream_csi_b [lhead=cluster_node4_k8s, style=dotted]
+  spire_server_2 -> spire_server_n2 [dir=back]
+}
+
@@ -0,0 +1,150 @@
+digraph G {
+    compound=true;
+  subgraph cluster_server1 {
+      label = "node name: n1"
+      #style = dashed
+        style="filled,solid,bold";
+        color="#b3b3b3";
+        fillcolor="#f5f5f5";
+        labeljust="l"; 
+
+    spire_server_1[label=<<table border="0"><tr><td><b>SPIRE Server A</b></td></tr><tr><td align="left"><font point-size="9">NodeAttestors: tpm</font></td></tr></table>>,shape="record",style="rounded,solid,filled,bold",color="#6c8ebf",fillcolor="#dae8fc"]
+  }
+
+  subgraph cluster_ps1 {
+      label = "Control Plane Node: X"
+      #style = dashed
+        style="filled,solid,bold";
+        color="#b3b3b3";
+        fillcolor="#f5f5f5";
+        labeljust="l"; 
+
+    pod_spire_server_1[label=<<table border="0"><tr><td><b>K8s SPIRE Server A</b></td></tr><tr><td align="left"><font point-size="9">NodeAttestors: x509pop</font></td></tr></table>>,shape="record",style="rounded,solid,filled,bold",color="#6c8ebf",fillcolor="#dae8fc"]
+  }
+
+  subgraph cluster_ps2 {
+      label = "Control Plane Node: Y"
+      #style = dashed
+        style="filled,solid,bold";
+        color="#b3b3b3";
+        fillcolor="#f5f5f5";
+        labeljust="l"; 
+
+    pod_spire_server_2[label=<<table border="0"><tr><td><b>K8s SPIRE Server B</b></td></tr><tr><td align="left"><font point-size="9">NodeAttestors: x509pop</font></td></tr></table>>,shape="record",style="rounded,solid,filled,bold",color="#6c8ebf",fillcolor="#dae8fc"]
+  }
+
+
+  subgraph cluster_server2 {
+
+      label = "node name: n2"
+      #style = dashed
+        style="filled,solid,bold";
+        color="#b3b3b3";
+        fillcolor="#f5f5f5";
+        labeljust="l"; 
+    spire_server_2[label=<<table border="0"><tr><td><b>SPIRE Server B</b></td></tr><tr><td align="left"><font point-size="9">NodeAttestors: tpm</font></td></tr></table>>,shape="record",style="rounded,solid,filled,bold",color="#6c8ebf",fillcolor="#dae8fc"]
+  }
+
+  
+  subgraph cluster_node3 {
+      label = "node name: n3"
+      #style = dashed
+        style="filled,solid,bold";
+        color="#b3b3b3";
+        fillcolor="#f5f5f5";
+        labeljust="l"; 
+
+      subgraph cluster_node1_systemd {
+          #label = "Systemd"
+          label = "systemd managed"
+          style = "dashed,filled"
+          color="#939393";
+          fillcolor="#d5d5d5";
+
+       labeljust="l"; 
+
+          spire_agent1[label=<<table border="0"><tr><td><b>SPIRE Agent A</b></td></tr><tr><td align="left"><font point-size="9">NodeAttestor: tpm</font></td></tr><tr><td align="left"><font point-size="9">WorkloadAttestor: systemd</font></td></tr><tr><td align="left"><font point-size="9">systemd: [email protected]</font></td></tr></table>>,shape="record",style="rounded,solid,filled,bold",color="#82b366",fillcolor="#d5e8d4"]
+          spire_agent2[label=<<table border="0"><tr><td><b>SPIRE Agent B</b></td></tr><tr><td align="left"><font point-size="9">NodeAttestor: tpm</font></td></tr><tr><td align="left"><font point-size="9">WorkloadAttestor: systemd</font></td></tr><tr><td align="left"><font point-size="9">systemd: [email protected]</font></td></tr></table>>,shape="record",style="rounded,solid,filled,bold",color="#82b366",fillcolor="#d5e8d4"]
+    subgraph cluster_tb {
+        label=""
+        style="invis"
+          spire_ha_agent[label=<<table border="0"><tr><td><b>SPIRE HA Agent</b></td></tr><tr><td align="left"><font point-size="9">systemd: [email protected]</font></td></tr></table>>,shape="record",style="rounded,solid,filled,bold",color="#82b366",fillcolor="#d5e8d4"]
+subgraph cluster_storage {
+          #spire_ha_agent_state_a[label="Trust Bundle A", shape=note,style="rounded,solid,filled,bold",fillcolor="#ffffff"]
+          #spire_ha_agent_state_b[label="Trust Bundle B", shape=note,style="rounded,solid,filled,bold",fillcolor="#ffffff"]
+}
+    }
+          sshd1[label="sshd",shape="box",style="rounded,solid,filled,bold",color="#d6b656",fillcolor="#fff2cc"]
+          kubelet1[label="kubelet",shape="box",style="rounded,solid,filled,bold",color="#d6b656",fillcolor="#fff2cc"]
+          k8s_spire_server_a[label=<<table border="0"><tr><td><b>K8s SPIRE Agent A Identity</b></td></tr><tr><td align="left"><font point-size="9">systemd: spiffe-socat-unix@k8s-spire-agent-a</font></td></tr><tr><td align="left"><font point-size="9">tool: socat</font></td></tr></table>>,shape="box",style="rounded,solid,filled,bold",color="#d79b00",fillcolor="#ffe6cc"]
+          k8s_spire_server_b[label=<<table border="0"><tr><td><b>K8s SPIRE Agent B Identity</b></td></tr><tr><td align="left"><font point-size="9">systemd: spiffe-socat-unix@k8s-spire-agent-b</font></td></tr><tr><td align="left"><font point-size="9">tool: socat</font></td></tr></table>>,shape="box",style="rounded,solid,filled,bold",color="#d79b00",fillcolor="#ffe6cc"]
+      }
+
+      subgraph cluster_node1_k8s {
+          #label = "Systemd"
+          label = "k8s managed"
+          style = "dashed,filled"
+          color="#939393";
+          fillcolor="#d5d5d5";
+
+       labeljust="l"; 
+
+  //  k8s_upstream_csi_a[label=<<table border="0"><tr><td><b>Downstream CSI A</b></td></tr><tr><td align="left"><font point-size="9">path: /var/run/spiffe/socat/unix/k8s-spire-server-a/public/agent.sock</font></td></tr></table>>,shape="box",style="rounded,solid,filled,bold",color="#d79b00",fillcolor="#ffe6cc"]
+  //  k8s_upstream_csi_b[label=<<table border="0"><tr><td><b>Downstream CSI B</b></td></tr><tr><td align="left"><font point-size="9">path: /var/run/spiffe/socat/unix/k8s-spire-server-b/public/agent.sock</font></td></tr></table>>,shape="box",style="rounded,solid,filled,bold",color="#d79b00",fillcolor="#ffe6cc"]
+
+    k8s_downstream_csi[label=<<table border="0"><tr><td><b>Downstream CSI</b></td></tr><tr><td align="left"><font point-size="9">path: /var/run/spire/agent-sockets/spire-agent.sock</font></td></tr></table>>,shape="box",style="rounded,solid,filled,bold",color="#d79b00",fillcolor="#ffe6cc"]
+
+    spire_server_n1[label=<<table border="0"><tr><td><b>K8s SPIRE Agent A</b></td></tr><tr><td align="left"><font point-size="9">systemd: [email protected]</font></td></tr></table>>,shape="record",style="rounded,solid,filled,bold",color="#82b366",fillcolor="#d5e8d4"]
+
+    spire_server_n2[label=<<table border="0"><tr><td><b>K8s SPIRE Agent B</b></td></tr><tr><td align="left"><font point-size="9">systemd: [email protected]</font></td></tr></table>>,shape="record",style="rounded,solid,filled,bold",color="#82b366",fillcolor="#d5e8d4"]
+
+
+              spire_ha_agent_pod[label=<<table border="0"><tr><td><b>SPIRE HA Agent</b></td></tr><tr><td align="left"><font point-size="9">systemd: [email protected]</font></td></tr></table>>,shape="record",style="rounded,solid,filled,bold",color="#82b366",fillcolor="#d5e8d4"]
+          pod1[label="pod",shape="box",style="rounded,solid,filled,bold",color="#d6b656",fillcolor="#fff2cc"]
+          pod2[label="pod",shape="box",style="rounded,solid,filled,bold",color="#d6b656",fillcolor="#fff2cc"]
+          pod3[label="pod",shape="box",style="rounded,solid,filled,bold",color="#d6b656",fillcolor="#fff2cc"]
+      }
+  }
+  
+  spire_server_1 -> spire_agent1[dir=back]
+  spire_server_2 -> spire_agent2[dir=back]
+  spire_agent1 -> spire_ha_agent[dir=back]
+  spire_agent2 -> spire_ha_agent[dir=back]
+  spire_ha_agent -> sshd1[dir=back]
+  spire_ha_agent -> kubelet1[dir=back]
+  spire_agent1 -> k8s_spire_server_a[dir=back]
+  spire_agent2 -> k8s_spire_server_b[dir=back]
+
+  spire_server_1 -> pod_spire_server_1[dir=back]
+  spire_server_2 -> pod_spire_server_2[dir=back]
+
+  //k8s_spire_server_a -> k8s_upstream_csi_a[dir=back]
+  k8s_spire_server_a -> spire_server_n1[dir=back]
+  k8s_spire_server_b -> spire_server_n2[dir=back]
+ // k8s_upstream_csi_b -> k8s_spire_server_b
+
+//  k8s_upstream_csi_a -> spire_server_n1[dir=back]
+ // k8s_upstream_csi_b -> spire_server_n2[dir=back]
+//  k8s_spire_server_b -> spire_server_n2[dir=back]
+
+  //kubelet1 -> cluster_node1_k8s
+  kubelet1 -> spire_server_n1 [lhead=cluster_node1_k8s, style=dotted]
+  pod_spire_server_1 -> spire_server_n1 [dir=back]
+  pod_spire_server_2 -> spire_server_n2 [dir=back]
+  //kubelet1 -> spire_server_n1
+  //kubelet1 -> spire_server_n2
+   spire_server_n1 -> spire_ha_agent_pod [dir=back]
+   spire_server_n2 -> spire_ha_agent_pod [dir=back]
+
+   spire_ha_agent_pod -> k8s_downstream_csi [dir=back]
+    k8s_downstream_csi -> pod1 [dir=back]
+    k8s_downstream_csi -> pod2 [dir=back]
+    k8s_downstream_csi -> pod3 [dir=back]
+//  spire_ha_agent -> spire_ha_agent_state[dir=both, constraint=false]
+ // spire_ha_agent_state_a -> spire_ha_agent_state_b 
+  //spire_agent1 -> spire_ha_agent_state_a
+  //spire_agent2 -> spire_ha_agent_state_b
+  //spire_ha_agent_state_a -> spire_ha_agent
+  //spire_ha_agent_state_b -> spire_ha_agent
+}
+
@@ -0,0 +1,12 @@
+apiVersion: spire.spiffe.io/v1alpha1
+kind: ClusterStaticEntry
+metadata:
+  name: node1-k8s-spire-server
+spec:
+  parentID: spiffe://${SPIFFE_TRUST_DOMAIN}/agent/node1
+  spiffeID: spiffe://${SPIFFE_TRUST_DOMAIN}/k8s-spire-server/node1
+  selectors:
+  - systemd:id:spiffe-socat-unix@k8s-spire-server-${SUBINSTANCE}.service
+  downstream: true
+  federatesWith:
+  - spire-ha
@@ -0,0 +1,9 @@
+apiVersion: spire.spiffe.io/v1alpha1
+kind: ClusterStaticEntry
+metadata:
+  name: node1-spire-trust-sync-a
+spec:
+  parentID: spiffe://${SPIFFE_TRUST_DOMAIN}/agent/node1
+  spiffeID: spiffe://${SPIFFE_TRUST_DOMAIN}/spire-trust-sync-a
+  selectors:
+  - systemd:id:[email protected]
@@ -0,0 +1,9 @@
+apiVersion: spire.spiffe.io/v1alpha1
+kind: ClusterStaticEntry
+metadata:
+  name: node1-spire-trust-sync-b
+spec:
+  parentID: spiffe://${SPIFFE_TRUST_DOMAIN}/agent/node1
+  spiffeID: spiffe://${SPIFFE_TRUST_DOMAIN}/spire-trust-sync-b
+  selectors:
+  - systemd:id:[email protected]
@@ -0,0 +1,9 @@
+apiVersion: spire.spiffe.io/v1alpha1
+kind: ClusterStaticEntry
+metadata:
+  name: node2-k8s-spire-server
+spec:
+  parentID: spiffe://${SPIFFE_TRUST_DOMAIN}/agent/node1
+  spiffeID: spiffe://${SPIFFE_TRUST_DOMAIN}/spire-exchange/k8s/production/node2.${SPIFFE_TRUST_DOMAIN}
+  selectors:
+  - systemd:id:spiffe-socat-unix@k8s-spire-agent-2-${SUBINSTANCE}.service
@@ -0,0 +1,9 @@
+apiVersion: spire.spiffe.io/v1alpha1
+kind: ClusterStaticEntry
+metadata:
+  name: node3-k8s-spire-server
+spec:
+  parentID: spiffe://${SPIFFE_TRUST_DOMAIN}/agent/node1
+  spiffeID: spiffe://${SPIFFE_TRUST_DOMAIN}/spire-exchange/k8s/production/node3.${SPIFFE_TRUST_DOMAIN}
+  selectors:
+  - systemd:id:spiffe-socat-unix@k8s-spire-agent-3-${SUBINSTANCE}.service
@@ -0,0 +1,9 @@
+apiVersion: spire.spiffe.io/v1alpha1
+kind: ClusterStaticEntry
+metadata:
+  name: node4-k8s-spire-server
+spec:
+  parentID: spiffe://${SPIFFE_TRUST_DOMAIN}/agent/node1
+  spiffeID: spiffe://${SPIFFE_TRUST_DOMAIN}/spire-exchange/k8s/production/node4.${SPIFFE_TRUST_DOMAIN}
+  selectors:
+  - systemd:id:spiffe-socat-unix@k8s-spire-agent-4-${SUBINSTANCE}.service
@@ -0,0 +1,293 @@
+#!/usr/bin/env bash
+# shellcheck disable=SC2317
+
+set -xe
+
+SCRIPT="$(readlink -f "$0")"
+SCRIPTPATH="$(dirname "${SCRIPT}")"
+TESTDIR="${SCRIPTPATH}/../../.github/tests"
+#DEPS="${TESTDIR}/dependencies"
+
+# shellcheck source=/dev/null
+source "${SCRIPTPATH}/../../.github/scripts/parse-versions.sh"
+# shellcheck source=/dev/null
+source "${TESTDIR}/common.sh"
+
+CLEANUP=1
+
+for i in "$@"; do
+  case $i in
+    -c)
+      CLEANUP=0
+      shift # past argument=value
+      ;;
+  esac
+done
+
+if [ "x${GITHUB_JOB}" != "x" ]; then
+  echo "Running in GitHub"
+else
+  echo "Do not run this script on your own box. For testing, it deploys a testing local spire ha setup using sudo. This is likely not what you want. Only use this script as a reference."
+  exit 1
+fi
+
+teardown() {
+  echo ---------------------------
+  docker exec -i chart-testing-worker /bin/bash -c "more /var/lib/kubelet/pods/*/volumes/kubernetes.io~empty-dir/disk-keymanager/keys.json /var/lib/kubelet/pods/*/volumes/kubernetes.io~empty-dir/spire-agent-persistence/agent-data.json | cat"
+  sudo systemctl status spire-server@a || true
+  sudo systemctl status spire-server@b || true
+  sudo spire-server entry show -instance a || true
+  sudo spire-server entry show -instance b || true
+  sudo systemctl status spire-controller-manager@a || true
+  sudo systemctl status spire-controller-manager@b || true
+  sudo systemctl status spire-agent@a || true
+  sudo systemctl status spire-agent@b || true
+  sudo systemctl status spire-trust-sync@a || true
+  sudo systemctl status spire-trust-sync@b || true
+  sudo systemctl status spiffe-socat-unix@k8s-spire-server-a || true
+  sudo systemctl status spiffe-socat-unix@k8s-spire-server-b || true
+  sudo systemctl status spiffe-socat-unix@k8s-spire-agent-2-a || true
+  sudo systemctl status spiffe-socat-unix@k8s-spire-agent-2-b || true
+  sudo systemctl status spiffe-socat-unix@k8s-spire-agent-3-a || true
+  sudo systemctl status spiffe-socat-unix@k8s-spire-agent-3-b || true
+  sudo systemctl status spiffe-socat-unix@k8s-spire-agent-4-a || true
+  sudo systemctl status spiffe-socat-unix@k8s-spire-agent-4-b || true
+  sudo spire-server bundle list -socketPath /var/run/spire/server/sockets/a/private/api.sock || true
+  sudo spire-server bundle list -socketPath /var/run/spire/server/sockets/b/private/api.sock || true
+  kubectl exec -i -n spire-server spire-a-internal-server-0 -- spire-server entry show || true
+  kubectl exec -i -n spire-server spire-b-internal-server-0 -- spire-server entry show || true
+  kubectl exec -i -n spire-server spire-a-internal-server-0 -- spire-server agent list -output json | yq e . - -P || true
+  kubectl exec -i -n spire-server spire-b-internal-server-0 -- spire-server agent list -output json | yq e . - -P || true
+
+  print_helm_releases
+
+  if [[ "$1" -ne 0 ]]; then
+    get_namespace_details spire-server spire-system
+    kubectl describe pod -n spire-system
+  fi
+
+  if [ "${CLEANUP}" -eq 1 ]; then
+    helm uninstall --namespace spire-mgmt spire-b 2>/dev/null || true
+    helm uninstall --namespace spire-mgmt spire-a 2>/dev/null || true
+    helm uninstall --namespace spire-mgmt spire 2>/dev/null || true
+    kubectl delete ns spire-server 2>/dev/null || true
+    kubectl delete ns spire-system 2>/dev/null || true
+    kubectl delete ns spire-mgmt 2>/dev/null || true
+  fi
+}
+
+trap 'EC=$? && trap - SIGTERM && teardown $EC' SIGINT SIGTERM EXIT
+
+wait_for_healthcheck() {
+  local app="$1"
+  local socket="$2"
+  local timeout=30
+  local count=0
+  while [ "$count" -lt "$timeout" ]; do
+    rc=0
+    sudo "$app" healthcheck -socketPath "$socket" || rc=$?
+    if [ "$rc" -eq 0 ]; then
+      return 0
+    fi
+    sleep 1
+    ((count++)) || true
+  done
+  return 1
+}
+
+wait_for_trust_sync() {
+  local socket="$1"
+  local timeout=30
+  local count=0
+  while [ "$count" -lt "$timeout" ]; do
+    entries=$(sudo spire-server bundle list -socketPath "$socket" | wc -l)
+    if [ "$entries" -ne 0 ]; then
+      return 0
+    fi
+    sleep 1
+    ((count++)) || true
+  done
+  return 1
+}
+
+wait_for_jwt() {
+  local socket="$1"
+  local timeout=30
+  local count=0
+  while [ "$count" -lt "$timeout" ]; do
+      rc=0
+      sudo spire-agent api fetch jwt -audience test -socketPath "$socket" || rc=$?
+      if [ "$rc" -eq 0 ]; then
+        return 0
+      fi
+      sleep 1
+      ((count++)) || true
+  done
+  return 1
+}
+
+"${SCRIPTPATH}/../../.github/scripts/prepare-local-chart-deps.sh"
+
+# Get the package repo and install the packages
+sudo curl -s -o /etc/apt/sources.list.d/spire-examples.list https://raw.githubusercontent.com/spiffe/spire-examples/refs/heads/main/examples/debs/amd64/spire-examples.list
+sudo apt-get update
+sudo apt-get install -y spire-common spire-agent spire-server spire-controller-manager spiffe-socat-unix socat spire-trust-sync spiffe-helper
+
+# Set our testing trust domain
+sudo sed -i 's/example.org/production.other/' /etc/spiffe/default-trust-domain.env
+
+# register some workloads with the spire server using manifests
+sudo mkdir -p /etc/spire/server/a/manifests/ /etc/spire/server/b/manifests/
+sudo cp "${SCRIPTPATH}/example-manifests"/* /etc/spire/server/a/manifests/
+sudo cp "${SCRIPTPATH}/example-manifests"/* /etc/spire/server/b/manifests/
+
+# For testing, help speed up the sync
+sudo rm -f /etc/spire/server/a/manifests/node1-k8s-spire-server.yaml
+sudo rm -f /etc/spire/server/b/manifests/node1-k8s-spire-server.yaml
+
+# Since we are running the two root spire servers on the same machine, we need to ensure ports do not conflict for server b
+sudo /bin/bash -c 'echo SPIRE_BIND_PORT=8082 > /etc/spire/server/b.env'
+sudo /bin/bash -c '(echo METRICS_BIND_ADDRESS="0.0.0.0:9125"; echo HEALTH_PROBE_BIND_ADDRESS="0.0.0.0:9126") > /etc/spire/controller-manager/b.env'
+
+# Startup servers and make sure they are ready
+sudo systemctl start spire-server@a spire-server@b spire-controller-manager@a spire-controller-manager@b
+wait_for_healthcheck spire-server /run/spire/server/sockets/a/private/api.sock
+wait_for_healthcheck spire-server /run/spire/server/sockets/b/private/api.sock
+
+# Configure our agents. For the test, create join tokens for both agents. You should really use a node attestor other then join tokens such as tpm-direct, http_challenge, or a cloud provider one
+JOIN_TOKEN_A=$(sudo spire-server token generate -spiffeID spiffe://production.other/agent/node1 -socketPath /run/spire/server/sockets/a/private/api.sock | awk '{print "\""$2"\""}')
+JOIN_TOKEN_B=$(sudo spire-server token generate -spiffeID spiffe://production.other/agent/node1 -socketPath /run/spire/server/sockets/b/private/api.sock | awk '{print "\""$2"\""}')
+export JOIN_TOKEN_A
+export JOIN_TOKEN_B
+sudo /bin/bash -c "echo JOIN_TOKEN=${JOIN_TOKEN_A} > /etc/spire/agent/a.env"
+sudo /bin/bash -c "echo JOIN_TOKEN=${JOIN_TOKEN_B} > /etc/spire/agent/b.env"
+sudo /bin/bash -c "echo SPIRE_SERVER_PORT=8082 >> /etc/spire/agent/b.env"
+
+# Since we are running the two root spire servers on the same machine, we need to configure the trust sync instances to point to the opposite server
+sudo /bin/bash -c 'echo "SPIRE_SERVER_SOCKET=/var/run/spire/server/sockets/b/private/api.sock" > /etc/spire/trust-sync/a.conf'
+sudo /bin/bash -c 'echo "SPIRE_SERVER_SOCKET=/var/run/spire/server/sockets/a/private/api.sock" > /etc/spire/trust-sync/b.conf'
+
+# Startup the agent
+sudo systemctl start spire-agent@a spire-agent@b
+sudo systemctl start spire-trust-sync@a spire-trust-sync@b
+wait_for_healthcheck spire-agent /var/run/spire/agent/sockets/a/public/api.sock
+wait_for_healthcheck spire-agent /var/run/spire/agent/sockets/b/public/api.sock
+wait_for_trust_sync /var/run/spire/server/sockets/a/private/api.sock
+wait_for_trust_sync /var/run/spire/server/sockets/b/private/api.sock
+
+sudo cp "${SCRIPTPATH}/example-manifests"/node1-k8s-spire-server.yaml /etc/spire/server/a/manifests/
+sudo cp "${SCRIPTPATH}/example-manifests"/node1-k8s-spire-server.yaml /etc/spire/server/b/manifests/
+
+# Startup the socat bridge to allow the k8s spire servers to get an identity/trust bundles from the host
+sudo systemctl start spiffe-socat-unix@k8s-spire-server-a spiffe-socat-unix@k8s-spire-server-b
+wait_for_healthcheck spire-agent /var/run/spiffe/socat/unix/k8s-spire-server-a/public/spire-agent.sock
+wait_for_healthcheck spire-agent /var/run/spiffe/socat/unix/k8s-spire-server-b/public/spire-agent.sock
+wait_for_jwt /var/run/spiffe/socat/unix/k8s-spire-server-a/public/spire-agent.sock
+wait_for_jwt /var/run/spiffe/socat/unix/k8s-spire-server-b/public/spire-agent.sock
+
+# Configure and start up the socat bridges to allow the k8s spire-agents to get an identity/trust bundles from the host.
+# We only have one vm mapped to multiple k8s virtual nodes in kind, so we run a pair per k8s virtual node. Normally you would only run one pair per host/vm.
+sudo /bin/bash -c "echo SPIFFE_INSTANCE=a > /etc/spiffe/socat/unix/k8s-spire-agent-2-a.conf"
+sudo /bin/bash -c "echo SPIFFE_INSTANCE=a > /etc/spiffe/socat/unix/k8s-spire-agent-3-a.conf"
+sudo /bin/bash -c "echo SPIFFE_INSTANCE=a > /etc/spiffe/socat/unix/k8s-spire-agent-4-a.conf"
+sudo /bin/bash -c "echo SPIFFE_INSTANCE=b > /etc/spiffe/socat/unix/k8s-spire-agent-2-b.conf"
+sudo /bin/bash -c "echo SPIFFE_INSTANCE=b > /etc/spiffe/socat/unix/k8s-spire-agent-3-b.conf"
+sudo /bin/bash -c "echo SPIFFE_INSTANCE=b > /etc/spiffe/socat/unix/k8s-spire-agent-4-b.conf"
+sudo systemctl start spiffe-socat-unix@k8s-spire-agent-2-a spiffe-socat-unix@k8s-spire-agent-2-b
+sudo systemctl start spiffe-socat-unix@k8s-spire-agent-3-a spiffe-socat-unix@k8s-spire-agent-3-b
+sudo systemctl start spiffe-socat-unix@k8s-spire-agent-4-a spiffe-socat-unix@k8s-spire-agent-4-b
+wait_for_healthcheck spire-agent /var/run/spiffe/socat/unix/k8s-spire-agent-2-a/public/api.sock
+wait_for_healthcheck spire-agent /var/run/spiffe/socat/unix/k8s-spire-agent-2-b/public/api.sock
+wait_for_healthcheck spire-agent /var/run/spiffe/socat/unix/k8s-spire-agent-3-a/public/api.sock
+wait_for_healthcheck spire-agent /var/run/spiffe/socat/unix/k8s-spire-agent-3-b/public/api.sock
+wait_for_healthcheck spire-agent /var/run/spiffe/socat/unix/k8s-spire-agent-4-a/public/api.sock
+wait_for_healthcheck spire-agent /var/run/spiffe/socat/unix/k8s-spire-agent-4-b/public/api.sock
+wait_for_jwt /var/run/spiffe/socat/unix/k8s-spire-agent-2-a/public/api.sock
+wait_for_jwt /var/run/spiffe/socat/unix/k8s-spire-agent-2-b/public/api.sock
+wait_for_jwt /var/run/spiffe/socat/unix/k8s-spire-agent-3-a/public/api.sock
+wait_for_jwt /var/run/spiffe/socat/unix/k8s-spire-agent-3-b/public/api.sock
+wait_for_jwt /var/run/spiffe/socat/unix/k8s-spire-agent-4-a/public/api.sock
+wait_for_jwt /var/run/spiffe/socat/unix/k8s-spire-agent-4-b/public/api.sock
+
+# Deploy an ingress controller
+IP=$(kubectl get nodes chart-testing-control-plane -o go-template='{{ range .status.addresses }}{{ if eq .type "InternalIP" }}{{ .address }}{{ end }}{{ end }}')
+helm upgrade --install ingress-nginx ingress-nginx --version "$VERSION_INGRESS_NGINX" --repo "$HELM_REPO_INGRESS_NGINX" \
+  --namespace ingress-nginx \
+  --create-namespace \
+  --set "controller.extraArgs.enable-ssl-passthrough=,controller.admissionWebhooks.enabled=false,controller.service.type=ClusterIP,controller.service.externalIPs[0]=$IP" \
+  --set controller.ingressClassResource.default=true \
+  --wait
+
+# Test the ingress controller. Should 404 as there is no services yet.
+common_test_url "$IP"
+
+# Get the host IP And add spire-server-[ab].${trust_domain} records to it so the spire-servers can talk back to root servers running on the host
+HOSTIP=$(ip addr show docker0 | grep 'inet ' | awk '{print $2}' | cut -d/ -f1)
+kubectl get configmap -n kube-system coredns -o yaml | grep hosts || kubectl get configmap -n kube-system coredns -o yaml | sed "/ready/a\        hosts {\n           fallthrough\n        }" | kubectl apply -f -
+kubectl get configmap -n kube-system coredns -o yaml | grep production.other || kubectl get configmap -n kube-system coredns -o yaml | sed "/hosts/a\           $HOSTIP spire-server-a.production.other\n           $HOSTIP oidc-discovery.production.other\n           $HOSTIP spire-server-b.production.other\n" | kubectl apply -f -
+kubectl rollout restart -n kube-system deployment/coredns
+kubectl rollout status -n kube-system -w --timeout=1m deploy/coredns
+
+# Install the common components
+helm upgrade --install --create-namespace --namespace spire-mgmt --values "${COMMON_TEST_YOUR_VALUES},${SCRIPTPATH}/spire-values.yaml" \
+  spire charts/spire-nested \
+  --set tags.haAgentCommon=true \
+  --set "global.spire.namespaces.create=true" \
+  --set "global.spire.ingressControllerType=ingress-nginx" \
+  --set "spiffe-oidc-discovery-provider.ingress.enabled=true"
+
+# Install server side a
+helm upgrade --install --namespace spire-mgmt --values "${COMMON_TEST_YOUR_VALUES},${SCRIPTPATH}/spire-values.yaml" \
+  --wait spire-a charts/spire-nested \
+  --set tags.bottomTurtleHAA=true \
+  --set "global.spire.ingressControllerType=ingress-nginx"
+
+docker exec -i chart-testing-worker /bin/bash -c "more /var/lib/kubelet/pods/*/volumes/kubernetes.io~empty-dir/disk-keymanager/keys.json /var/lib/kubelet/pods/*/volumes/kubernetes.io~empty-dir/spire-agent-persistence/agent-data.json | cat"
+
+# Rollout just to sped up the tests
+kubectl patch deployment spiffe-oidc-discovery-provider -n spire-server --type='strategic' -p '{"spec": {"strategy": {"type": "Recreate", "rollingUpdate": null}}}'
+kubectl rollout restart daemonset -n spire-system spire-ha-agent
+kubectl rollout status daemonset -n spire-system spire-ha-agent
+kubectl rollout restart deployment -n spire-server spiffe-oidc-discovery-provider
+kubectl rollout status deployment -n spire-server spiffe-oidc-discovery-provider --timeout=1m
+kubectl wait -n spire-server --for=condition=ready pod -l "app.kubernetes.io/name=spiffe-oidc-discovery-provider" --field-selector=status.phase=Running --timeout=90s
+curl -k --resolve "oidc-discovery.production.other:443:$IP" "https://oidc-discovery.production.other/.well-known/openid-configuration" -s --fail
+
+# Install server side b
+helm upgrade --install --namespace spire-mgmt --values "${COMMON_TEST_YOUR_VALUES},${SCRIPTPATH}/spire-values.yaml" \
+  --wait spire-b charts/spire-nested \
+  --set tags.bottomTurtleHAB=true \
+  --set internal-spire-server-bottom-turtle-ha-b.upstreamAuthority.spire.server.port=8082 \
+  --set "global.spire.ingressControllerType=ingress-nginx"
+
+docker ps
+docker exec -i chart-testing-worker /bin/bash -c "more /var/lib/kubelet/pods/*/volumes/kubernetes.io~empty-dir/disk-keymanager/keys.json /var/lib/kubelet/pods/*/volumes/kubernetes.io~empty-dir/spire-agent-persistence/agent-data.json | cat"
+
+# From here on out, we sanity check that everything is working properly with both servers running.
+
+ENTRIES="$(kubectl exec -i -n spire-server spire-b-internal-server-0 -- spire-server entry show)"
+if [[ "${ENTRIES}" == "Found 0 entries" ]]; then
+  echo "${ENTRIES}"
+  exit 1
+fi
+
+ENTRIES="$(kubectl exec -i -n spire-server spire-a-internal-server-0 -- spire-server entry show)"
+if [[ "${ENTRIES}" == "Found 0 entries" ]]; then
+  echo "${ENTRIES}"
+  exit 1
+fi
+
+kubectl get pods -A -o wide
+
+helm test --namespace spire-mgmt spire-a
+helm test --namespace spire-mgmt spire-b
+curl -k --resolve "oidc-discovery.production.other:443:$IP" "https://oidc-discovery.production.other/.well-known/openid-configuration" -s --fail
+
+#Test out running only on side b since we know already only both servers work together, and that only side a works if we made it this far.
+helm delete -n spire-mgmt spire-a
+kubectl rollout restart daemonset -n spire-system spire-ha-agent
+kubectl rollout status daemonset -n spire-system spire-ha-agent
+kubectl rollout restart deployment -n spire-server spiffe-oidc-discovery-provider
+kubectl rollout status deployment -n spire-server spiffe-oidc-discovery-provider --timeout=1m
+curl -k --resolve "oidc-discovery.production.other:443:$IP" "https://oidc-discovery.production.other/.well-known/openid-configuration" -s --fail
+
@@ -0,0 +1,13 @@
+global:
+  spire:
+    recommendations:
+      enabled: true
+    namespaces:
+      create: false
+    #ingressControllerType: ""
+    #clusterName: example-cluster
+    #trustDomain: example.org
+    #caSubject:
+    #  country: ""
+    #  organization: ""
+    #  commonName: ""