From f4c7df239e97d08f47f5b75df1bf95a518354cc7 Mon Sep 17 00:00:00 2001 From: kfox1111 Date: Thu, 30 Jul 2026 13:15:55 -0700 Subject: [PATCH] Add broker suport to the spire-ha-agent (#884) * Add broker suport to the spire-ha-agent Signed-off-by: Kevin Fox --------- Signed-off-by: Kevin Fox Signed-off-by: kfox1111 --------- Signed-off-by: Kevin Fox Signed-off-by: kfox1111 Co-authored-by: Faisal Memon --- .github/workflows/helm-chart-ci.yaml | 2 +- charts/spire-ha-agent/Chart.yaml | 2 +- charts/spire-ha-agent/README.md | 13 +- .../spire-ha-agent/templates/daemonset.yaml | 59 ++++++++- charts/spire-ha-agent/values.yaml | 27 +++- charts/spire-nested/README.md | 6 + charts/spire-nested/values.yaml | 28 +++++ .../bottom-turtle-ha/federation-test-job.yaml | 74 +++++++++++ examples/bottom-turtle-ha/run-tests.sh | 117 +++++++++++++++++- .../bottom-turtle-ha-broker/run-tests.sh | 9 ++ 10 files changed, 322 insertions(+), 15 deletions(-) create mode 100644 examples/bottom-turtle-ha/federation-test-job.yaml create mode 100755 tests/integration/bottom-turtle-ha-broker/run-tests.sh diff --git a/.github/workflows/helm-chart-ci.yaml b/.github/workflows/helm-chart-ci.yaml index 22aeecf..083939d 100644 --- a/.github/workflows/helm-chart-ci.yaml +++ b/.github/workflows/helm-chart-ci.yaml @@ -28,7 +28,7 @@ concurrency: env: HELM_VERSION: v3.16.2 PYTHON_VERSION: 3.11.3 - KIND_VERSION: v0.24.0 + KIND_VERSION: v0.32.0 CHART_TESTING_VERSION: v3.8.0 jobs: diff --git a/charts/spire-ha-agent/Chart.yaml b/charts/spire-ha-agent/Chart.yaml index 6a1f5cb..99077c0 100644 --- a/charts/spire-ha-agent/Chart.yaml +++ b/charts/spire-ha-agent/Chart.yaml @@ -3,7 +3,7 @@ name: spire-ha-agent description: A Helm chart to install the SPIRE HA agent. type: application version: 0.2.0 -appVersion: "0.0.17" +appVersion: "0.2.0" keywords: ["spiffe", "spire-ha-agent"] home: https://github.com/spiffe/helm-charts-hardened/tree/main/charts/spire-ha-agent sources: diff --git a/charts/spire-ha-agent/README.md b/charts/spire-ha-agent/README.md index d261ab5..0130833 100644 --- a/charts/spire-ha-agent/README.md +++ b/charts/spire-ha-agent/README.md @@ -30,10 +30,17 @@ A Helm chart to install the SPIRE HA agent. | `image.repository` | The repository within the registry | `spiffe/spire-ha-agent` | | `image.pullPolicy` | The image pull policy | `IfNotPresent` | | `image.tag` | Overrides the image tag whose default is the chart appVersion | `""` | +| `mode` | If the spire-ha-agent will run in delegated or broker mode | `delegated` | | `singleSocket` | If in singleSocket mode, only one driver is used | `false` | -| `sockets.single.admin.hostPath` | Where the sockets are on disk when in single socket mode | `/var/run/spire/agent/sockets/main/csi.spiffe.io/admin` | -| `sockets.a.admin.hostPath` | Where the sockets are on disk | `/var/run/spire/agent/sockets/a/csi.spiffe.io/admin` | -| `sockets.b.admin.hostPath` | Where the sockets are on disk | `/var/run/spire/agent/sockets/b/csi.spiffe.io/admin` | +| `sockets.single.admin.hostPath` | Where the admin socket is on disk when in single socket mode | `/var/run/spire/agent/sockets/main/csi.spiffe.io/admin` | +| `sockets.a.admin.hostPath` | Where the a admin socket is on disk | `/var/run/spire/agent/sockets/a/csi.spiffe.io/admin` | +| `sockets.b.admin.hostPath` | Where the b admin sockets is on disk | `/var/run/spire/agent/sockets/b/csi.spiffe.io/admin` | +| `sockets.single.broker.hostPath` | Where the broker socket is on disk when in single socket mode | `/var/run/spire/agent/sockets/main/csi.spiffe.io/broker` | +| `sockets.a.broker.hostPath` | Where the a broker socket is on disk | `/var/run/spire/agent/sockets/a/csi.spiffe.io/broker` | +| `sockets.b.broker.hostPath` | Where the b broker socket is on disk | `/var/run/spire/agent/sockets/b/csi.spiffe.io/broker` | +| `sockets.single.workload.hostPath` | Where the broker socket is on disk when in single socket mode | `/var/run/spire/agent-sockets` | +| `sockets.a.workload.hostPath` | Where the a workload socket is on disk | `/var/run/spire/agent/sockets/a/csi.spiffe.io/public` | +| `sockets.b.workload.hostPath` | Where the b workload socket is on disk | `/var/run/spire/agent/sockets/b/csi.spiffe.io/public` | | `vsock` | Use a vsockets to expose the service rather then a unix socket | `false` | | `port` | Port number to listen on | `999` | | `imagePullSecrets` | Pull secrets for images | `[]` | diff --git a/charts/spire-ha-agent/templates/daemonset.yaml b/charts/spire-ha-agent/templates/daemonset.yaml index 0708792..46f542f 100644 --- a/charts/spire-ha-agent/templates/daemonset.yaml +++ b/charts/spire-ha-agent/templates/daemonset.yaml @@ -119,6 +119,9 @@ spec: - name: {{ .Chart.Name | quote }} image: {{ template "spire-lib.image" (dict "appVersion" $.Chart.AppVersion "image" .Values.image "global" .Values.global) }} imagePullPolicy: {{ .Values.image.pullPolicy | quote }} + args: + - "-mode" + - {{ .Values.mode | quote }} securityContext: privileged: true #FIXME read permission to api socket @@ -131,13 +134,31 @@ spec: {{- if .Values.singleSocket }} - name: SPIRE_HA_AGENT_SINGLE value: enabled + {{- if eq .Values.mode "delegated" }} - name: SPIRE_HA_AGENT_SOCKET value: unix:///var/run/spire/agent/sockets/a/csi.spiffe.io/admin/admin.sock + {{- else }} + - name: SPIRE_HA_AGENT_BROKER + value: unix:///var/run/spire/agent/sockets/a/csi.spiffe.io/broker/broker.sock + - name: SPIRE_HA_AGENT_WORKLOAD_SOCKET + value: unix:///var/run/spire/agent/sockets/a/csi.spiffe.io/public/spire-agent.sock + {{- end }} {{- else }} + {{- if eq .Values.mode "delegated" }} - name: SPIRE_HA_AGENT_SOCKET_A value: unix:///var/run/spire/agent/sockets/a/csi.spiffe.io/admin/admin.sock - name: SPIRE_HA_AGENT_SOCKET_B value: unix:///var/run/spire/agent/sockets/b/csi.spiffe.io/admin/admin.sock + {{- else }} + - name: SPIRE_HA_AGENT_BROKER_A + value: unix:///var/run/spire/agent/sockets/a/csi.spiffe.io/broker/broker.sock + - name: SPIRE_HA_AGENT_BROKER_B + value: unix:///var/run/spire/agent/sockets/b/csi.spiffe.io/broker/broker.sock + - name: SPIRE_HA_AGENT_WORKLOAD_SOCKET_A + value: unix:///var/run/spire/agent/sockets/a/csi.spiffe.io/public/spire-agent.sock + - name: SPIRE_HA_AGENT_WORKLOAD_SOCKET_B + value: unix:///var/run/spire/agent/sockets/b/csi.spiffe.io/public/spire-agent.sock + {{- end }} {{- end }} {{- if .Values.vsock }} - name: SPIRE_HA_AGENT_VSOCK @@ -161,11 +182,24 @@ spec: mountPath: /tmp/spire-ha-agent/public readOnly: false {{- end }} + {{- if eq .Values.mode "delegated" }} - name: spire-ha-admin-socket-dir-upstream-a mountPath: /var/run/spire/agent/sockets/a/csi.spiffe.io/admin - {{- if not .Values.singleSocket }} + {{- if not .Values.singleSocket }} - name: spire-ha-admin-socket-dir-upstream-b mountPath: /var/run/spire/agent/sockets/b/csi.spiffe.io/admin + {{- end }} + {{- else }} + - name: spire-ha-broker-socket-dir-upstream-a + mountPath: /var/run/spire/agent/sockets/a/csi.spiffe.io/broker + - name: spire-ha-workload-socket-dir-upstream-a + mountPath: /var/run/spire/agent/sockets/a/csi.spiffe.io/public + {{- if not .Values.singleSocket }} + - name: spire-ha-broker-socket-dir-upstream-b + mountPath: /var/run/spire/agent/sockets/b/csi.spiffe.io/broker + - name: spire-ha-workload-socket-dir-upstream-b + mountPath: /var/run/spire/agent/sockets/b/csi.spiffe.io/public + {{- end }} {{- end }} - name: dev mountPath: /dev @@ -196,15 +230,36 @@ spec: path: {{ $socketPath | dir }} type: DirectoryOrCreate {{- end }} + {{- if eq .Values.mode "delegated" }} - name: spire-ha-admin-socket-dir-upstream-a hostPath: path: {{ if .Values.singleSocket }}{{ .Values.sockets.single.admin.hostPath }}{{ else }}{{ .Values.sockets.a.admin.hostPath }}{{ end }} type: DirectoryOrCreate - {{- if not .Values.singleSocket }} + {{- if not .Values.singleSocket }} - name: spire-ha-admin-socket-dir-upstream-b hostPath: path: {{ .Values.sockets.b.admin.hostPath }} type: DirectoryOrCreate + {{- end }} + {{- else }} + - name: spire-ha-broker-socket-dir-upstream-a + hostPath: + path: {{ if .Values.singleSocket }}{{ .Values.sockets.single.broker.hostPath }}{{ else }}{{ .Values.sockets.a.broker.hostPath }}{{ end }} + type: DirectoryOrCreate + - name: spire-ha-workload-socket-dir-upstream-a + hostPath: + path: {{ if .Values.singleSocket }}{{ .Values.sockets.single.workload.hostPath }}{{ else }}{{ .Values.sockets.a.workload.hostPath }}{{ end }} + type: DirectoryOrCreate + {{- if not .Values.singleSocket }} + - name: spire-ha-broker-socket-dir-upstream-b + hostPath: + path: {{ .Values.sockets.b.broker.hostPath }} + type: DirectoryOrCreate + - name: spire-ha-workload-socket-dir-upstream-b + hostPath: + path: {{ .Values.sockets.b.workload.hostPath }} + type: DirectoryOrCreate + {{- end }} {{- end }} - name: dev hostPath: diff --git a/charts/spire-ha-agent/values.yaml b/charts/spire-ha-agent/values.yaml index d6d44b3..31d3837 100644 --- a/charts/spire-ha-agent/values.yaml +++ b/charts/spire-ha-agent/values.yaml @@ -17,22 +17,43 @@ image: pullPolicy: IfNotPresent tag: "" +## @param mode If the spire-ha-agent will run in delegated or broker mode +mode: delegated + ## @param singleSocket If in singleSocket mode, only one driver is used singleSocket: false -## @param sockets.single.admin.hostPath Where the sockets are on disk when in single socket mode -## @param sockets.a.admin.hostPath Where the sockets are on disk -## @param sockets.b.admin.hostPath Where the sockets are on disk +## @param sockets.single.admin.hostPath Where the admin socket is on disk when in single socket mode +## @param sockets.a.admin.hostPath Where the a admin socket is on disk +## @param sockets.b.admin.hostPath Where the b admin sockets is on disk +## @param sockets.single.broker.hostPath Where the broker socket is on disk when in single socket mode +## @param sockets.a.broker.hostPath Where the a broker socket is on disk +## @param sockets.b.broker.hostPath Where the b broker socket is on disk +## @param sockets.single.workload.hostPath Where the broker socket is on disk when in single socket mode +## @param sockets.a.workload.hostPath Where the a workload socket is on disk +## @param sockets.b.workload.hostPath Where the b workload socket is on disk sockets: single: admin: hostPath: /var/run/spire/agent/sockets/main/csi.spiffe.io/admin + broker: + hostPath: /var/run/spire/agent/sockets/main/csi.spiffe.io/broker + workload: + hostPath: /var/run/spire/agent-sockets a: admin: hostPath: /var/run/spire/agent/sockets/a/csi.spiffe.io/admin + broker: + hostPath: /var/run/spire/agent/sockets/a/csi.spiffe.io/broker + workload: + hostPath: /var/run/spire/agent/sockets/a/csi.spiffe.io/public b: admin: hostPath: /var/run/spire/agent/sockets/b/csi.spiffe.io/admin + broker: + hostPath: /var/run/spire/agent/sockets/b/csi.spiffe.io/broker + workload: + hostPath: /var/run/spire/agent/sockets/b/csi.spiffe.io/public ## @param vsock Use a vsockets to expose the service rather then a unix socket vsock: false diff --git a/charts/spire-nested/README.md b/charts/spire-nested/README.md index ccc3053..4318c19 100644 --- a/charts/spire-nested/README.md +++ b/charts/spire-nested/README.md @@ -466,6 +466,9 @@ Now you can interact with the Spire agent socket from your own application. The | `downstream-spire-agent-bottom-turtle-ha-a.sockets.admin.enabled` | Enable admin socket | `true` | | `downstream-spire-agent-bottom-turtle-ha-a.sockets.admin.mountOnHost` | Mount admin socket on host | `true` | | `downstream-spire-agent-bottom-turtle-ha-a.authorizedDelegates` | List of workloads able to use the delegation api | `["/spire-ha-agent"]` | +| `downstream-spire-agent-bottom-turtle-ha-a.brokerAPI.brokers.spire-ha-agent.enabled` | Enable the spire-ha-agent by default | `true` | +| `downstream-spire-agent-bottom-turtle-ha-a.workloadAttestors.k8s.brokerAPI.accessPolicy` | The default accessPolicy | `permissive` | +| `downstream-spire-agent-bottom-turtle-ha-a.workloadAttestors.k8s.brokerAPI.brokers.spire-ha-agent.enabled` | Enable the spire-ha-agent by default | `true` | | `downstream-spire-agent-bottom-turtle-ha-b.nameOverride` | Overrides the name of Spire agent pods | `agent-downstream` | | `downstream-spire-agent-bottom-turtle-ha-b.server.nameOverride` | The name override setting of the internal SPIRE server | `internal-server` | | `downstream-spire-agent-bottom-turtle-ha-b.bundleConfigMap` | The name of the configmap that contains the downstream bundle | `spire-server-b-bundle` | @@ -485,6 +488,9 @@ Now you can interact with the Spire agent socket from your own application. The | `downstream-spire-agent-bottom-turtle-ha-b.sockets.admin.enabled` | Enable admin socket | `true` | | `downstream-spire-agent-bottom-turtle-ha-b.sockets.admin.mountOnHost` | Mount admin socket on host | `true` | | `downstream-spire-agent-bottom-turtle-ha-b.authorizedDelegates` | List of workloads able to use the delegation api | `["/spire-ha-agent"]` | +| `downstream-spire-agent-bottom-turtle-ha-b.brokerAPI.brokers.spire-ha-agent.enabled` | Enable the spire-ha-agent by default | `true` | +| `downstream-spire-agent-bottom-turtle-ha-b.workloadAttestors.k8s.brokerAPI.accessPolicy` | The default accessPolicy | `permissive` | +| `downstream-spire-agent-bottom-turtle-ha-b.workloadAttestors.k8s.brokerAPI.brokers.spire-ha-agent.enabled` | Enable the spire-ha-agent by default | `true` | | `downstream-spiffe-csi-driver-bottom-turtle-ha-a.fullnameOverride` | Fullname override | `spiffe-csi-driver-downstream-a` | | `downstream-spiffe-csi-driver-bottom-turtle-ha-a.agentSocketPath` | path to agent socket | `/var/run/spire/agent/sockets/a/csi.spiffe.io/public/spire-agent.sock` | | `downstream-spiffe-csi-driver-bottom-turtle-ha-a.pluginName` | The name of the plugin instance | `a.csi.spiffe.io` | diff --git a/charts/spire-nested/values.yaml b/charts/spire-nested/values.yaml index 0dc3de6..b551356 100644 --- a/charts/spire-nested/values.yaml +++ b/charts/spire-nested/values.yaml @@ -680,6 +680,20 @@ downstream-spire-agent-bottom-turtle-ha-a: ## @param downstream-spire-agent-bottom-turtle-ha-a.authorizedDelegates List of workloads able to use the delegation api authorizedDelegates: - /spire-ha-agent + brokerAPI: + brokers: + spire-ha-agent: + ## @param downstream-spire-agent-bottom-turtle-ha-a.brokerAPI.brokers.spire-ha-agent.enabled Enable the spire-ha-agent by default + enabled: true + workloadAttestors: + k8s: + brokerAPI: + ## @param downstream-spire-agent-bottom-turtle-ha-a.workloadAttestors.k8s.brokerAPI.accessPolicy The default accessPolicy + accessPolicy: permissive + brokers: + spire-ha-agent: + ## @param downstream-spire-agent-bottom-turtle-ha-a.workloadAttestors.k8s.brokerAPI.brokers.spire-ha-agent.enabled Enable the spire-ha-agent by default + enabled: true # Used with tags [bottomTurtleHAB] downstream-spire-agent-bottom-turtle-ha-b: @@ -737,6 +751,20 @@ downstream-spire-agent-bottom-turtle-ha-b: ## @param downstream-spire-agent-bottom-turtle-ha-b.authorizedDelegates List of workloads able to use the delegation api authorizedDelegates: - /spire-ha-agent + brokerAPI: + brokers: + spire-ha-agent: + ## @param downstream-spire-agent-bottom-turtle-ha-b.brokerAPI.brokers.spire-ha-agent.enabled Enable the spire-ha-agent by default + enabled: true + workloadAttestors: + k8s: + brokerAPI: + ## @param downstream-spire-agent-bottom-turtle-ha-b.workloadAttestors.k8s.brokerAPI.accessPolicy The default accessPolicy + accessPolicy: permissive + brokers: + spire-ha-agent: + ## @param downstream-spire-agent-bottom-turtle-ha-b.workloadAttestors.k8s.brokerAPI.brokers.spire-ha-agent.enabled Enable the spire-ha-agent by default + enabled: true # Used with tags [bottomTurtleHAA] downstream-spiffe-csi-driver-bottom-turtle-ha-a: diff --git a/examples/bottom-turtle-ha/federation-test-job.yaml b/examples/bottom-turtle-ha/federation-test-job.yaml new file mode 100644 index 0000000..26923d3 --- /dev/null +++ b/examples/bottom-turtle-ha/federation-test-job.yaml @@ -0,0 +1,74 @@ +apiVersion: batch/v1 +kind: Job +metadata: + name: federation-test +spec: + backoffLimit: 0 + template: + metadata: + labels: + app: federation-test + spec: + restartPolicy: Never + initContainers: + - name: static-busybox + # Replaced by run-tests.sh with the image from the spiffe-oidc-discovery-provider chart + image: replaced-by-run-tests.sh + command: + - sh + - -c + - | + cp /bin/busybox /data/busybox + chmod +x /data/busybox + volumeMounts: + - name: data-volume + mountPath: /data + containers: + - name: main + # Replaced by run-tests.sh with the image from the spire-agent chart + image: replaced-by-run-tests.sh + command: + - /data/busybox + - sh + - -xc + - | + SOCK=/spire-agent/spire-agent.sock + i=0 + while [ "$i" -lt 60 ]; do + if XOUT=$(/opt/spire/bin/spire-agent api fetch x509 -socketPath "$SOCK" -write /data -timeout 5s 2>&1) && + echo "$XOUT" | /data/busybox grep -q "for trust domain other.org" && + JOUT=$(/opt/spire/bin/spire-agent api fetch jwt -audience test -socketPath "$SOCK" -timeout 5s 2>&1) && + echo "$JOUT" | /data/busybox grep -q "bundle(other.org)"; then + # The other.org bundle was statically set to the same single CA on both sides, + # so every federated bundle delivered must contain exactly one certificate. + for f in /data/federated_bundle.*.pem; do + COUNT=$(/data/busybox grep -c "BEGIN CERTIFICATE" "$f") + if [ "$COUNT" != "1" ]; then + echo "expected 1 CA in $f, got $COUNT" + exit 1 + fi + done + echo "$XOUT" + echo "$JOUT" + echo FEDERATION-OK + exit 0 + fi + i=$((i+1)) + /data/busybox sleep 2 + done + echo "$XOUT" + echo "$JOUT" + exit 1 + volumeMounts: + - name: data-volume + mountPath: /data + - name: spire-api + mountPath: /spire-agent + readOnly: true + volumes: + - name: data-volume + emptyDir: {} + - name: spire-api + csi: + driver: csi.spiffe.io + readOnly: true diff --git a/examples/bottom-turtle-ha/run-tests.sh b/examples/bottom-turtle-ha/run-tests.sh index c993c8e..1b6428f 100755 --- a/examples/bottom-turtle-ha/run-tests.sh +++ b/examples/bottom-turtle-ha/run-tests.sh @@ -14,6 +14,7 @@ source "${SCRIPTPATH}/../../.github/scripts/parse-versions.sh" source "${TESTDIR}/common.sh" CLEANUP=1 +BROKER=0 for i in "$@"; do case $i in @@ -21,9 +22,38 @@ for i in "$@"; do CLEANUP=0 shift # past argument=value ;; + -b) + BROKER=1 + shift # past argument=value + ;; esac done +# With -b, test the spire-ha-agent broker api instead of the delegated api. +# Broker mode also supports federated trust bundles, so federate the ha-agent's own entry and a +# dedicated federation-test workload entry with the other.org trust domain on both sides. Delegated +# mode only tolerates the local and spire-ha bundles, so none of this may apply without -b. +BROKER_MODE_ARGS=() +BROKER_SOCKET_ARGS_A=() +BROKER_SOCKET_ARGS_B=() +if [ "${BROKER}" -eq 1 ]; then + BROKER_MODE_ARGS=(--set "spire-ha-agent.mode=broker") + BROKER_SOCKET_ARGS_A=( + --set downstream-spire-agent-bottom-turtle-ha-a.sockets.broker.enabled=true + --set downstream-spire-agent-bottom-turtle-ha-a.sockets.broker.mountOnHost=true + --set 'internal-spire-server-bottom-turtle-ha-a.controllerManager.identities.clusterSPIFFEIDs.spire-ha-agent.federatesWith={spire-ha,other.org}' + --set 'internal-spire-server-bottom-turtle-ha-a.controllerManager.identities.clusterSPIFFEIDs.federation-test.federatesWith={other.org}' + --set 'internal-spire-server-bottom-turtle-ha-a.controllerManager.identities.clusterSPIFFEIDs.federation-test.podSelector.matchLabels.app=federation-test' + ) + BROKER_SOCKET_ARGS_B=( + --set downstream-spire-agent-bottom-turtle-ha-b.sockets.broker.enabled=true + --set downstream-spire-agent-bottom-turtle-ha-b.sockets.broker.mountOnHost=true + --set 'internal-spire-server-bottom-turtle-ha-b.controllerManager.identities.clusterSPIFFEIDs.spire-ha-agent.federatesWith={spire-ha,other.org}' + --set 'internal-spire-server-bottom-turtle-ha-b.controllerManager.identities.clusterSPIFFEIDs.federation-test.federatesWith={other.org}' + --set 'internal-spire-server-bottom-turtle-ha-b.controllerManager.identities.clusterSPIFFEIDs.federation-test.podSelector.matchLabels.app=federation-test' + ) +fi + if [ "x${GITHUB_JOB}" != "x" ]; then echo "Running in GitHub" else @@ -36,6 +66,9 @@ teardown() { docker exec -i chart-testing-worker /bin/bash -c "more /var/lib/kubelet/pods/*/volumes/kubernetes.io~empty-dir/disk-keymanager/keys.json /var/lib/kubelet/pods/*/volumes/kubernetes.io~empty-dir/spire-agent-persistence/agent-data.json | cat" sudo systemctl status spire-server@a || true sudo systemctl status spire-server@b || true + sudo systemctl status spire-server@other || true + kubectl describe job federation-test || true + kubectl logs job/federation-test || true sudo spire-server entry show -instance a || true sudo spire-server entry show -instance b || true sudo systemctl status spire-controller-manager@a || true @@ -59,6 +92,9 @@ teardown() { kubectl exec -i -n spire-server spire-a-internal-server-0 -- spire-server agent list -output json | yq e . - -P || true kubectl exec -i -n spire-server spire-b-internal-server-0 -- spire-server agent list -output json | yq e . - -P || true kubectl get pods -A -o wide || true + kubectl describe daemonset pods -n spire-system || true + kubectl get configmap -n spire-system || true + kubectl get configmap -n spire-system spire-a-agent-downstream -o yaml || true print_helm_releases @@ -68,6 +104,7 @@ teardown() { fi if [ "${CLEANUP}" -eq 1 ]; then + kubectl delete job federation-test 2>/dev/null || true helm uninstall --namespace spire-mgmt spire-b 2>/dev/null || true helm uninstall --namespace spire-mgmt spire-a 2>/dev/null || true helm uninstall --namespace spire-mgmt spire 2>/dev/null || true @@ -127,6 +164,30 @@ wait_for_jwt() { return 1 } +wait_for_entry_federation() { + local pod="$1" + local trustdomain="$2" + local timeout=60 + local count=0 + while [ "$count" -lt "$timeout" ]; do + if kubectl exec -i -n spire-server "$pod" -- spire-server entry show -spiffeID spiffe://production.other/spire-ha-agent | grep -q "$trustdomain"; then + return 0 + fi + sleep 2 + ((count++)) || true + done + return 1 +} + +run_federation_test_job() { + kubectl delete job federation-test 2>/dev/null || true + # Inject the images from the charts into the job so they always sync up + yq e "(.spec.template.spec.initContainers[] | select(.name == \"static-busybox\") | .image) = \"${BUSYBOX_IMAGE}\" | (.spec.template.spec.containers[] | select(.name == \"main\") | .image) = \"${AGENT_IMAGE}\"" \ + "${SCRIPTPATH}/federation-test-job.yaml" | kubectl apply -f - + kubectl wait --for=condition=complete --timeout=240s job/federation-test + kubectl logs job/federation-test | grep FEDERATION-OK +} + "${SCRIPTPATH}/../../.github/scripts/prepare-local-chart-deps.sh" # Get the package repo and install the packages @@ -137,6 +198,25 @@ sudo apt-get install -y spire-common spire-agent spire-server spire-controller-m # Set our testing trust domain sudo sed -i 's/example.org/production.other/' /etc/spiffe/default-trust-domain.env +if [ "${BROKER}" -eq 1 ]; then + # Pull the federation test job images out of the charts so they always sync up. + AGENT_IMAGE=$(helm template t charts/spire -s charts/spire-agent/templates/daemonset.yaml --values "${COMMON_TEST_YOUR_VALUES}" --set spire-agent.enabled=true | yq e 'select(.kind=="DaemonSet") | .spec.template.spec.containers[] | select(.name=="spire-agent") | .image' -) + BUSYBOX_IMAGE=$(helm template t charts/spire -s charts/spiffe-oidc-discovery-provider/templates/tests/test-keys.yaml --values "${COMMON_TEST_YOUR_VALUES}" --set spiffe-oidc-discovery-provider.enabled=true | yq e 'select(.kind=="Pod") | .spec.initContainers[] | select(.name=="static-busybox") | .image' -) + echo "federation test job images: ${AGENT_IMAGE} ${BUSYBOX_IMAGE}" + + # Mint a trust bundle for a foreign trust domain (other.org) to test federated trust bundle + # support. A throwaway third spire-server instance produces a genuine spiffe format bundle + # carrying both x509 and jwt authorities. The instance env file overrides the global trust + # domain since systemd applies later EnvironmentFiles last. + sudo /bin/bash -c '(echo SPIFFE_TRUST_DOMAIN=other.org; echo SPIRE_BIND_PORT=8083) > /etc/spire/server/other.env' + sudo systemctl start spire-server@other + wait_for_healthcheck spire-server /run/spire/server/sockets/other/private/api.sock + sudo spire-server bundle show -format spiffe -socketPath /run/spire/server/sockets/other/private/api.sock | sudo tee /tmp/other-org-bundle.json > /dev/null + sudo systemctl stop spire-server@other + grep -q '"x509-svid"' /tmp/other-org-bundle.json + grep -q '"jwt-svid"' /tmp/other-org-bundle.json +fi + # register some workloads with the spire server using manifests sudo mkdir -p /etc/spire/server/a/manifests/ /etc/spire/server/b/manifests/ sudo cp "${SCRIPTPATH}/example-manifests"/* /etc/spire/server/a/manifests/ @@ -235,7 +315,8 @@ helm upgrade --install --create-namespace --namespace spire-mgmt --values "${COM --set tags.haAgentCommon=true \ --set "global.spire.namespaces.create=true" \ --set "global.spire.ingressControllerType=ingress-nginx" \ - --set "spiffe-oidc-discovery-provider.ingress.enabled=true" + --set "spiffe-oidc-discovery-provider.ingress.enabled=true" \ + "${BROKER_MODE_ARGS[@]}" # Create spire-identity-exchange cert for testing. mkdir -p certs @@ -253,14 +334,21 @@ helm upgrade --install --namespace spire-mgmt --values "${COMMON_TEST_YOUR_VALUE --set tags.bottomTurtleHAA=true \ --values "${SCRIPTPATH}/spire-identity-exchange-values.yaml" \ --set "spire-identity-exchange-bottom-turtle-ha-a.enabled=true" \ - --set "global.spire.ingressControllerType=ingress-nginx" + --set "global.spire.ingressControllerType=ingress-nginx" \ + "${BROKER_SOCKET_ARGS_A[@]}" + +if [ "${BROKER}" -eq 1 ]; then + # Install the other.org bundle so the controller manager can create the entries that federate + # with it. It retries any entries that failed with "unable to find federated bundle". + kubectl exec -i -n spire-server spire-a-internal-server-0 -- spire-server bundle set -format spiffe -id spiffe://other.org < /tmp/other-org-bundle.json +fi docker exec -i chart-testing-worker /bin/bash -c "more /var/lib/kubelet/pods/*/volumes/kubernetes.io~empty-dir/disk-keymanager/keys.json /var/lib/kubelet/pods/*/volumes/kubernetes.io~empty-dir/spire-agent-persistence/agent-data.json | cat" # Rollout just to sped up the tests kubectl patch deployment spiffe-oidc-discovery-provider -n spire-server --type='strategic' -p '{"spec": {"strategy": {"type": "Recreate", "rollingUpdate": null}}}' kubectl rollout restart daemonset -n spire-system spire-ha-agent -kubectl rollout status daemonset -n spire-system spire-ha-agent +kubectl rollout status daemonset -n spire-system spire-ha-agent --timeout=1m kubectl rollout restart deployment -n spire-server spiffe-oidc-discovery-provider kubectl rollout status deployment -n spire-server spiffe-oidc-discovery-provider --timeout=1m kubectl wait -n spire-server --for=condition=ready pod -l "app.kubernetes.io/name=spiffe-oidc-discovery-provider" --field-selector=status.phase=Running --timeout=90s @@ -273,7 +361,15 @@ helm upgrade --install --namespace spire-mgmt --values "${COMMON_TEST_YOUR_VALUE --set internal-spire-server-bottom-turtle-ha-b.upstreamAuthority.spire.server.port=8082 \ --values "${SCRIPTPATH}/spire-identity-exchange-values.yaml" \ --set "spire-identity-exchange-bottom-turtle-ha-b.enabled=true" \ - --set "global.spire.ingressControllerType=ingress-nginx" + --set "global.spire.ingressControllerType=ingress-nginx" \ + "${BROKER_SOCKET_ARGS_B[@]}" + +if [ "${BROKER}" -eq 1 ]; then + kubectl exec -i -n spire-server spire-b-internal-server-0 -- spire-server bundle set -format spiffe -id spiffe://other.org < /tmp/other-org-bundle.json + # Both sides' spire-ha-agent entries must federate with other.org before the workload test. + wait_for_entry_federation spire-a-internal-server-0 other.org + wait_for_entry_federation spire-b-internal-server-0 other.org +fi docker ps docker exec -i chart-testing-worker /bin/bash -c "more /var/lib/kubelet/pods/*/volumes/kubernetes.io~empty-dir/disk-keymanager/keys.json /var/lib/kubelet/pods/*/volumes/kubernetes.io~empty-dir/spire-agent-persistence/agent-data.json | cat" @@ -305,11 +401,22 @@ TOKEN=$(kubectl logs job/test) curl -f -H "Authorization: Bearer ${TOKEN}" -X POST --resolve "spire-identity-exchange-a-rest.production.other:443:$IP" "https://spire-identity-exchange-a-rest.production.other/api/v1/svid/k8s_psat/x509" -k -sS -q curl -f -H "Authorization: Bearer ${TOKEN}" -X POST --resolve "spire-identity-exchange-b-rest.production.other:443:$IP" "https://spire-identity-exchange-b-rest.production.other/api/v1/svid/k8s_psat/x509" -k -sS -q +if [ "${BROKER}" -eq 1 ]; then + # Verify a workload on the ha-agent socket receives the other.org federated trust bundles, + # x509 and jwt, merged from both sides. + run_federation_test_job +fi + #Test out running only on side b since we know already only both servers work together, and that only side a works if we made it this far. helm delete -n spire-mgmt spire-a kubectl rollout restart daemonset -n spire-system spire-ha-agent kubectl rollout status daemonset -n spire-system spire-ha-agent kubectl rollout restart deployment -n spire-server spiffe-oidc-discovery-provider -kubectl rollout status deployment -n spire-server spiffe-oidc-discovery-provider --timeout=1m +kubectl rollout status deployment -n spire-server spiffe-oidc-discovery-provider --timeout=5m curl -k --resolve "oidc-discovery.production.other:443:$IP" "https://oidc-discovery.production.other/.well-known/openid-configuration" -s --fail +if [ "${BROKER}" -eq 1 ]; then + # Verify the other.org federated trust bundles still serve with only side b running. + run_federation_test_job +fi + diff --git a/tests/integration/bottom-turtle-ha-broker/run-tests.sh b/tests/integration/bottom-turtle-ha-broker/run-tests.sh new file mode 100755 index 0000000..a180a84 --- /dev/null +++ b/tests/integration/bottom-turtle-ha-broker/run-tests.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash + +set -xe + +SCRIPT="$(readlink -f "$0")" +SCRIPTPATH="$(dirname "${SCRIPT}")" + +# Run the bottom-turtle-ha example test with the spire-ha-agent broker api enabled. +exec "${SCRIPTPATH}/../../../examples/bottom-turtle-ha/run-tests.sh" -b "$@"