Bottom turtle ha registry tests (#929)

* Bottom turtle ha registry tests

Signed-off-by: Kevin Fox <[email protected]>

* Fix test because of spire-ha-agent

Signed-off-by: Kevin Fox <[email protected]>

* Fix config options

Signed-off-by: Kevin Fox <[email protected]>

* Fix yq issue and log better

Signed-off-by: Kevin Fox <[email protected]>

* Fix some things

Signed-off-by: Kevin Fox <[email protected]>

* Change timeout and fix image

Signed-off-by: Kevin Fox <[email protected]>

* Bump up zot version.

Signed-off-by: Kevin Fox <[email protected]>

* More debugging stuff

Signed-off-by: Kevin Fox <[email protected]>

* More debugging stuff

Signed-off-by: Kevin Fox <[email protected]>

* More fixes

Signed-off-by: Kevin Fox <[email protected]>

* More logging

Signed-off-by: Kevin Fox <[email protected]>

* More logging

Signed-off-by: Kevin Fox <[email protected]>

* More logging

Signed-off-by: Kevin Fox <[email protected]>

* More logging

Signed-off-by: Kevin Fox <[email protected]>

* More logging

Signed-off-by: Kevin Fox <[email protected]>

* Fix test

Signed-off-by: Kevin Fox <[email protected]>

* Log less

Signed-off-by: Kevin Fox <[email protected]>

* Cleanup

Signed-off-by: Kevin Fox <[email protected]>

* Make test less different then real deployment

Signed-off-by: Kevin Fox <[email protected]>

---------

Signed-off-by: Kevin Fox <[email protected]>
Signed-off-by: kfox1111 <[email protected]>

.github/kind/conf/credential-provider-config.yaml

@@ -0,0 +1,25 @@
+apiVersion: kubelet.config.k8s.io/v1
+kind: CredentialProviderConfig
+providers:
+  - name: k8s-image-cred-spire-identity-exchange
+    matchImages:
+      - "zot.production.other"
+    defaultCacheDuration: "0s"
+    apiVersion: credentialprovider.kubelet.k8s.io/v1
+    args:
+      - "--username=zot"
+      - "--mode=spire-identity-exchange"
+      - "--url=https://spire-identity-exchange-rest-spiffe.production.other"
+      - "--stack=image_pull"
+      - "--registry-audience=zot"
+      - "--spiffe-audience=spire-identity-exchange"
+      - "--spiffe-hint=image-pull"
+      - "--spiffe-id=spiffe://production.other/service/spire-identity-exchange"
+      - "--timeout=10s"
+    env:
+      - name: SPIFFE_ENDPOINT_SOCKET
+        value: unix:///var/run/spire/agent/sockets/main/public/api.sock
+    tokenAttributes:
+      serviceAccountTokenAudience: "spire-identity-exchange"
+      cacheType: "Token"
+      requireServiceAccount: true

.github/kind/conf/kind-config.yaml

@@ -30,6 +30,39 @@ nodes:
         containerPath: /var/run/spiffe/socat/unix/k8s-spire-agent-a/public
       - hostPath: /var/run/spiffe/socat/unix/k8s-spire-agent-2-b/public
         containerPath: /var/run/spiffe/socat/unix/k8s-spire-agent-b/public
+      # One spire-ha-agent on the host is shared by all three virtual nodes, with a
+      # spiffe-socat-unix bridge per node in front of it. Mount that bridge where a
+      # package installed spire-ha-agent@main listens, so kubelet's configuration in
+      # the node is the same one a real bare metal node would use.
+      - hostPath: /var/run/spiffe/socat/unix/k8s-kubelet-2/public
+        containerPath: /var/run/spire/agent/sockets/main/public
+      - hostPath: ./.github/kind/conf/credential-providers
+        containerPath: /credential-plugins
+      - hostPath: ./.github/kind/conf/credential-provider-config.yaml
+        containerPath: /etc/kubernetes/credential-provider-config.yaml
+    kubeadmConfigPatches:
+      - |
+        apiVersion: kubeadm.k8s.io/v1beta3
+        kind: JoinConfiguration
+        nodeRegistration:
+          kubeletExtraArgs:
+            image-credential-provider-config: /etc/kubernetes/credential-provider-config.yaml
+            image-credential-provider-bin-dir: /credential-plugins
+    kubeadmConfigPatchesJSON6902:
+      - group: kubeadm.k8s.io
+        version: v1beta4
+        kind: JoinConfiguration
+        patch: |
+          - op: add
+            path: /nodeRegistration/kubeletExtraArgs/-
+            value:
+              name: image-credential-provider-config
+              value: /etc/kubernetes/credential-provider-config.yaml
+          - op: add
+            path: /nodeRegistration/kubeletExtraArgs/-
+            value:
+              name: image-credential-provider-bin-dir
+              value: /credential-plugins
   - role: worker
     extraMounts:
       - hostPath: /var/run/spiffe/socat/unix/k8s-spire-server-a/public
@@ -40,6 +73,39 @@ nodes:
         containerPath: /var/run/spiffe/socat/unix/k8s-spire-agent-a/public
       - hostPath: /var/run/spiffe/socat/unix/k8s-spire-agent-3-b/public
         containerPath: /var/run/spiffe/socat/unix/k8s-spire-agent-b/public
+      # One spire-ha-agent on the host is shared by all three virtual nodes, with a
+      # spiffe-socat-unix bridge per node in front of it. Mount that bridge where a
+      # package installed spire-ha-agent@main listens, so kubelet's configuration in
+      # the node is the same one a real bare metal node would use.
+      - hostPath: /var/run/spiffe/socat/unix/k8s-kubelet-3/public
+        containerPath: /var/run/spire/agent/sockets/main/public
+      - hostPath: ./.github/kind/conf/credential-providers
+        containerPath: /credential-plugins
+      - hostPath: ./.github/kind/conf/credential-provider-config.yaml
+        containerPath: /etc/kubernetes/credential-provider-config.yaml
+    kubeadmConfigPatches:
+      - |
+        apiVersion: kubeadm.k8s.io/v1beta3
+        kind: JoinConfiguration
+        nodeRegistration:
+          kubeletExtraArgs:
+            image-credential-provider-config: /etc/kubernetes/credential-provider-config.yaml
+            image-credential-provider-bin-dir: /credential-plugins
+    kubeadmConfigPatchesJSON6902:
+      - group: kubeadm.k8s.io
+        version: v1beta4
+        kind: JoinConfiguration
+        patch: |
+          - op: add
+            path: /nodeRegistration/kubeletExtraArgs/-
+            value:
+              name: image-credential-provider-config
+              value: /etc/kubernetes/credential-provider-config.yaml
+          - op: add
+            path: /nodeRegistration/kubeletExtraArgs/-
+            value:
+              name: image-credential-provider-bin-dir
+              value: /credential-plugins
   - role: worker
     extraMounts:
       - hostPath: /var/run/spiffe/socat/unix/k8s-spire-server-a/public
@@ -50,3 +116,36 @@ nodes:
         containerPath: /var/run/spiffe/socat/unix/k8s-spire-agent-a/public
       - hostPath: /var/run/spiffe/socat/unix/k8s-spire-agent-4-b/public
         containerPath: /var/run/spiffe/socat/unix/k8s-spire-agent-b/public
+      # One spire-ha-agent on the host is shared by all three virtual nodes, with a
+      # spiffe-socat-unix bridge per node in front of it. Mount that bridge where a
+      # package installed spire-ha-agent@main listens, so kubelet's configuration in
+      # the node is the same one a real bare metal node would use.
+      - hostPath: /var/run/spiffe/socat/unix/k8s-kubelet-4/public
+        containerPath: /var/run/spire/agent/sockets/main/public
+      - hostPath: ./.github/kind/conf/credential-providers
+        containerPath: /credential-plugins
+      - hostPath: ./.github/kind/conf/credential-provider-config.yaml
+        containerPath: /etc/kubernetes/credential-provider-config.yaml
+    kubeadmConfigPatches:
+      - |
+        apiVersion: kubeadm.k8s.io/v1beta3
+        kind: JoinConfiguration
+        nodeRegistration:
+          kubeletExtraArgs:
+            image-credential-provider-config: /etc/kubernetes/credential-provider-config.yaml
+            image-credential-provider-bin-dir: /credential-plugins
+    kubeadmConfigPatchesJSON6902:
+      - group: kubeadm.k8s.io
+        version: v1beta4
+        kind: JoinConfiguration
+        patch: |
+          - op: add
+            path: /nodeRegistration/kubeletExtraArgs/-
+            value:
+              name: image-credential-provider-config
+              value: /etc/kubernetes/credential-provider-config.yaml
+          - op: add
+            path: /nodeRegistration/kubeletExtraArgs/-
+            value:
+              name: image-credential-provider-bin-dir
+              value: /credential-plugins

.github/scripts/install-image-cred-provider.sh

@@ -0,0 +1,37 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+VERSION="${IMAGE_CRED_PROVIDER_VERSION:-v0.5.0}"
+BIN_NAME="k8s-image-cred-spire-identity-exchange"
+
+SCRIPTPATH="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+BIN_DIR="${SCRIPTPATH}/../kind/conf/credential-providers"
+
+case "$(uname -m)" in
+  x86_64|amd64) ARCH="x86_64" ;;
+  aarch64|arm64) ARCH="arm64" ;;
+  *) echo "unsupported architecture: $(uname -m)" >&2; exit 1 ;;
+esac
+
+if [ -x "${BIN_DIR}/${BIN_NAME}" ]; then
+  echo "${BIN_NAME} already staged in ${BIN_DIR}"
+  exit 0
+fi
+
+BASE_URL="https://github.com/spiffe/spire-identity-exchange/releases/download/${VERSION}"
+ARCHIVE="${BIN_NAME}_Linux_${ARCH}.tar.gz"
+CHECKSUMS="spire-identity-exchange_${VERSION#v}_checksums.txt"
+
+WORKDIR="$(mktemp -d)"
+trap 'rm -rf "${WORKDIR}"' EXIT
+
+curl -fsSL --retry 5 --retry-all-errors -o "${WORKDIR}/${ARCHIVE}" "${BASE_URL}/${ARCHIVE}"
+curl -fsSL --retry 5 --retry-all-errors -o "${WORKDIR}/${CHECKSUMS}" "${BASE_URL}/${CHECKSUMS}"
+(cd "${WORKDIR}" && grep " ${ARCHIVE}\$" "${CHECKSUMS}" | sha256sum -c -)
+
+tar -xzf "${WORKDIR}/${ARCHIVE}" -C "${WORKDIR}" "${BIN_NAME}"
+mkdir -p "${BIN_DIR}"
+install -m 0755 "${WORKDIR}/${BIN_NAME}" "${BIN_DIR}/${BIN_NAME}"
+
+echo "Staged ${BIN_NAME} ${VERSION} in ${BIN_DIR}"

.github/tests/charts.json

@@ -13,5 +13,10 @@
     "name": "ingress-nginx",
     "repo": "https://kubernetes.github.io/ingress-nginx",
     "version": "4.15.1"
+  },
+  {
+    "name": "zot",
+    "repo": "https://zotregistry.dev/helm-charts",
+    "version": "0.1.122"
   }
 ]

.github/workflows/helm-chart-ci-ignore.yaml

@@ -31,9 +31,9 @@ jobs:
     strategy:
       matrix:
         k8s:
-          - v1.33.7
+          - v1.34.8
-          - v1.34.3
+          - v1.35.5
-          - v1.35.1
+          - v1.36.1
 
     steps:
       - run: 'echo "Skipping tests"'
@@ -75,9 +75,9 @@ jobs:
     strategy:
       matrix:
         k8s:
-          - v1.33.7
+          - v1.34.8
-          - v1.34.3
+          - v1.35.5
-          - v1.35.1
+          - v1.36.1
         example:
           - ${{ fromJson(needs.build-matrix.outputs.examples) }}
 
@@ -93,9 +93,9 @@ jobs:
     strategy:
       matrix:
         k8s:
-          - v1.33.7
+          - v1.34.8
-          - v1.34.3
+          - v1.35.5
-          - v1.35.1
+          - v1.36.1
         example:
           - ${{ fromJson(needs.build-matrix.outputs.integrationtests) }}
 
@@ -111,9 +111,9 @@ jobs:
     strategy:
       matrix:
         k8s:
-          - v1.33.7
+          - v1.34.8
-          - v1.34.3
+          - v1.35.5
-          - v1.35.1
+          - v1.36.1
 
     steps:
       - run: 'echo "Skipping upgrade-test"'

.github/workflows/helm-chart-ci.yaml

@@ -17,6 +17,7 @@ on:
       - '.github/tests/**/*.sh'
       - '.github/tests/**/*.json'
       - '.github/scripts/check-readme-versions.sh'
+      - '.github/scripts/install-image-cred-provider.sh'
       - 'examples/**/*.yaml'
       - 'examples/**/*.sh'
       - 'tests/**/*'
@@ -31,6 +32,7 @@ env:
   PYTHON_VERSION: 3.11.3
   KIND_VERSION: v0.32.0
   CHART_TESTING_VERSION: v3.8.0
+  IMAGE_CRED_PROVIDER_VERSION: v0.5.0
 
 jobs:
   checks:
@@ -160,9 +162,9 @@ jobs:
         # Kubernetes, but can go back farther as long as we don't need heroics
         # to pull it off (i.e. kubectl version juggling).
         k8s:
-          - v1.33.7
+          - v1.34.8
-          - v1.34.3
+          - v1.35.5
-          - v1.35.1
+          - v1.36.1
 
     steps:
       - name: Checkout
@@ -185,6 +187,9 @@ jobs:
         with:
           version: ${{ env.CHART_TESTING_VERSION }}
 
+      - name: Install image credential provider
+        run: ./.github/scripts/install-image-cred-provider.sh
+
       - name: Create kind ${{ matrix.k8s }} cluster
         uses: helm/[email protected]
         # Only build a kind cluster if there are chart changes to test.
@@ -251,9 +256,9 @@ jobs:
       fail-fast: false
       matrix:
         k8s:
-          - v1.33.7
+          - v1.34.8
-          - v1.34.3
+          - v1.35.5
-          - v1.35.1
+          - v1.36.1
         example:
           - ${{ fromJson(needs.build-matrix.outputs.examples) }}
 
@@ -271,6 +276,9 @@ jobs:
         with:
           python-version: ${{ env.PYTHON_VERSION }}
 
+      - name: Install image credential provider
+        run: ./.github/scripts/install-image-cred-provider.sh
+
       - name: Create kind cluster
         uses: helm/[email protected]
         # Only build a kind cluster if there are chart changes to test.
@@ -306,9 +314,9 @@ jobs:
       fail-fast: false
       matrix:
         k8s:
-          - v1.33.7
+          - v1.34.8
-          - v1.34.3
+          - v1.35.5
-          - v1.35.1
+          - v1.36.1
         integrationtest:
           - ${{ fromJson(needs.build-matrix.outputs.integrationtests) }}
 
@@ -326,6 +334,9 @@ jobs:
         with:
           python-version: ${{ env.PYTHON_VERSION }}
 
+      - name: Install image credential provider
+        run: ./.github/scripts/install-image-cred-provider.sh
+
       - name: Create kind cluster
         uses: helm/[email protected]
         # Only build a kind cluster if there are chart changes to test.
@@ -354,9 +365,9 @@ jobs:
       fail-fast: false
       matrix:
         k8s:
-          - v1.33.7
+          - v1.34.8
-          - v1.34.3
+          - v1.35.5
-          - v1.35.1
+          - v1.36.1
 
     steps:
       - name: Checkout
@@ -372,6 +383,9 @@ jobs:
         with:
           python-version: ${{ env.PYTHON_VERSION }}
 
+      - name: Install image credential provider
+        run: ./.github/scripts/install-image-cred-provider.sh
+
       - name: Create kind cluster
         uses: helm/[email protected]
         # Only build a kind cluster if there are chart changes to test.

.gitignore

@@ -4,3 +4,4 @@ bin/
 *.swp
 charts/**/*.tgz
 .DS_Store
+.github/kind/conf/credential-providers/

charts/spire-ha-agent/Chart.yaml

@@ -3,7 +3,7 @@ name: spire-ha-agent
 description: A Helm chart to install the SPIRE HA agent.
 type: application
 version: 0.3.1
-appVersion: "0.3.0"
+appVersion: "0.4.0"
 keywords: ["spiffe", "spire-ha-agent"]
 home: https://github.com/spiffe/helm-charts-hardened/tree/main/charts/spire-ha-agent
 sources:

charts/spire-ha-agent/README.md

@@ -1,6 +1,6 @@
 # spire-ha-agent
 
-![Version: 0.3.1](https://img.shields.io/badge/Version-0.3.1-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 0.3.0](https://img.shields.io/badge/AppVersion-0.3.0-informational?style=flat-square)
+![Version: 0.3.1](https://img.shields.io/badge/Version-0.3.1-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 0.4.0](https://img.shields.io/badge/AppVersion-0.4.0-informational?style=flat-square)
 
 A Helm chart to install the SPIRE HA agent.
 

examples/bottom-turtle-ha/README.md

@@ -136,3 +136,78 @@ helm upgrade --install --namespace spire-mgmt --values "spire-values.yaml" \
   --set tags.bottomTurtleHAB=true \
   --set "global.spire.ingressControllerType=ingress-nginx"
 ```
+
+## Host services on the bottom turtle
+
+The diagrams above show a `spire-ha-agent` on each host, fed by `spire-agent@a` and
+`spire-agent@b`, serving host services such as sshd and kubelet. That is what makes a host
+service's identity survive one root server going away: the ha-agent merges both sides into
+a single Workload API and answers from whichever side is up.
+
+It attests every caller by pid, so one ha-agent can serve many callers with different
+identities. Register the caller against the root servers and it gets its own SVID:
+
+```
+apiVersion: spire.spiffe.io/v1alpha1
+kind: ClusterStaticEntry
+metadata:
+  name: node1-spire-ha-agent
+spec:
+  parentID: spiffe://${SPIFFE_TRUST_DOMAIN}/agent/node1
+  spiffeID: spiffe://${SPIFFE_TRUST_DOMAIN}/spire-ha-agent
+  selectors:
+  - systemd:id:[email protected]
+  federatesWith:
+  - spire-ha
+```
+
+The packaged `spire-agent` config already names `spiffe://${SPIFFE_TRUST_DOMAIN}/spire-ha-agent`
+in its `authorized_delegates`, so no agent configuration is needed, only the entry.
+
+## Registry image pull
+
+Kubelet can use that host identity to pull images, without any pull secret. On seeing an
+image from the registry, kubelet runs an image credential provider on the node, which
+presents two credentials to the spire-identity-exchange: the pod's projected service
+account token and the node's own JWT-SVID from the ha-agent. The exchange mints a registry
+token, and the registry authorizes by SPIFFE ID.
+
+The registry in this example is zot, deployed with the upstream chart. Its serving
+certificate is a SPIRE SVID delivered by `spiffe-helper` as an init container plus a
+sidecar, so nothing carries a long lived key. The identity needs an explicit DNS name,
+because an X509-SVID has only a URI SAN by default and containerd validates the registry
+by hostname:
+
+```
+zot:
+  spiffeIDTemplate: spiffe://{{ .TrustDomain }}/zot
+  podSelector:
+    matchLabels:
+      app.kubernetes.io/name: zot
+  dnsNameTemplates:
+    - zot.{{ .TrustDomain }}
+```
+
+Push and pull share one exchange stack. They are kept apart by their registration entries,
+whose selectors are disjoint, and by the registry's own access control, which grants the
+push identity write and the pull identity read only.
+
+### How the test deviates from the diagrams
+
+The test runs a single VM behind several virtual Kubernetes nodes, so a few things differ
+from what you would deploy. Worth knowing if you are using this as a reference:
+
+* One `spire-ha-agent` is shared by every virtual node, with a `spiffe-socat-unix` bridge
+  per node in front of it. On a real host kubelet talks to its local ha-agent directly.
+  Each bridge is mounted into its node at `/var/run/spire/agent/sockets/main/public`, which
+  is where a package installed ha-agent listens, so kubelet's own configuration is not a
+  deviation: what you see here is what you would deploy.
+* The registration entries do deviate. Because the caller the ha-agent attests by pid is the
+  bridge, they select on the socat unit rather than on kubelet's own unit. On a real host
+  that selector is the only line that changes.
+* One ha-agent behind every node means the test covers a root server failing, which is the
+  part that matters here, but not a single node's ha-agent failing.
+* The credential provider binary and its configuration are staged into every kind cluster
+  by `.github/scripts/install-image-cred-provider.sh` before the cluster is created.
+  Kubelet refuses to start when a provider named in its configuration is missing, so this
+  cannot be deferred to the test itself.

examples/bottom-turtle-ha/example-manifests/node1-spire-ha-agent.yaml

@@ -0,0 +1,11 @@
+apiVersion: spire.spiffe.io/v1alpha1
+kind: ClusterStaticEntry
+metadata:
+  name: node1-spire-ha-agent
+spec:
+  parentID: spiffe://${SPIFFE_TRUST_DOMAIN}/agent/node1
+  spiffeID: spiffe://${SPIFFE_TRUST_DOMAIN}/spire-ha-agent
+  selectors:
+  - systemd:id:[email protected]
+  federatesWith:
+  - spire-ha

examples/bottom-turtle-ha/example-manifests/node2-image-pull.yaml

@@ -0,0 +1,10 @@
+apiVersion: spire.spiffe.io/v1alpha1
+kind: ClusterStaticEntry
+metadata:
+  name: node2-image-pull
+spec:
+  parentID: spiffe://${SPIFFE_TRUST_DOMAIN}/agent/node1
+  spiffeID: spiffe://${SPIFFE_TRUST_DOMAIN}/kubelet
+  hint: image-pull
+  selectors:
+  - systemd:id:[email protected]

examples/bottom-turtle-ha/example-manifests/node2-kubelet.yaml

@@ -0,0 +1,10 @@
+apiVersion: spire.spiffe.io/v1alpha1
+kind: ClusterStaticEntry
+metadata:
+  name: node2-kubelet
+spec:
+  parentID: spiffe://${SPIFFE_TRUST_DOMAIN}/agent/node1
+  spiffeID: spiffe://${SPIFFE_TRUST_DOMAIN}/kubelet/node2.${SPIFFE_TRUST_DOMAIN}
+  hint: kubelet
+  selectors:
+  - systemd:id:[email protected]

examples/bottom-turtle-ha/example-manifests/node3-image-pull.yaml

@@ -0,0 +1,10 @@
+apiVersion: spire.spiffe.io/v1alpha1
+kind: ClusterStaticEntry
+metadata:
+  name: node3-image-pull
+spec:
+  parentID: spiffe://${SPIFFE_TRUST_DOMAIN}/agent/node1
+  spiffeID: spiffe://${SPIFFE_TRUST_DOMAIN}/kubelet
+  hint: image-pull
+  selectors:
+  - systemd:id:[email protected]

examples/bottom-turtle-ha/example-manifests/node3-kubelet.yaml

@@ -0,0 +1,10 @@
+apiVersion: spire.spiffe.io/v1alpha1
+kind: ClusterStaticEntry
+metadata:
+  name: node3-kubelet
+spec:
+  parentID: spiffe://${SPIFFE_TRUST_DOMAIN}/agent/node1
+  spiffeID: spiffe://${SPIFFE_TRUST_DOMAIN}/kubelet/node3.${SPIFFE_TRUST_DOMAIN}
+  hint: kubelet
+  selectors:
+  - systemd:id:[email protected]

examples/bottom-turtle-ha/example-manifests/node4-image-pull.yaml

@@ -0,0 +1,10 @@
+apiVersion: spire.spiffe.io/v1alpha1
+kind: ClusterStaticEntry
+metadata:
+  name: node4-image-pull
+spec:
+  parentID: spiffe://${SPIFFE_TRUST_DOMAIN}/agent/node1
+  spiffeID: spiffe://${SPIFFE_TRUST_DOMAIN}/kubelet
+  hint: image-pull
+  selectors:
+  - systemd:id:[email protected]

examples/bottom-turtle-ha/example-manifests/node4-kubelet.yaml

@@ -0,0 +1,10 @@
+apiVersion: spire.spiffe.io/v1alpha1
+kind: ClusterStaticEntry
+metadata:
+  name: node4-kubelet
+spec:
+  parentID: spiffe://${SPIFFE_TRUST_DOMAIN}/agent/node1
+  spiffeID: spiffe://${SPIFFE_TRUST_DOMAIN}/kubelet/node4.${SPIFFE_TRUST_DOMAIN}
+  hint: kubelet
+  selectors:
+  - systemd:id:[email protected]

examples/bottom-turtle-ha/image-pull-job.yaml

@@ -0,0 +1,34 @@
+# Pulls the image zot only serves to an exchange-minted identity.
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: zot-pull
+---
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: image-pull
+spec:
+  backoffLimit: 0
+  template:
+    metadata:
+      labels:
+        app: image-pull
+    spec:
+      serviceAccountName: zot-pull
+      restartPolicy: Never
+      containers:
+      - name: main
+        image: zot.production.other/test/busybox:latest
+        imagePullPolicy: Always
+        command: ["sh", "-c", "echo IMAGE-PULL-OK"]
+      # The node authorizer only lets kubelet mint a service account token for an audience
+      # that appears in the pod spec. This must be here to work, even if it looks unused.
+      volumes:
+      - name: spire-token-gate
+        projected:
+          sources:
+          - serviceAccountToken:
+              path: token
+              audience: spire-identity-exchange
+              expirationSeconds: 3600

examples/bottom-turtle-ha/image-push-denied-job.yaml

@@ -0,0 +1,122 @@
+# Negative control: the pull identity must not be able to write. Identical to image-push-job.yaml except the service account
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: image-push-denied
+spec:
+  backoffLimit: 0
+  template:
+    metadata:
+      labels:
+        app: image-push-denied
+    spec:
+      serviceAccountName: zot-pull
+      restartPolicy: Never
+      initContainers:
+      - name: static-busybox
+        # Replaced by run-tests.sh with the image from the spiffe-oidc-discovery-provider chart
+        image: IMAGE_BUSYBOX
+        command: ["sh", "-c", "cp /bin/busybox /data/busybox && chmod +x /data/busybox"]
+        volumeMounts:
+        - name: data-volume
+          mountPath: /data
+      - name: fetch-svid
+        # Replaced by run-tests.sh with the image from the spire-agent chart
+        image: IMAGE_SPIRE_AGENT
+        command:
+        - /data/busybox
+        - sh
+        - -xec
+        - |
+          SOCK=/spire-agent/spire-agent.sock
+          i=0
+          while [ "$i" -lt 30 ]; do
+            if /opt/spire/bin/spire-agent api fetch x509 -socketPath "$SOCK" -write /data -timeout 5s &&
+               /opt/spire/bin/spire-agent api fetch jwt -audience spire-identity-exchange -socketPath "$SOCK" -timeout 5s > /data/jwt.txt; then
+              break
+            fi
+            i=$((i+1))
+            /data/busybox sleep 2
+          done
+          if [ ! -s /data/jwt.txt ]; then
+            echo "no SVID for this pod after ${i} attempts"
+            exit 1
+          fi
+          /data/busybox grep -A1 'token(' /data/jwt.txt | /data/busybox tail -1 | /data/busybox tr -d '[:space:]' > /data/svid.jwt
+          test -s /data/svid.jwt
+        volumeMounts:
+        - name: data-volume
+          mountPath: /data
+        - name: spire-api
+          mountPath: /spire-agent
+          readOnly: true
+      - name: exchange
+        # Replaced by run-tests.sh with the toolkit image from the spiffe-oidc-discovery-provider chart
+        image: IMAGE_TOOLKIT
+        command:
+        - sh
+        - -xec
+        - |
+          cat /etc/ssl/certs/ca-certificates.crt /data/bundle.0.pem > /data/ca-bundle.pem
+
+          PSAT="$(cat /var/run/secrets/tokens/token)"
+          SVID="$(cat /data/svid.jwt)"
+          # This must still succeed. A separate registration entry matches this pod's
+          # credentials and mints the read only identity, so the exchange hands back a
+          # token; it is zot that refuses the write.
+          TOKEN="$(curl -k -sS --fail-with-body --max-time 60 --connect-timeout 10 -X POST \
+            -H "Authorization: Bearer k8s_psat=${PSAT}:spiffe=${SVID}" \
+            -H "Content-Type: application/json" \
+            -d '{"audiences": ["zot"]}' \
+            "https://spire-identity-exchange-rest.production.other/api/v1/svid/image_pull/jwt" \
+            | sed -n 's/.*"token":"\([^"]*\)".*/\1/p')"
+          test -n "${TOKEN}"
+
+          AUTH="$(printf 'zot:%s' "${TOKEN}" | base64 | tr -d '\n')"
+          printf '{"auths":{"zot.production.other":{"auth":"%s"}}}' "${AUTH}" > /docker-config/config.json
+        volumeMounts:
+        - name: data-volume
+          mountPath: /data
+        - name: docker-config
+          mountPath: /docker-config
+        - name: psat
+          mountPath: /var/run/secrets/tokens
+          readOnly: true
+      containers:
+      - name: push
+        image: gcr.io/go-containerregistry/crane:v0.21.9
+        env:
+        - name: DOCKER_CONFIG
+          value: /docker-config
+        - name: SSL_CERT_FILE
+          value: /data/ca-bundle.pem
+        # The crane image is distroless, so borrow the static busybox copied out earlier.
+        command: ["/data/busybox", "sh", "-c"]
+        args:
+        - |
+          if /ko-app/crane copy docker.io/library/busybox:latest zot.production.other/test/denied:latest; then
+            echo "PUSH-SHOULD-HAVE-BEEN-DENIED"
+            exit 1
+          fi
+          echo PUSH-DENIED-OK
+        volumeMounts:
+        - name: data-volume
+          mountPath: /data
+        - name: docker-config
+          mountPath: /docker-config
+      volumes:
+      - name: data-volume
+        emptyDir: {}
+      - name: docker-config
+        emptyDir: {}
+      - name: spire-api
+        csi:
+          driver: csi.spiffe.io
+          readOnly: true
+      - name: psat
+        projected:
+          sources:
+          - serviceAccountToken:
+              path: token
+              audience: spire-identity-exchange
+              expirationSeconds: 3600

examples/bottom-turtle-ha/image-push-job.yaml

@@ -0,0 +1,121 @@
+# Pushes an image into zot using an identity minted by the spire-identity-exchange.
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: zot-push
+---
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: image-push
+spec:
+  backoffLimit: 0
+  template:
+    metadata:
+      labels:
+        app: image-push
+    spec:
+      serviceAccountName: zot-push
+      restartPolicy: Never
+      initContainers:
+      - name: static-busybox
+        # Replaced by run-tests.sh with the image from the spiffe-oidc-discovery-provider chart
+        image: IMAGE_BUSYBOX
+        command: ["sh", "-c", "cp /bin/busybox /data/busybox && chmod +x /data/busybox"]
+        volumeMounts:
+        - name: data-volume
+          mountPath: /data
+      - name: fetch-svid
+        # Replaced by run-tests.sh with the image from the spire-agent chart
+        image: IMAGE_SPIRE_AGENT
+        command:
+        - /data/busybox
+        - sh
+        - -xec
+        - |
+          SOCK=/spire-agent/spire-agent.sock
+          i=0
+          while [ "$i" -lt 30 ]; do
+            if /opt/spire/bin/spire-agent api fetch x509 -socketPath "$SOCK" -write /data -timeout 5s &&
+               /opt/spire/bin/spire-agent api fetch jwt -audience spire-identity-exchange -socketPath "$SOCK" -timeout 5s > /data/jwt.txt; then
+              break
+            fi
+            i=$((i+1))
+            /data/busybox sleep 2
+          done
+          if [ ! -s /data/jwt.txt ]; then
+            echo "no SVID for this pod after ${i} attempts"
+            exit 1
+          fi
+          /data/busybox grep -A1 'token(' /data/jwt.txt | /data/busybox tail -1 | /data/busybox tr -d '[:space:]' > /data/svid.jwt
+          test -s /data/svid.jwt
+        volumeMounts:
+        - name: data-volume
+          mountPath: /data
+        - name: spire-api
+          mountPath: /spire-agent
+          readOnly: true
+      - name: exchange
+        # Replaced by run-tests.sh with the toolkit image from the spiffe-oidc-discovery-provider chart
+        image: IMAGE_TOOLKIT
+        command:
+        - sh
+        - -xec
+        - |
+          # crane talks to both docker.io and zot, so it needs the public roots and the
+          # SPIRE bundle in one file.
+          cat /etc/ssl/certs/ca-certificates.crt /data/bundle.0.pem > /data/ca-bundle.pem
+
+          PSAT="$(cat /var/run/secrets/tokens/token)"
+          SVID="$(cat /data/svid.jwt)"
+          TOKEN="$(curl -k -sS --fail-with-body --max-time 60 --connect-timeout 10 -X POST \
+            -H "Authorization: Bearer k8s_psat=${PSAT}:spiffe=${SVID}" \
+            -H "Content-Type: application/json" \
+            -d '{"audiences": ["zot"]}' \
+            "https://spire-identity-exchange-rest.production.other/api/v1/svid/image_pull/jwt" \
+            | sed -n 's/.*"token":"\([^"]*\)".*/\1/p')"
+          test -n "${TOKEN}"
+
+          AUTH="$(printf 'zot:%s' "${TOKEN}" | base64 | tr -d '\n')"
+          printf '{"auths":{"zot.production.other":{"auth":"%s"}}}' "${AUTH}" > /docker-config/config.json
+        volumeMounts:
+        - name: data-volume
+          mountPath: /data
+        - name: docker-config
+          mountPath: /docker-config
+        - name: psat
+          mountPath: /var/run/secrets/tokens
+          readOnly: true
+      containers:
+      - name: push
+        image: gcr.io/go-containerregistry/crane:v0.21.9
+        env:
+        - name: DOCKER_CONFIG
+          value: /docker-config
+        - name: SSL_CERT_FILE
+          value: /data/ca-bundle.pem
+        args:
+        - copy
+        - docker.io/library/busybox:latest
+        - zot.production.other/test/busybox:latest
+        volumeMounts:
+        - name: data-volume
+          mountPath: /data
+        - name: docker-config
+          mountPath: /docker-config
+      volumes:
+      - name: data-volume
+        emptyDir: {}
+      - name: docker-config
+        emptyDir: {}
+      - name: spire-api
+        csi:
+          driver: csi.spiffe.io
+          readOnly: true
+      - name: psat
+        projected:
+          sources:
+          - serviceAccountToken:
+              path: token
+              audience: spire-identity-exchange
+              expirationSeconds: 3600

examples/bottom-turtle-ha/run-tests.sh

@@ -74,6 +74,10 @@ teardown() {
   sudo systemctl status spire-server@other || true
   kubectl describe job federation-test || true
   kubectl logs job/federation-test || true
+  sudo systemctl status spire-ha-agent@main || true
+  sudo systemctl status spiffe-socat-unix@k8s-kubelet-2 || true
+  sudo systemctl status spiffe-socat-unix@k8s-kubelet-3 || true
+  sudo systemctl status spiffe-socat-unix@k8s-kubelet-4 || true
   sudo spire-server entry show -instance a || true
   sudo spire-server entry show -instance b || true
   sudo systemctl status spire-controller-manager@a || true
@@ -107,10 +111,20 @@ teardown() {
   if [[ "$1" -ne 0 ]]; then
     get_namespace_details spire-server spire-system
     kubectl describe pod -n spire-system
+    # Only on failure: these are verbose, and the kubelet journal in particular is what
+    # tells you why an image pull came back anonymous rather than using the plugin.
+    for JOB in image-push image-pull image-push-denied; do
+      dump_job "${JOB}"
+    done
+    dump_zot
+    dump_kubelet_all
   fi
 
   if [ "${CLEANUP}" -eq 1 ]; then
     kubectl delete job federation-test 2>/dev/null || true
+    kubectl delete job image-push image-pull image-push-denied 2>/dev/null || true
+    helm uninstall --namespace zot zot 2>/dev/null || true
+    kubectl delete ns zot 2>/dev/null || true
     helm uninstall --namespace spire-mgmt spire-b 2>/dev/null || true
     helm uninstall --namespace spire-mgmt spire-a 2>/dev/null || true
     helm uninstall --namespace spire-mgmt spire 2>/dev/null || true
@@ -154,6 +168,120 @@ wait_for_trust_sync() {
   return 1
 }
 
+# Dump everything useful about a job in one block. The trace goes to stderr while command
+# output goes to stdout, and the two are separate streams in the CI log, so anything printed
+# here can interleave or drop. Suspend the trace, merge each command's streams, and bracket
+# the whole thing so it stays readable.
+dump_job() {
+  local job="$1"
+  set +x
+  echo "===== BEGIN ${job} ====="
+  kubectl get pods -l "job-name=${job}" -o wide 2>&1 || true
+  kubectl describe job "${job}" 2>&1 || true
+  # Pod events are where image pull and volume failures show up; the job has none of this.
+  kubectl describe pod -l "job-name=${job}" 2>&1 || true
+  # --prefix labels each line with its container. These jobs have three init containers and
+  # the interesting output is rarely the last one.
+  kubectl logs "job/${job}" --all-containers --prefix 2>&1 || true
+  # Everything kubelet said about this pod on the node that ran it. Filtering on the image
+  # misses the credential provider path, which names the pod and service account instead,
+  # and only the first pull attempt carries the real error; the retries are all backoff.
+  local pod node
+  for pod in $(kubectl get pods -l "job-name=${job}" -o name 2>/dev/null | cut -d/ -f2); do
+    node="$(kubectl get pod "${pod}" -o jsonpath='{.spec.nodeName}' 2>/dev/null)"
+    [ -n "${node}" ] || continue
+    echo "----- kubelet ${node} for pod ${pod} -----"
+    # Only this pod. A broader filter matches every provider line since boot and the
+    # window fills long before the pull happens.
+    docker exec -i "${node}" journalctl -u kubelet --no-pager 2>&1 \
+      | grep -F "${pod}" | head -60 || true
+    # The plugin exec is logged against the image and plugin name, not the pod, so a pod
+    # filter hides exactly the line that says whether it ran and what it returned.
+    echo "----- kubelet ${node} credential provider decisions -----"
+    docker exec -i "${node}" journalctl -u kubelet --no-pager 2>&1 \
+      | grep -E 'exec plugin|image credentials|k8s-image-cred|without credentials|zot\.production\.other|[Ss]ervice account' \
+      | tail -40 || true
+  done
+  echo "===== END ${job} ====="
+  set -x
+}
+
+# Same treatment as dump_job. zot logs at debug, and its rejection reason for a bearer
+# token only appears there, so take the whole log rather than a tail.
+dump_zot() {
+  set +x
+  echo "===== BEGIN zot ====="
+  kubectl get pods -n zot -o wide 2>&1 || true
+  kubectl describe pod -n zot -l app.kubernetes.io/name=zot 2>&1 || true
+  kubectl logs -n zot -l app.kubernetes.io/name=zot --all-containers --prefix 2>&1 || true
+  echo "===== END zot ====="
+  set -x
+}
+
+# Whether kubelet was configured with the image credential provider at all, and whether it
+# ran it. The kubeadm patch landing is the whole question when a pull comes back anonymous.
+dump_kubelet() {
+  local node="$1"
+  set +x
+  echo "===== BEGIN kubelet ${node} ====="
+  docker exec -i "${node}" cat /var/lib/kubelet/kubeadm-flags.env 2>&1 || true
+  docker exec -i "${node}" ps ax 2>&1 | grep '[k]ubelet' || true
+  docker exec -i "${node}" ls -l /credential-plugins /etc/kubernetes/credential-provider-config.yaml 2>&1 || true
+  # The plugin reaches the workload API through this bridge. If the socket is absent the
+  # plugin fails immediately, kubelet falls back to anonymous, and the pull 401s.
+  docker exec -i "${node}" ls -l /var/run/spire/agent/sockets/main/public/ 2>&1 || true
+  # Only the registry we care about. A broad credential grep is pure noise at v=4, which
+  # logs a provider line for every image pull on the node.
+  docker exec -i "${node}" journalctl -u kubelet --no-pager 2>&1 \
+    | grep -E 'zot\.production\.other' -A3 | tail -60 || true
+  echo "===== END kubelet ${node} ====="
+  set -x
+}
+
+dump_kubelet_all() {
+  for NODE in $(kubectl get nodes -o name 2>/dev/null | cut -d/ -f2); do
+    dump_kubelet "${NODE}"
+  done
+}
+
+# kubectl wait --for=condition=complete blocks the full timeout when a job has already
+# failed, which makes every failure look like a hang and delays the dump by minutes. Poll
+# both terminal conditions instead.
+wait_for_job() {
+  local job="$1"
+  local timeout="${2:-120}"
+  local count=0
+  while [ "$count" -lt "$timeout" ]; do
+    if kubectl get job "$job" -o jsonpath='{.status.conditions[?(@.type=="Complete")].status}' 2>/dev/null | grep -q True; then
+      return 0
+    fi
+    if kubectl get job "$job" -o jsonpath='{.status.conditions[?(@.type=="Failed")].status}' 2>/dev/null | grep -q True; then
+      echo "job/$job failed"
+      dump_job "$job"
+      return 1
+    fi
+    sleep 1
+    ((count++)) || true
+  done
+  echo "job/$job did not finish within ${timeout}s"
+  dump_job "$job"
+  return 1
+}
+
+wait_for_socket() {
+  local socket="$1"
+  local timeout=30
+  local count=0
+  while [ "$count" -lt "$timeout" ]; do
+    if [ -S "$socket" ]; then
+      return 0
+    fi
+    sleep 1
+    ((count++)) || true
+  done
+  return 1
+}
+
 wait_for_jwt() {
   local socket="$1"
   local timeout=30
@@ -199,11 +327,19 @@ run_federation_test_job() {
 # Get the package repo and install the packages
 sudo curl -s -o /etc/apt/sources.list.d/spire-examples.list https://raw.githubusercontent.com/spiffe/spire-examples/refs/heads/main/examples/debs/amd64/spire-examples.list
 sudo apt-get update
-sudo apt-get install -y spire-common spire-agent spire-server spire-controller-manager spiffe-socat-unix socat spire-trust-sync spiffe-helper
+sudo apt-get install -y spire-common spire-agent spire-server spire-controller-manager spiffe-socat-unix socat spire-trust-sync spiffe-helper spire-ha-agent
 
 # Set our testing trust domain
 sudo sed -i 's/example.org/production.other/' /etc/spiffe/default-trust-domain.env
 
+# A trust domain has one OIDC discovery endpoint, but the packaged root server config
+# advertises oidc-discovery-provider.<trust domain> while the charts advertise
+# oidc-discovery.<trust domain>. The identity exchange checks the iss claim by exact string,
+# so a JWT-SVID minted by a root server, which is what kubelet's credential provider
+# presents, is rejected unless the two agree. Align the roots with the charts.
+sudo sed -i 's|jwt_issuer = "https://oidc-discovery-provider\.|jwt_issuer = "https://oidc-discovery.|' /etc/spire/server/default.conf
+grep jwt_issuer /etc/spire/server/default.conf
+
 if [ "${BROKER}" -eq 1 ]; then
   # Pull the federation test job images out of the charts so they always sync up.
   AGENT_IMAGE=$(helm template t charts/spire -s charts/spire-agent/templates/daemonset.yaml --values "${COMMON_TEST_YOUR_VALUES}" --set spire-agent.enabled=true | yq e 'select(.kind=="DaemonSet") | .spec.template.spec.containers[] | select(.name=="spire-agent") | .image' -)
@@ -317,6 +453,36 @@ wait_for_jwt /var/run/spiffe/socat/unix/k8s-spire-agent-3-b/public/api.sock
 wait_for_jwt /var/run/spiffe/socat/unix/k8s-spire-agent-4-a/public/api.sock
 wait_for_jwt /var/run/spiffe/socat/unix/k8s-spire-agent-4-b/public/api.sock
 
+# Start the host spire-ha-agent. It merges the two root agents into one workload API, which is
+# what lets host services keep their identity when a single root server goes away. The compiled
+# in defaults already point at /var/run/spire/agent/sockets/{a,b}/private/admin.sock and listen
+# on the main instance socket, and the packaged agent config already lists the ha-agent in its
+# authorized_delegates, so no configuration is needed.
+sudo systemctl start spire-ha-agent@main
+# Not wait_for_healthcheck: that calls the grpc.health.v1 service, which the ha-agent does
+# not serve, so it always reports "unable to determine health". Not wait_for_jwt either:
+# the ha-agent attests callers by pid, and the cli invoking it is in no registered unit.
+# Readiness is proven through the bridges below, where the caller does have an entry.
+wait_for_socket /var/run/spire/agent/sockets/main/public/api.sock
+
+# Bridge the merged workload API into each virtual node for kubelet's image credential provider.
+# A real deployment runs one ha-agent per host and kubelet talks to it directly. Here a single VM
+# backs three virtual nodes, so we put one socat instance in front of the shared ha-agent per
+# node. The ha-agent attests each caller by pid, so every bridge resolves to its own entry and
+# each node still gets a distinct identity. Each bridge is mounted into its node at
+# /var/run/spire/agent/sockets/main/public, where a package installed ha-agent listens, so
+# kubelet's configuration inside the node is the same one a real host would use.
+sudo /bin/bash -c "echo SPIFFE_INSTANCE=main > /etc/spiffe/socat/unix/k8s-kubelet-2.conf"
+sudo /bin/bash -c "echo SPIFFE_INSTANCE=main > /etc/spiffe/socat/unix/k8s-kubelet-3.conf"
+sudo /bin/bash -c "echo SPIFFE_INSTANCE=main > /etc/spiffe/socat/unix/k8s-kubelet-4.conf"
+sudo systemctl start spiffe-socat-unix@k8s-kubelet-2 spiffe-socat-unix@k8s-kubelet-3 spiffe-socat-unix@k8s-kubelet-4
+# These front the ha-agent rather than a spire-agent, so healthcheck does not apply here
+# either. Fetching an svid is the real signal: it exercises the bridge, the ha-agent and
+# whichever root agent answered.
+wait_for_jwt /var/run/spiffe/socat/unix/k8s-kubelet-2/public/api.sock
+wait_for_jwt /var/run/spiffe/socat/unix/k8s-kubelet-3/public/api.sock
+wait_for_jwt /var/run/spiffe/socat/unix/k8s-kubelet-4/public/api.sock
+
 # Deploy an ingress controller
 IP=$(kubectl get nodes chart-testing-control-plane -o go-template='{{ range .status.addresses }}{{ if eq .type "InternalIP" }}{{ .address }}{{ end }}{{ end }}')
 helm upgrade --install ingress-nginx ingress-nginx --version "$VERSION_INGRESS_NGINX" --repo "$HELM_REPO_INGRESS_NGINX" \
@@ -332,7 +498,7 @@ common_test_url "$IP"
 # Get the host IP And add spire-server-[ab].${trust_domain} records to it so the spire-servers can talk back to root servers running on the host
 HOSTIP=$(ip addr show docker0 | grep 'inet ' | awk '{print $2}' | cut -d/ -f1)
 kubectl get configmap -n kube-system coredns -o yaml | grep hosts || kubectl get configmap -n kube-system coredns -o yaml | sed "/ready/a\        hosts {\n           fallthrough\n        }" | kubectl apply -f -
-kubectl get configmap -n kube-system coredns -o yaml | grep production.other || kubectl get configmap -n kube-system coredns -o yaml | sed "/hosts/a\           $HOSTIP spire-server-a.production.other\n           $IP oidc-discovery.production.other\n           $HOSTIP spire-server-b.production.other\n           127.0.0.1 $FEDERATION_ENDPOINT_HOST\n" | kubectl apply -f -
+kubectl get configmap -n kube-system coredns -o yaml | grep production.other || kubectl get configmap -n kube-system coredns -o yaml | sed "/hosts/a\           $HOSTIP spire-server-a.production.other\n           $IP oidc-discovery.production.other\n           $HOSTIP spire-server-b.production.other\n           $IP zot.production.other\n           $IP spire-identity-exchange-rest.production.other\n           127.0.0.1 $FEDERATION_ENDPOINT_HOST\n" | kubectl apply -f -
 kubectl rollout restart -n kube-system deployment/coredns
 kubectl rollout status -n kube-system -w --timeout=1m deploy/coredns
 
@@ -343,6 +509,10 @@ helm upgrade --install --create-namespace --namespace spire-mgmt --values "${COM
   --set "global.spire.namespaces.create=true" \
   --set "global.spire.ingressControllerType=ingress-nginx" \
   --set "spiffe-oidc-discovery-provider.ingress.enabled=true" \
+  --set "spireIdentityExchange.tls.rest.enabled=true" \
+  --set "spireIdentityExchange.tls.rest.ingress.enabled=true" \
+  --set "spireIdentityExchange.spiffe.rest.enabled=true" \
+  --set "spireIdentityExchange.spiffe.rest.ingress.enabled=true" \
   "${BROKER_MODE_ARGS[@]}"
 
 # Create spire-identity-exchange cert for testing.
@@ -422,6 +592,80 @@ TOKEN=$(kubectl logs job/test)
 curl --fail-with-body -H "Authorization: Bearer ${TOKEN}" -X POST --resolve "spire-identity-exchange-a-rest.production.other:443:$IP" "https://spire-identity-exchange-a-rest.production.other/api/v1/svid/k8s_psat/x509" -k -sS -q
 curl --fail-with-body -H "Authorization: Bearer ${TOKEN}" -X POST --resolve "spire-identity-exchange-b-rest.production.other:443:$IP" "https://spire-identity-exchange-b-rest.production.other/api/v1/svid/k8s_psat/x509" -k -sS -q
 
+# Registry image pull. zot serves a SPIRE issued certificate, an in cluster job pushes an
+# image with an identity minted by the exchange, and kubelet pulls it back through the
+# image credential provider staged on every node by .github/scripts.
+
+# Nodes are not cluster DNS clients, so coredns does nothing for containerd. Give each
+# node the name directly, and a hosts.toml so it trusts the registry's SPIRE certificate.
+# The bundle has to carry both roots: after a failover the certificate is issued by the
+# other side's chain.
+sudo spire-server bundle show -socketPath /run/spire/server/sockets/a/private/api.sock | sudo tee /tmp/zot-ca.pem > /dev/null
+sudo spire-server bundle show -socketPath /run/spire/server/sockets/b/private/api.sock | sudo tee -a /tmp/zot-ca.pem > /dev/null
+for NODE in $(kubectl get nodes -o name | cut -d/ -f2); do
+  # The credential provider runs on the node, not in a pod, so it resolves the registry
+  # and the exchange here rather than through coredns.
+  docker exec -i "${NODE}" /bin/bash -c "grep -q zot.production.other /etc/hosts || echo '$IP zot.production.other spire-identity-exchange-rest-spiffe.production.other' >> /etc/hosts"
+  docker exec -i "${NODE}" /bin/bash -c "mkdir -p /etc/containerd/certs.d/zot.production.other"
+  docker exec -i "${NODE}" /bin/bash -c "cat > /etc/containerd/certs.d/zot.production.other/zot-ca.pem" < /tmp/zot-ca.pem
+  docker exec -i "${NODE}" /bin/bash -c "cat > /etc/containerd/certs.d/zot.production.other/hosts.toml" <<EOF
+server = "https://zot.production.other"
+
+[host."https://zot.production.other"]
+  capabilities = ["pull", "resolve"]
+  ca = "/etc/containerd/certs.d/zot.production.other/zot-ca.pem"
+EOF
+done
+
+helm upgrade --install zot zot --version "$VERSION_ZOT" --repo "$HELM_REPO_ZOT" \
+  --namespace zot --create-namespace \
+  --values "${SCRIPTPATH}/zot-values.yaml" \
+  --wait --timeout 5m
+
+# Pull the job images out of the charts so they always sync up.
+BUSYBOX_IMAGE=$(helm template t charts/spire -s charts/spiffe-oidc-discovery-provider/templates/tests/test-keys.yaml --values "${COMMON_TEST_YOUR_VALUES}" --set spiffe-oidc-discovery-provider.enabled=true | yq e 'select(.kind=="Pod") | .spec.initContainers[] | select(.name=="static-busybox") | .image' -)
+AGENT_IMAGE=$(helm template t charts/spire -s charts/spire-agent/templates/daemonset.yaml --values "${COMMON_TEST_YOUR_VALUES}" --set spire-agent.enabled=true | yq e 'select(.kind=="DaemonSet") | .spec.template.spec.containers[] | select(.name=="spire-agent") | .image' -)
+TOOLKIT_IMAGE=$(helm template t charts/spire -s charts/spiffe-oidc-discovery-provider/templates/tests/test-keys.yaml --values "${COMMON_TEST_YOUR_VALUES}" --set spiffe-oidc-discovery-provider.enabled=true | yq e 'select(.kind=="Pod") | .spec.containers[] | select(.name=="verify-keys") | .image' -)
+echo "image pull job images: ${BUSYBOX_IMAGE} ${AGENT_IMAGE} ${TOOLKIT_IMAGE}"
+
+# Substitute rather than rewrite. yq edits the document structure, which is version
+# dependent and on a multi document file can graft fields onto the wrong document; sed
+# cannot restructure anything. Writing to a file first also keeps the applied manifest
+# inspectable and avoids the pipeline trap, since this script sets -e but not -o pipefail.
+apply_registry_job() {
+  local rendered
+  rendered="/tmp/$(basename "$1")"
+  sed -e "s|IMAGE_BUSYBOX|${BUSYBOX_IMAGE}|g" \
+      -e "s|IMAGE_SPIRE_AGENT|${AGENT_IMAGE}|g" \
+      -e "s|IMAGE_TOOLKIT|${TOOLKIT_IMAGE}|g" \
+      "$1" > "${rendered}"
+  # Fail loudly rather than applying a half substituted manifest.
+  if grep -q 'IMAGE_BUSYBOX\|IMAGE_SPIRE_AGENT\|IMAGE_TOOLKIT' "${rendered}"; then
+    echo "unsubstituted image placeholder left in ${rendered}"
+    exit 1
+  fi
+  kubectl apply -f "${rendered}"
+}
+
+# Push with the writer identity.
+apply_registry_job "${SCRIPTPATH}/image-push-job.yaml"
+wait_for_job image-push
+
+# Pull it back. Nothing in the job fetches a credential; kubelet runs the plugin, which is
+# the whole point of the test.
+kubectl apply -f "${SCRIPTPATH}/image-pull-job.yaml"
+wait_for_job image-pull
+# Completing at all is the assertion: zot grants no anonymous access, so the image only
+# comes down if kubelet ran the plugin and the exchange minted a token zot accepted. The
+# kubelet log line naming the plugin needs -v=4, which is not worth turning on for every
+# example, so teardown prints it as a diagnostic rather than asserting on it.
+kubectl logs job/image-pull | grep IMAGE-PULL-OK
+
+# The pull identity is read only. This job completes only when zot refuses the write.
+apply_registry_job "${SCRIPTPATH}/image-push-denied-job.yaml"
+wait_for_job image-push-denied
+kubectl logs job/image-push-denied | grep PUSH-DENIED-OK
+
 if [ "${BROKER}" -eq 1 ]; then
   # Verify a workload on the ha-agent socket receives the other.invalid federated trust bundles,
   # x509 and jwt, merged from both sides.
@@ -441,3 +685,12 @@ if [ "${BROKER}" -eq 1 ]; then
   run_federation_test_job
 fi
 
+# The image pull path has to survive losing a side too. Everything it depends on is HA:
+# the node's identity comes from the host spire-ha-agent, and the credential provider
+# talks to the combined exchange endpoint rather than either side directly. Delete the
+# job first so this is a genuine second pull rather than a cached result.
+kubectl delete job image-pull
+kubectl apply -f "${SCRIPTPATH}/image-pull-job.yaml"
+wait_for_job image-pull
+kubectl logs job/image-pull | grep IMAGE-PULL-OK
+

examples/bottom-turtle-ha/spire-identity-exchange-values.yaml

@@ -1,6 +1,14 @@
 internal-spire-server-bottom-turtle-ha-a: &server
   controllerManager:
     identities:
+      clusterSPIFFEIDs:
+        zot:
+          spiffeIDTemplate: spiffe://{{ .TrustDomain }}/zot
+          podSelector:
+            matchLabels:
+              app.kubernetes.io/name: zot
+          dnsNameTemplates:
+            - zot.{{ .TrustDomain }}
       clusterStaticEntries:
         test:
           parentID: spiffe://production.other/spire-identity-exchange
@@ -8,6 +16,33 @@ internal-spire-server-bottom-turtle-ha-a: &server
           selectors:
             - k8s_psat:namespace:default
             - k8s_psat:service_account_name:default
+
+        image-push:
+          parentID: spiffe://production.other/spire-identity-exchange
+          spiffeID: spiffe://production.other/image-push
+          selectors:
+            - k8s_psat:namespace:default
+            - k8s_psat:service_account_name:zot-push
+            - spiffe:source_path:/ns/default/sa/zot-push
+            - spire_identity_exchange:stack:name:image_pull
+
+        image-pull:
+          parentID: spiffe://production.other/spire-identity-exchange
+          spiffeID: spiffe://production.other/image-pull
+          selectors:
+            - k8s_psat:namespace:default
+            - k8s_psat:service_account_name:zot-pull
+            - spiffe:source_path:/kubelet
+            - spire_identity_exchange:stack:name:image_pull
+
+        image-pull-from-pod:
+          parentID: spiffe://production.other/spire-identity-exchange
+          spiffeID: spiffe://production.other/image-pull
+          selectors:
+            - k8s_psat:namespace:default
+            - k8s_psat:service_account_name:zot-pull
+            - spiffe:source_path:/ns/default/sa/zot-pull
+            - spire_identity_exchange:stack:name:image_pull
   spireIdentityExchange:
     enabled: true
 
@@ -30,6 +65,14 @@ spire-identity-exchange-bottom-turtle-ha-a: &six
         config:
           allowedServiceAccounts:
             - default/default
+            - default/zot-push
+            - default/zot-pull
+      spiffe:
+        config:
+          pathPatterns:
+            - "^/kubelet$"
+            - "^/ns/default/sa/zot-push$"
+            - "^/ns/default/sa/zot-pull$"
 
 #Set the same settings on the B side
 spire-identity-exchange-bottom-turtle-ha-b: *six

examples/bottom-turtle-ha/zot-values.yaml

@@ -0,0 +1,154 @@
+# zot registry with a SPIRE issued serving certificate.
+
+image:
+  tag: v2.1.20
+
+initContainers:
+  - name: spiffe-helper-init
+    image: ghcr.io/spiffe/spiffe-helper:0.11.0
+    args: ["-config", "/etc/spiffe-helper.conf", "-daemon-mode=false"]
+    securityContext:
+      allowPrivilegeEscalation: false
+      runAsNonRoot: true
+      readOnlyRootFilesystem: true
+      capabilities:
+        drop: [ALL]
+      seccompProfile:
+        type: RuntimeDefault
+    volumeMounts:
+      - name: spiffe-workload-api
+        mountPath: /spiffe-workload-api
+        readOnly: true
+      - name: zot-config
+        mountPath: /etc/spiffe-helper.conf
+        subPath: spiffe-helper.conf
+        readOnly: true
+      - name: spire-svid
+        mountPath: /svid
+
+extraContainers:
+  - name: spiffe-helper
+    image: ghcr.io/spiffe/spiffe-helper:0.11.0
+    args: ["-config", "/etc/spiffe-helper.conf"]
+    securityContext:
+      allowPrivilegeEscalation: false
+      runAsNonRoot: true
+      readOnlyRootFilesystem: true
+      capabilities:
+        drop: [ALL]
+      seccompProfile:
+        type: RuntimeDefault
+    volumeMounts:
+      - name: spiffe-workload-api
+        mountPath: /spiffe-workload-api
+        readOnly: true
+      - name: zot-config
+        mountPath: /etc/spiffe-helper.conf
+        subPath: spiffe-helper.conf
+        readOnly: true
+      - name: spire-svid
+        mountPath: /svid
+
+extraVolumes:
+  - name: spire-svid
+    emptyDir: {}
+  - name: spiffe-workload-api
+    csi:
+      driver: csi.spiffe.io
+      readOnly: true
+
+extraVolumeMounts:
+  - name: spire-svid
+    mountPath: /svid
+    readOnly: true
+
+podSecurityContext:
+  fsGroupChangePolicy: OnRootMismatch
+  runAsUser: 1000
+  runAsGroup: 1000
+  fsGroup: 1000
+
+securityContext:
+  allowPrivilegeEscalation: false
+  runAsNonRoot: true
+  capabilities:
+    drop: [ALL]
+  seccompProfile:
+    type: RuntimeDefault
+
+resources: {}
+
+httpGet:
+  scheme: HTTPS
+  port: 5000
+
+startupProbe:
+  initialDelaySeconds: 5
+  periodSeconds: 10
+  failureThreshold: 30
+
+mountConfig: true
+configFiles:
+  spiffe-helper.conf: |-
+    agent_address = "/spiffe-workload-api/spire-agent.sock"
+    cert_dir = "/svid"
+    svid_file_name = "tls.crt"
+    svid_key_file_name = "tls.key"
+    svid_bundle_file_name = "ca.pem"
+  config.json: |-
+    {
+      "storage": { "rootDirectory": "/var/lib/registry" },
+      "http": {
+        "address": "0.0.0.0",
+        "port": "5000",
+        "compat": ["docker2s2"],
+        "tls": { "cert": "/svid/tls.crt", "key": "/svid/tls.key" },
+        "realm": "zot",
+        "auth": {
+          "bearer": {
+            "realm": "https://zot.production.other/zot/auth/token",
+            "service": "https://zot.production.other",
+            "oidc": [
+              {
+                "issuer": "https://oidc-discovery.production.other",
+                "audiences": ["zot"],
+                "certificateAuthorityFile": "/svid/ca.pem",
+                "claimMapping": { "username": "claims.sub" }
+              }
+            ]
+          }
+        },
+        "accessControl": {
+          "repositories": {
+            "**": {
+              "policies": [
+                {
+                  "users": ["spiffe://production.other/image-push"],
+                  "actions": ["read", "create", "update", "delete"]
+                },
+                {
+                  "users": ["spiffe://production.other/image-pull"],
+                  "actions": ["read"]
+                }
+              ]
+            }
+          }
+        }
+      },
+      "log": { "level": "debug" }
+    }
+
+service:
+  type: ClusterIP
+
+ingress:
+  enabled: true
+  className: nginx
+  pathtype: Prefix
+  annotations:
+    nginx.ingress.kubernetes.io/ssl-passthrough: "true"
+    nginx.ingress.kubernetes.io/backend-protocol: "HTTPS"
+  hosts:
+    - host: zot.production.other
+      paths:
+        - path: /