From 2a8c1b63c45588f7d978bbd06d5ec2305f5e2a15 Mon Sep 17 00:00:00 2001 From: kfox1111 Date: Fri, 4 Sep 2026 12:50:34 -0700 Subject: [PATCH] Bottom turtle ha registry tests (#929) * Bottom turtle ha registry tests Signed-off-by: Kevin Fox * Fix test because of spire-ha-agent Signed-off-by: Kevin Fox * Fix config options Signed-off-by: Kevin Fox * Fix yq issue and log better Signed-off-by: Kevin Fox * Fix some things Signed-off-by: Kevin Fox * Change timeout and fix image Signed-off-by: Kevin Fox * Bump up zot version. Signed-off-by: Kevin Fox * More debugging stuff Signed-off-by: Kevin Fox * More debugging stuff Signed-off-by: Kevin Fox * More fixes Signed-off-by: Kevin Fox * More logging Signed-off-by: Kevin Fox * More logging Signed-off-by: Kevin Fox * More logging Signed-off-by: Kevin Fox * More logging Signed-off-by: Kevin Fox * More logging Signed-off-by: Kevin Fox * Fix test Signed-off-by: Kevin Fox * Log less Signed-off-by: Kevin Fox * Cleanup Signed-off-by: Kevin Fox * Make test less different then real deployment Signed-off-by: Kevin Fox --------- Signed-off-by: Kevin Fox Signed-off-by: kfox1111 --- .../kind/conf/credential-provider-config.yaml | 25 ++ .github/kind/conf/kind-config.yaml | 99 +++++++ .../scripts/install-image-cred-provider.sh | 37 +++ .github/tests/charts.json | 5 + .github/workflows/helm-chart-ci-ignore.yaml | 24 +- .github/workflows/helm-chart-ci.yaml | 38 ++- .gitignore | 1 + charts/spire-ha-agent/Chart.yaml | 2 +- charts/spire-ha-agent/README.md | 2 +- examples/bottom-turtle-ha/README.md | 75 +++++ .../node1-spire-ha-agent.yaml | 11 + .../example-manifests/node2-image-pull.yaml | 10 + .../example-manifests/node2-kubelet.yaml | 10 + .../example-manifests/node3-image-pull.yaml | 10 + .../example-manifests/node3-kubelet.yaml | 10 + .../example-manifests/node4-image-pull.yaml | 10 + .../example-manifests/node4-kubelet.yaml | 10 + examples/bottom-turtle-ha/image-pull-job.yaml | 34 +++ .../image-push-denied-job.yaml | 122 +++++++++ examples/bottom-turtle-ha/image-push-job.yaml | 121 +++++++++ examples/bottom-turtle-ha/run-tests.sh | 257 +++++++++++++++++- .../spire-identity-exchange-values.yaml | 43 +++ examples/bottom-turtle-ha/zot-values.yaml | 154 +++++++++++ 23 files changed, 1082 insertions(+), 28 deletions(-) create mode 100644 .github/kind/conf/credential-provider-config.yaml create mode 100755 .github/scripts/install-image-cred-provider.sh create mode 100644 examples/bottom-turtle-ha/example-manifests/node1-spire-ha-agent.yaml create mode 100644 examples/bottom-turtle-ha/example-manifests/node2-image-pull.yaml create mode 100644 examples/bottom-turtle-ha/example-manifests/node2-kubelet.yaml create mode 100644 examples/bottom-turtle-ha/example-manifests/node3-image-pull.yaml create mode 100644 examples/bottom-turtle-ha/example-manifests/node3-kubelet.yaml create mode 100644 examples/bottom-turtle-ha/example-manifests/node4-image-pull.yaml create mode 100644 examples/bottom-turtle-ha/example-manifests/node4-kubelet.yaml create mode 100644 examples/bottom-turtle-ha/image-pull-job.yaml create mode 100644 examples/bottom-turtle-ha/image-push-denied-job.yaml create mode 100644 examples/bottom-turtle-ha/image-push-job.yaml create mode 100644 examples/bottom-turtle-ha/zot-values.yaml diff --git a/.github/kind/conf/credential-provider-config.yaml b/.github/kind/conf/credential-provider-config.yaml new file mode 100644 index 0000000..40d1e5b --- /dev/null +++ b/.github/kind/conf/credential-provider-config.yaml @@ -0,0 +1,25 @@ +apiVersion: kubelet.config.k8s.io/v1 +kind: CredentialProviderConfig +providers: + - name: k8s-image-cred-spire-identity-exchange + matchImages: + - "zot.production.other" + defaultCacheDuration: "0s" + apiVersion: credentialprovider.kubelet.k8s.io/v1 + args: + - "--username=zot" + - "--mode=spire-identity-exchange" + - "--url=https://spire-identity-exchange-rest-spiffe.production.other" + - "--stack=image_pull" + - "--registry-audience=zot" + - "--spiffe-audience=spire-identity-exchange" + - "--spiffe-hint=image-pull" + - "--spiffe-id=spiffe://production.other/service/spire-identity-exchange" + - "--timeout=10s" + env: + - name: SPIFFE_ENDPOINT_SOCKET + value: unix:///var/run/spire/agent/sockets/main/public/api.sock + tokenAttributes: + serviceAccountTokenAudience: "spire-identity-exchange" + cacheType: "Token" + requireServiceAccount: true diff --git a/.github/kind/conf/kind-config.yaml b/.github/kind/conf/kind-config.yaml index d98b5c6..e0a289f 100644 --- a/.github/kind/conf/kind-config.yaml +++ b/.github/kind/conf/kind-config.yaml @@ -30,6 +30,39 @@ nodes: containerPath: /var/run/spiffe/socat/unix/k8s-spire-agent-a/public - hostPath: /var/run/spiffe/socat/unix/k8s-spire-agent-2-b/public containerPath: /var/run/spiffe/socat/unix/k8s-spire-agent-b/public + # One spire-ha-agent on the host is shared by all three virtual nodes, with a + # spiffe-socat-unix bridge per node in front of it. Mount that bridge where a + # package installed spire-ha-agent@main listens, so kubelet's configuration in + # the node is the same one a real bare metal node would use. + - hostPath: /var/run/spiffe/socat/unix/k8s-kubelet-2/public + containerPath: /var/run/spire/agent/sockets/main/public + - hostPath: ./.github/kind/conf/credential-providers + containerPath: /credential-plugins + - hostPath: ./.github/kind/conf/credential-provider-config.yaml + containerPath: /etc/kubernetes/credential-provider-config.yaml + kubeadmConfigPatches: + - | + apiVersion: kubeadm.k8s.io/v1beta3 + kind: JoinConfiguration + nodeRegistration: + kubeletExtraArgs: + image-credential-provider-config: /etc/kubernetes/credential-provider-config.yaml + image-credential-provider-bin-dir: /credential-plugins + kubeadmConfigPatchesJSON6902: + - group: kubeadm.k8s.io + version: v1beta4 + kind: JoinConfiguration + patch: | + - op: add + path: /nodeRegistration/kubeletExtraArgs/- + value: + name: image-credential-provider-config + value: /etc/kubernetes/credential-provider-config.yaml + - op: add + path: /nodeRegistration/kubeletExtraArgs/- + value: + name: image-credential-provider-bin-dir + value: /credential-plugins - role: worker extraMounts: - hostPath: /var/run/spiffe/socat/unix/k8s-spire-server-a/public @@ -40,6 +73,39 @@ nodes: containerPath: /var/run/spiffe/socat/unix/k8s-spire-agent-a/public - hostPath: /var/run/spiffe/socat/unix/k8s-spire-agent-3-b/public containerPath: /var/run/spiffe/socat/unix/k8s-spire-agent-b/public + # One spire-ha-agent on the host is shared by all three virtual nodes, with a + # spiffe-socat-unix bridge per node in front of it. Mount that bridge where a + # package installed spire-ha-agent@main listens, so kubelet's configuration in + # the node is the same one a real bare metal node would use. + - hostPath: /var/run/spiffe/socat/unix/k8s-kubelet-3/public + containerPath: /var/run/spire/agent/sockets/main/public + - hostPath: ./.github/kind/conf/credential-providers + containerPath: /credential-plugins + - hostPath: ./.github/kind/conf/credential-provider-config.yaml + containerPath: /etc/kubernetes/credential-provider-config.yaml + kubeadmConfigPatches: + - | + apiVersion: kubeadm.k8s.io/v1beta3 + kind: JoinConfiguration + nodeRegistration: + kubeletExtraArgs: + image-credential-provider-config: /etc/kubernetes/credential-provider-config.yaml + image-credential-provider-bin-dir: /credential-plugins + kubeadmConfigPatchesJSON6902: + - group: kubeadm.k8s.io + version: v1beta4 + kind: JoinConfiguration + patch: | + - op: add + path: /nodeRegistration/kubeletExtraArgs/- + value: + name: image-credential-provider-config + value: /etc/kubernetes/credential-provider-config.yaml + - op: add + path: /nodeRegistration/kubeletExtraArgs/- + value: + name: image-credential-provider-bin-dir + value: /credential-plugins - role: worker extraMounts: - hostPath: /var/run/spiffe/socat/unix/k8s-spire-server-a/public @@ -50,3 +116,36 @@ nodes: containerPath: /var/run/spiffe/socat/unix/k8s-spire-agent-a/public - hostPath: /var/run/spiffe/socat/unix/k8s-spire-agent-4-b/public containerPath: /var/run/spiffe/socat/unix/k8s-spire-agent-b/public + # One spire-ha-agent on the host is shared by all three virtual nodes, with a + # spiffe-socat-unix bridge per node in front of it. Mount that bridge where a + # package installed spire-ha-agent@main listens, so kubelet's configuration in + # the node is the same one a real bare metal node would use. + - hostPath: /var/run/spiffe/socat/unix/k8s-kubelet-4/public + containerPath: /var/run/spire/agent/sockets/main/public + - hostPath: ./.github/kind/conf/credential-providers + containerPath: /credential-plugins + - hostPath: ./.github/kind/conf/credential-provider-config.yaml + containerPath: /etc/kubernetes/credential-provider-config.yaml + kubeadmConfigPatches: + - | + apiVersion: kubeadm.k8s.io/v1beta3 + kind: JoinConfiguration + nodeRegistration: + kubeletExtraArgs: + image-credential-provider-config: /etc/kubernetes/credential-provider-config.yaml + image-credential-provider-bin-dir: /credential-plugins + kubeadmConfigPatchesJSON6902: + - group: kubeadm.k8s.io + version: v1beta4 + kind: JoinConfiguration + patch: | + - op: add + path: /nodeRegistration/kubeletExtraArgs/- + value: + name: image-credential-provider-config + value: /etc/kubernetes/credential-provider-config.yaml + - op: add + path: /nodeRegistration/kubeletExtraArgs/- + value: + name: image-credential-provider-bin-dir + value: /credential-plugins diff --git a/.github/scripts/install-image-cred-provider.sh b/.github/scripts/install-image-cred-provider.sh new file mode 100755 index 0000000..29e0768 --- /dev/null +++ b/.github/scripts/install-image-cred-provider.sh @@ -0,0 +1,37 @@ +#!/usr/bin/env bash + +set -euo pipefail + +VERSION="${IMAGE_CRED_PROVIDER_VERSION:-v0.5.0}" +BIN_NAME="k8s-image-cred-spire-identity-exchange" + +SCRIPTPATH="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +BIN_DIR="${SCRIPTPATH}/../kind/conf/credential-providers" + +case "$(uname -m)" in + x86_64|amd64) ARCH="x86_64" ;; + aarch64|arm64) ARCH="arm64" ;; + *) echo "unsupported architecture: $(uname -m)" >&2; exit 1 ;; +esac + +if [ -x "${BIN_DIR}/${BIN_NAME}" ]; then + echo "${BIN_NAME} already staged in ${BIN_DIR}" + exit 0 +fi + +BASE_URL="https://github.com/spiffe/spire-identity-exchange/releases/download/${VERSION}" +ARCHIVE="${BIN_NAME}_Linux_${ARCH}.tar.gz" +CHECKSUMS="spire-identity-exchange_${VERSION#v}_checksums.txt" + +WORKDIR="$(mktemp -d)" +trap 'rm -rf "${WORKDIR}"' EXIT + +curl -fsSL --retry 5 --retry-all-errors -o "${WORKDIR}/${ARCHIVE}" "${BASE_URL}/${ARCHIVE}" +curl -fsSL --retry 5 --retry-all-errors -o "${WORKDIR}/${CHECKSUMS}" "${BASE_URL}/${CHECKSUMS}" +(cd "${WORKDIR}" && grep " ${ARCHIVE}\$" "${CHECKSUMS}" | sha256sum -c -) + +tar -xzf "${WORKDIR}/${ARCHIVE}" -C "${WORKDIR}" "${BIN_NAME}" +mkdir -p "${BIN_DIR}" +install -m 0755 "${WORKDIR}/${BIN_NAME}" "${BIN_DIR}/${BIN_NAME}" + +echo "Staged ${BIN_NAME} ${VERSION} in ${BIN_DIR}" diff --git a/.github/tests/charts.json b/.github/tests/charts.json index 19e8692..4c5666d 100644 --- a/.github/tests/charts.json +++ b/.github/tests/charts.json @@ -13,5 +13,10 @@ "name": "ingress-nginx", "repo": "https://kubernetes.github.io/ingress-nginx", "version": "4.15.1" + }, + { + "name": "zot", + "repo": "https://zotregistry.dev/helm-charts", + "version": "0.1.122" } ] diff --git a/.github/workflows/helm-chart-ci-ignore.yaml b/.github/workflows/helm-chart-ci-ignore.yaml index 0bf2124..5bde7e2 100644 --- a/.github/workflows/helm-chart-ci-ignore.yaml +++ b/.github/workflows/helm-chart-ci-ignore.yaml @@ -31,9 +31,9 @@ jobs: strategy: matrix: k8s: - - v1.33.7 - - v1.34.3 - - v1.35.1 + - v1.34.8 + - v1.35.5 + - v1.36.1 steps: - run: 'echo "Skipping tests"' @@ -75,9 +75,9 @@ jobs: strategy: matrix: k8s: - - v1.33.7 - - v1.34.3 - - v1.35.1 + - v1.34.8 + - v1.35.5 + - v1.36.1 example: - ${{ fromJson(needs.build-matrix.outputs.examples) }} @@ -93,9 +93,9 @@ jobs: strategy: matrix: k8s: - - v1.33.7 - - v1.34.3 - - v1.35.1 + - v1.34.8 + - v1.35.5 + - v1.36.1 example: - ${{ fromJson(needs.build-matrix.outputs.integrationtests) }} @@ -111,9 +111,9 @@ jobs: strategy: matrix: k8s: - - v1.33.7 - - v1.34.3 - - v1.35.1 + - v1.34.8 + - v1.35.5 + - v1.36.1 steps: - run: 'echo "Skipping upgrade-test"' diff --git a/.github/workflows/helm-chart-ci.yaml b/.github/workflows/helm-chart-ci.yaml index ec7b9b3..4867fb8 100644 --- a/.github/workflows/helm-chart-ci.yaml +++ b/.github/workflows/helm-chart-ci.yaml @@ -17,6 +17,7 @@ on: - '.github/tests/**/*.sh' - '.github/tests/**/*.json' - '.github/scripts/check-readme-versions.sh' + - '.github/scripts/install-image-cred-provider.sh' - 'examples/**/*.yaml' - 'examples/**/*.sh' - 'tests/**/*' @@ -31,6 +32,7 @@ env: PYTHON_VERSION: 3.11.3 KIND_VERSION: v0.32.0 CHART_TESTING_VERSION: v3.8.0 + IMAGE_CRED_PROVIDER_VERSION: v0.5.0 jobs: checks: @@ -160,9 +162,9 @@ jobs: # Kubernetes, but can go back farther as long as we don't need heroics # to pull it off (i.e. kubectl version juggling). k8s: - - v1.33.7 - - v1.34.3 - - v1.35.1 + - v1.34.8 + - v1.35.5 + - v1.36.1 steps: - name: Checkout @@ -185,6 +187,9 @@ jobs: with: version: ${{ env.CHART_TESTING_VERSION }} + - name: Install image credential provider + run: ./.github/scripts/install-image-cred-provider.sh + - name: Create kind ${{ matrix.k8s }} cluster uses: helm/kind-action@v1.14.0 # Only build a kind cluster if there are chart changes to test. @@ -251,9 +256,9 @@ jobs: fail-fast: false matrix: k8s: - - v1.33.7 - - v1.34.3 - - v1.35.1 + - v1.34.8 + - v1.35.5 + - v1.36.1 example: - ${{ fromJson(needs.build-matrix.outputs.examples) }} @@ -271,6 +276,9 @@ jobs: with: python-version: ${{ env.PYTHON_VERSION }} + - name: Install image credential provider + run: ./.github/scripts/install-image-cred-provider.sh + - name: Create kind cluster uses: helm/kind-action@v1.14.0 # Only build a kind cluster if there are chart changes to test. @@ -306,9 +314,9 @@ jobs: fail-fast: false matrix: k8s: - - v1.33.7 - - v1.34.3 - - v1.35.1 + - v1.34.8 + - v1.35.5 + - v1.36.1 integrationtest: - ${{ fromJson(needs.build-matrix.outputs.integrationtests) }} @@ -326,6 +334,9 @@ jobs: with: python-version: ${{ env.PYTHON_VERSION }} + - name: Install image credential provider + run: ./.github/scripts/install-image-cred-provider.sh + - name: Create kind cluster uses: helm/kind-action@v1.14.0 # Only build a kind cluster if there are chart changes to test. @@ -354,9 +365,9 @@ jobs: fail-fast: false matrix: k8s: - - v1.33.7 - - v1.34.3 - - v1.35.1 + - v1.34.8 + - v1.35.5 + - v1.36.1 steps: - name: Checkout @@ -372,6 +383,9 @@ jobs: with: python-version: ${{ env.PYTHON_VERSION }} + - name: Install image credential provider + run: ./.github/scripts/install-image-cred-provider.sh + - name: Create kind cluster uses: helm/kind-action@v1.14.0 # Only build a kind cluster if there are chart changes to test. diff --git a/.gitignore b/.gitignore index beb5ec8..3f4ab01 100644 --- a/.gitignore +++ b/.gitignore @@ -4,3 +4,4 @@ bin/ *.swp charts/**/*.tgz .DS_Store +.github/kind/conf/credential-providers/ diff --git a/charts/spire-ha-agent/Chart.yaml b/charts/spire-ha-agent/Chart.yaml index b0b9b1e..b576192 100644 --- a/charts/spire-ha-agent/Chart.yaml +++ b/charts/spire-ha-agent/Chart.yaml @@ -3,7 +3,7 @@ name: spire-ha-agent description: A Helm chart to install the SPIRE HA agent. type: application version: 0.3.1 -appVersion: "0.3.0" +appVersion: "0.4.0" keywords: ["spiffe", "spire-ha-agent"] home: https://github.com/spiffe/helm-charts-hardened/tree/main/charts/spire-ha-agent sources: diff --git a/charts/spire-ha-agent/README.md b/charts/spire-ha-agent/README.md index 915fdf6..a665cc8 100644 --- a/charts/spire-ha-agent/README.md +++ b/charts/spire-ha-agent/README.md @@ -1,6 +1,6 @@ # spire-ha-agent -![Version: 0.3.1](https://img.shields.io/badge/Version-0.3.1-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 0.3.0](https://img.shields.io/badge/AppVersion-0.3.0-informational?style=flat-square) +![Version: 0.3.1](https://img.shields.io/badge/Version-0.3.1-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 0.4.0](https://img.shields.io/badge/AppVersion-0.4.0-informational?style=flat-square) A Helm chart to install the SPIRE HA agent. diff --git a/examples/bottom-turtle-ha/README.md b/examples/bottom-turtle-ha/README.md index 396edb3..57796dd 100644 --- a/examples/bottom-turtle-ha/README.md +++ b/examples/bottom-turtle-ha/README.md @@ -136,3 +136,78 @@ helm upgrade --install --namespace spire-mgmt --values "spire-values.yaml" \ --set tags.bottomTurtleHAB=true \ --set "global.spire.ingressControllerType=ingress-nginx" ``` + +## Host services on the bottom turtle + +The diagrams above show a `spire-ha-agent` on each host, fed by `spire-agent@a` and +`spire-agent@b`, serving host services such as sshd and kubelet. That is what makes a host +service's identity survive one root server going away: the ha-agent merges both sides into +a single Workload API and answers from whichever side is up. + +It attests every caller by pid, so one ha-agent can serve many callers with different +identities. Register the caller against the root servers and it gets its own SVID: + +``` +apiVersion: spire.spiffe.io/v1alpha1 +kind: ClusterStaticEntry +metadata: + name: node1-spire-ha-agent +spec: + parentID: spiffe://${SPIFFE_TRUST_DOMAIN}/agent/node1 + spiffeID: spiffe://${SPIFFE_TRUST_DOMAIN}/spire-ha-agent + selectors: + - systemd:id:spire-ha-agent@main.service + federatesWith: + - spire-ha +``` + +The packaged `spire-agent` config already names `spiffe://${SPIFFE_TRUST_DOMAIN}/spire-ha-agent` +in its `authorized_delegates`, so no agent configuration is needed, only the entry. + +## Registry image pull + +Kubelet can use that host identity to pull images, without any pull secret. On seeing an +image from the registry, kubelet runs an image credential provider on the node, which +presents two credentials to the spire-identity-exchange: the pod's projected service +account token and the node's own JWT-SVID from the ha-agent. The exchange mints a registry +token, and the registry authorizes by SPIFFE ID. + +The registry in this example is zot, deployed with the upstream chart. Its serving +certificate is a SPIRE SVID delivered by `spiffe-helper` as an init container plus a +sidecar, so nothing carries a long lived key. The identity needs an explicit DNS name, +because an X509-SVID has only a URI SAN by default and containerd validates the registry +by hostname: + +``` +zot: + spiffeIDTemplate: spiffe://{{ .TrustDomain }}/zot + podSelector: + matchLabels: + app.kubernetes.io/name: zot + dnsNameTemplates: + - zot.{{ .TrustDomain }} +``` + +Push and pull share one exchange stack. They are kept apart by their registration entries, +whose selectors are disjoint, and by the registry's own access control, which grants the +push identity write and the pull identity read only. + +### How the test deviates from the diagrams + +The test runs a single VM behind several virtual Kubernetes nodes, so a few things differ +from what you would deploy. Worth knowing if you are using this as a reference: + +* One `spire-ha-agent` is shared by every virtual node, with a `spiffe-socat-unix` bridge + per node in front of it. On a real host kubelet talks to its local ha-agent directly. + Each bridge is mounted into its node at `/var/run/spire/agent/sockets/main/public`, which + is where a package installed ha-agent listens, so kubelet's own configuration is not a + deviation: what you see here is what you would deploy. +* The registration entries do deviate. Because the caller the ha-agent attests by pid is the + bridge, they select on the socat unit rather than on kubelet's own unit. On a real host + that selector is the only line that changes. +* One ha-agent behind every node means the test covers a root server failing, which is the + part that matters here, but not a single node's ha-agent failing. +* The credential provider binary and its configuration are staged into every kind cluster + by `.github/scripts/install-image-cred-provider.sh` before the cluster is created. + Kubelet refuses to start when a provider named in its configuration is missing, so this + cannot be deferred to the test itself. diff --git a/examples/bottom-turtle-ha/example-manifests/node1-spire-ha-agent.yaml b/examples/bottom-turtle-ha/example-manifests/node1-spire-ha-agent.yaml new file mode 100644 index 0000000..670abd3 --- /dev/null +++ b/examples/bottom-turtle-ha/example-manifests/node1-spire-ha-agent.yaml @@ -0,0 +1,11 @@ +apiVersion: spire.spiffe.io/v1alpha1 +kind: ClusterStaticEntry +metadata: + name: node1-spire-ha-agent +spec: + parentID: spiffe://${SPIFFE_TRUST_DOMAIN}/agent/node1 + spiffeID: spiffe://${SPIFFE_TRUST_DOMAIN}/spire-ha-agent + selectors: + - systemd:id:spire-ha-agent@main.service + federatesWith: + - spire-ha diff --git a/examples/bottom-turtle-ha/example-manifests/node2-image-pull.yaml b/examples/bottom-turtle-ha/example-manifests/node2-image-pull.yaml new file mode 100644 index 0000000..7dcb0da --- /dev/null +++ b/examples/bottom-turtle-ha/example-manifests/node2-image-pull.yaml @@ -0,0 +1,10 @@ +apiVersion: spire.spiffe.io/v1alpha1 +kind: ClusterStaticEntry +metadata: + name: node2-image-pull +spec: + parentID: spiffe://${SPIFFE_TRUST_DOMAIN}/agent/node1 + spiffeID: spiffe://${SPIFFE_TRUST_DOMAIN}/kubelet + hint: image-pull + selectors: + - systemd:id:spiffe-socat-unix@k8s-kubelet-2.service diff --git a/examples/bottom-turtle-ha/example-manifests/node2-kubelet.yaml b/examples/bottom-turtle-ha/example-manifests/node2-kubelet.yaml new file mode 100644 index 0000000..1960025 --- /dev/null +++ b/examples/bottom-turtle-ha/example-manifests/node2-kubelet.yaml @@ -0,0 +1,10 @@ +apiVersion: spire.spiffe.io/v1alpha1 +kind: ClusterStaticEntry +metadata: + name: node2-kubelet +spec: + parentID: spiffe://${SPIFFE_TRUST_DOMAIN}/agent/node1 + spiffeID: spiffe://${SPIFFE_TRUST_DOMAIN}/kubelet/node2.${SPIFFE_TRUST_DOMAIN} + hint: kubelet + selectors: + - systemd:id:spiffe-socat-unix@k8s-kubelet-2.service diff --git a/examples/bottom-turtle-ha/example-manifests/node3-image-pull.yaml b/examples/bottom-turtle-ha/example-manifests/node3-image-pull.yaml new file mode 100644 index 0000000..7538762 --- /dev/null +++ b/examples/bottom-turtle-ha/example-manifests/node3-image-pull.yaml @@ -0,0 +1,10 @@ +apiVersion: spire.spiffe.io/v1alpha1 +kind: ClusterStaticEntry +metadata: + name: node3-image-pull +spec: + parentID: spiffe://${SPIFFE_TRUST_DOMAIN}/agent/node1 + spiffeID: spiffe://${SPIFFE_TRUST_DOMAIN}/kubelet + hint: image-pull + selectors: + - systemd:id:spiffe-socat-unix@k8s-kubelet-3.service diff --git a/examples/bottom-turtle-ha/example-manifests/node3-kubelet.yaml b/examples/bottom-turtle-ha/example-manifests/node3-kubelet.yaml new file mode 100644 index 0000000..5a345e4 --- /dev/null +++ b/examples/bottom-turtle-ha/example-manifests/node3-kubelet.yaml @@ -0,0 +1,10 @@ +apiVersion: spire.spiffe.io/v1alpha1 +kind: ClusterStaticEntry +metadata: + name: node3-kubelet +spec: + parentID: spiffe://${SPIFFE_TRUST_DOMAIN}/agent/node1 + spiffeID: spiffe://${SPIFFE_TRUST_DOMAIN}/kubelet/node3.${SPIFFE_TRUST_DOMAIN} + hint: kubelet + selectors: + - systemd:id:spiffe-socat-unix@k8s-kubelet-3.service diff --git a/examples/bottom-turtle-ha/example-manifests/node4-image-pull.yaml b/examples/bottom-turtle-ha/example-manifests/node4-image-pull.yaml new file mode 100644 index 0000000..89cbe13 --- /dev/null +++ b/examples/bottom-turtle-ha/example-manifests/node4-image-pull.yaml @@ -0,0 +1,10 @@ +apiVersion: spire.spiffe.io/v1alpha1 +kind: ClusterStaticEntry +metadata: + name: node4-image-pull +spec: + parentID: spiffe://${SPIFFE_TRUST_DOMAIN}/agent/node1 + spiffeID: spiffe://${SPIFFE_TRUST_DOMAIN}/kubelet + hint: image-pull + selectors: + - systemd:id:spiffe-socat-unix@k8s-kubelet-4.service diff --git a/examples/bottom-turtle-ha/example-manifests/node4-kubelet.yaml b/examples/bottom-turtle-ha/example-manifests/node4-kubelet.yaml new file mode 100644 index 0000000..f665027 --- /dev/null +++ b/examples/bottom-turtle-ha/example-manifests/node4-kubelet.yaml @@ -0,0 +1,10 @@ +apiVersion: spire.spiffe.io/v1alpha1 +kind: ClusterStaticEntry +metadata: + name: node4-kubelet +spec: + parentID: spiffe://${SPIFFE_TRUST_DOMAIN}/agent/node1 + spiffeID: spiffe://${SPIFFE_TRUST_DOMAIN}/kubelet/node4.${SPIFFE_TRUST_DOMAIN} + hint: kubelet + selectors: + - systemd:id:spiffe-socat-unix@k8s-kubelet-4.service diff --git a/examples/bottom-turtle-ha/image-pull-job.yaml b/examples/bottom-turtle-ha/image-pull-job.yaml new file mode 100644 index 0000000..fe6b397 --- /dev/null +++ b/examples/bottom-turtle-ha/image-pull-job.yaml @@ -0,0 +1,34 @@ +# Pulls the image zot only serves to an exchange-minted identity. +apiVersion: v1 +kind: ServiceAccount +metadata: + name: zot-pull +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: image-pull +spec: + backoffLimit: 0 + template: + metadata: + labels: + app: image-pull + spec: + serviceAccountName: zot-pull + restartPolicy: Never + containers: + - name: main + image: zot.production.other/test/busybox:latest + imagePullPolicy: Always + command: ["sh", "-c", "echo IMAGE-PULL-OK"] + # The node authorizer only lets kubelet mint a service account token for an audience + # that appears in the pod spec. This must be here to work, even if it looks unused. + volumes: + - name: spire-token-gate + projected: + sources: + - serviceAccountToken: + path: token + audience: spire-identity-exchange + expirationSeconds: 3600 diff --git a/examples/bottom-turtle-ha/image-push-denied-job.yaml b/examples/bottom-turtle-ha/image-push-denied-job.yaml new file mode 100644 index 0000000..068711b --- /dev/null +++ b/examples/bottom-turtle-ha/image-push-denied-job.yaml @@ -0,0 +1,122 @@ +# Negative control: the pull identity must not be able to write. Identical to image-push-job.yaml except the service account +apiVersion: batch/v1 +kind: Job +metadata: + name: image-push-denied +spec: + backoffLimit: 0 + template: + metadata: + labels: + app: image-push-denied + spec: + serviceAccountName: zot-pull + restartPolicy: Never + initContainers: + - name: static-busybox + # Replaced by run-tests.sh with the image from the spiffe-oidc-discovery-provider chart + image: IMAGE_BUSYBOX + command: ["sh", "-c", "cp /bin/busybox /data/busybox && chmod +x /data/busybox"] + volumeMounts: + - name: data-volume + mountPath: /data + - name: fetch-svid + # Replaced by run-tests.sh with the image from the spire-agent chart + image: IMAGE_SPIRE_AGENT + command: + - /data/busybox + - sh + - -xec + - | + SOCK=/spire-agent/spire-agent.sock + i=0 + while [ "$i" -lt 30 ]; do + if /opt/spire/bin/spire-agent api fetch x509 -socketPath "$SOCK" -write /data -timeout 5s && + /opt/spire/bin/spire-agent api fetch jwt -audience spire-identity-exchange -socketPath "$SOCK" -timeout 5s > /data/jwt.txt; then + break + fi + i=$((i+1)) + /data/busybox sleep 2 + done + if [ ! -s /data/jwt.txt ]; then + echo "no SVID for this pod after ${i} attempts" + exit 1 + fi + /data/busybox grep -A1 'token(' /data/jwt.txt | /data/busybox tail -1 | /data/busybox tr -d '[:space:]' > /data/svid.jwt + test -s /data/svid.jwt + volumeMounts: + - name: data-volume + mountPath: /data + - name: spire-api + mountPath: /spire-agent + readOnly: true + - name: exchange + # Replaced by run-tests.sh with the toolkit image from the spiffe-oidc-discovery-provider chart + image: IMAGE_TOOLKIT + command: + - sh + - -xec + - | + cat /etc/ssl/certs/ca-certificates.crt /data/bundle.0.pem > /data/ca-bundle.pem + + PSAT="$(cat /var/run/secrets/tokens/token)" + SVID="$(cat /data/svid.jwt)" + # This must still succeed. A separate registration entry matches this pod's + # credentials and mints the read only identity, so the exchange hands back a + # token; it is zot that refuses the write. + TOKEN="$(curl -k -sS --fail-with-body --max-time 60 --connect-timeout 10 -X POST \ + -H "Authorization: Bearer k8s_psat=${PSAT}:spiffe=${SVID}" \ + -H "Content-Type: application/json" \ + -d '{"audiences": ["zot"]}' \ + "https://spire-identity-exchange-rest.production.other/api/v1/svid/image_pull/jwt" \ + | sed -n 's/.*"token":"\([^"]*\)".*/\1/p')" + test -n "${TOKEN}" + + AUTH="$(printf 'zot:%s' "${TOKEN}" | base64 | tr -d '\n')" + printf '{"auths":{"zot.production.other":{"auth":"%s"}}}' "${AUTH}" > /docker-config/config.json + volumeMounts: + - name: data-volume + mountPath: /data + - name: docker-config + mountPath: /docker-config + - name: psat + mountPath: /var/run/secrets/tokens + readOnly: true + containers: + - name: push + image: gcr.io/go-containerregistry/crane:v0.21.9 + env: + - name: DOCKER_CONFIG + value: /docker-config + - name: SSL_CERT_FILE + value: /data/ca-bundle.pem + # The crane image is distroless, so borrow the static busybox copied out earlier. + command: ["/data/busybox", "sh", "-c"] + args: + - | + if /ko-app/crane copy docker.io/library/busybox:latest zot.production.other/test/denied:latest; then + echo "PUSH-SHOULD-HAVE-BEEN-DENIED" + exit 1 + fi + echo PUSH-DENIED-OK + volumeMounts: + - name: data-volume + mountPath: /data + - name: docker-config + mountPath: /docker-config + volumes: + - name: data-volume + emptyDir: {} + - name: docker-config + emptyDir: {} + - name: spire-api + csi: + driver: csi.spiffe.io + readOnly: true + - name: psat + projected: + sources: + - serviceAccountToken: + path: token + audience: spire-identity-exchange + expirationSeconds: 3600 diff --git a/examples/bottom-turtle-ha/image-push-job.yaml b/examples/bottom-turtle-ha/image-push-job.yaml new file mode 100644 index 0000000..943a340 --- /dev/null +++ b/examples/bottom-turtle-ha/image-push-job.yaml @@ -0,0 +1,121 @@ +# Pushes an image into zot using an identity minted by the spire-identity-exchange. +apiVersion: v1 +kind: ServiceAccount +metadata: + name: zot-push +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: image-push +spec: + backoffLimit: 0 + template: + metadata: + labels: + app: image-push + spec: + serviceAccountName: zot-push + restartPolicy: Never + initContainers: + - name: static-busybox + # Replaced by run-tests.sh with the image from the spiffe-oidc-discovery-provider chart + image: IMAGE_BUSYBOX + command: ["sh", "-c", "cp /bin/busybox /data/busybox && chmod +x /data/busybox"] + volumeMounts: + - name: data-volume + mountPath: /data + - name: fetch-svid + # Replaced by run-tests.sh with the image from the spire-agent chart + image: IMAGE_SPIRE_AGENT + command: + - /data/busybox + - sh + - -xec + - | + SOCK=/spire-agent/spire-agent.sock + i=0 + while [ "$i" -lt 30 ]; do + if /opt/spire/bin/spire-agent api fetch x509 -socketPath "$SOCK" -write /data -timeout 5s && + /opt/spire/bin/spire-agent api fetch jwt -audience spire-identity-exchange -socketPath "$SOCK" -timeout 5s > /data/jwt.txt; then + break + fi + i=$((i+1)) + /data/busybox sleep 2 + done + if [ ! -s /data/jwt.txt ]; then + echo "no SVID for this pod after ${i} attempts" + exit 1 + fi + /data/busybox grep -A1 'token(' /data/jwt.txt | /data/busybox tail -1 | /data/busybox tr -d '[:space:]' > /data/svid.jwt + test -s /data/svid.jwt + volumeMounts: + - name: data-volume + mountPath: /data + - name: spire-api + mountPath: /spire-agent + readOnly: true + - name: exchange + # Replaced by run-tests.sh with the toolkit image from the spiffe-oidc-discovery-provider chart + image: IMAGE_TOOLKIT + command: + - sh + - -xec + - | + # crane talks to both docker.io and zot, so it needs the public roots and the + # SPIRE bundle in one file. + cat /etc/ssl/certs/ca-certificates.crt /data/bundle.0.pem > /data/ca-bundle.pem + + PSAT="$(cat /var/run/secrets/tokens/token)" + SVID="$(cat /data/svid.jwt)" + TOKEN="$(curl -k -sS --fail-with-body --max-time 60 --connect-timeout 10 -X POST \ + -H "Authorization: Bearer k8s_psat=${PSAT}:spiffe=${SVID}" \ + -H "Content-Type: application/json" \ + -d '{"audiences": ["zot"]}' \ + "https://spire-identity-exchange-rest.production.other/api/v1/svid/image_pull/jwt" \ + | sed -n 's/.*"token":"\([^"]*\)".*/\1/p')" + test -n "${TOKEN}" + + AUTH="$(printf 'zot:%s' "${TOKEN}" | base64 | tr -d '\n')" + printf '{"auths":{"zot.production.other":{"auth":"%s"}}}' "${AUTH}" > /docker-config/config.json + volumeMounts: + - name: data-volume + mountPath: /data + - name: docker-config + mountPath: /docker-config + - name: psat + mountPath: /var/run/secrets/tokens + readOnly: true + containers: + - name: push + image: gcr.io/go-containerregistry/crane:v0.21.9 + env: + - name: DOCKER_CONFIG + value: /docker-config + - name: SSL_CERT_FILE + value: /data/ca-bundle.pem + args: + - copy + - docker.io/library/busybox:latest + - zot.production.other/test/busybox:latest + volumeMounts: + - name: data-volume + mountPath: /data + - name: docker-config + mountPath: /docker-config + volumes: + - name: data-volume + emptyDir: {} + - name: docker-config + emptyDir: {} + - name: spire-api + csi: + driver: csi.spiffe.io + readOnly: true + - name: psat + projected: + sources: + - serviceAccountToken: + path: token + audience: spire-identity-exchange + expirationSeconds: 3600 diff --git a/examples/bottom-turtle-ha/run-tests.sh b/examples/bottom-turtle-ha/run-tests.sh index e5a69e1..965838c 100755 --- a/examples/bottom-turtle-ha/run-tests.sh +++ b/examples/bottom-turtle-ha/run-tests.sh @@ -74,6 +74,10 @@ teardown() { sudo systemctl status spire-server@other || true kubectl describe job federation-test || true kubectl logs job/federation-test || true + sudo systemctl status spire-ha-agent@main || true + sudo systemctl status spiffe-socat-unix@k8s-kubelet-2 || true + sudo systemctl status spiffe-socat-unix@k8s-kubelet-3 || true + sudo systemctl status spiffe-socat-unix@k8s-kubelet-4 || true sudo spire-server entry show -instance a || true sudo spire-server entry show -instance b || true sudo systemctl status spire-controller-manager@a || true @@ -107,10 +111,20 @@ teardown() { if [[ "$1" -ne 0 ]]; then get_namespace_details spire-server spire-system kubectl describe pod -n spire-system + # Only on failure: these are verbose, and the kubelet journal in particular is what + # tells you why an image pull came back anonymous rather than using the plugin. + for JOB in image-push image-pull image-push-denied; do + dump_job "${JOB}" + done + dump_zot + dump_kubelet_all fi if [ "${CLEANUP}" -eq 1 ]; then kubectl delete job federation-test 2>/dev/null || true + kubectl delete job image-push image-pull image-push-denied 2>/dev/null || true + helm uninstall --namespace zot zot 2>/dev/null || true + kubectl delete ns zot 2>/dev/null || true helm uninstall --namespace spire-mgmt spire-b 2>/dev/null || true helm uninstall --namespace spire-mgmt spire-a 2>/dev/null || true helm uninstall --namespace spire-mgmt spire 2>/dev/null || true @@ -154,6 +168,120 @@ wait_for_trust_sync() { return 1 } +# Dump everything useful about a job in one block. The trace goes to stderr while command +# output goes to stdout, and the two are separate streams in the CI log, so anything printed +# here can interleave or drop. Suspend the trace, merge each command's streams, and bracket +# the whole thing so it stays readable. +dump_job() { + local job="$1" + set +x + echo "===== BEGIN ${job} =====" + kubectl get pods -l "job-name=${job}" -o wide 2>&1 || true + kubectl describe job "${job}" 2>&1 || true + # Pod events are where image pull and volume failures show up; the job has none of this. + kubectl describe pod -l "job-name=${job}" 2>&1 || true + # --prefix labels each line with its container. These jobs have three init containers and + # the interesting output is rarely the last one. + kubectl logs "job/${job}" --all-containers --prefix 2>&1 || true + # Everything kubelet said about this pod on the node that ran it. Filtering on the image + # misses the credential provider path, which names the pod and service account instead, + # and only the first pull attempt carries the real error; the retries are all backoff. + local pod node + for pod in $(kubectl get pods -l "job-name=${job}" -o name 2>/dev/null | cut -d/ -f2); do + node="$(kubectl get pod "${pod}" -o jsonpath='{.spec.nodeName}' 2>/dev/null)" + [ -n "${node}" ] || continue + echo "----- kubelet ${node} for pod ${pod} -----" + # Only this pod. A broader filter matches every provider line since boot and the + # window fills long before the pull happens. + docker exec -i "${node}" journalctl -u kubelet --no-pager 2>&1 \ + | grep -F "${pod}" | head -60 || true + # The plugin exec is logged against the image and plugin name, not the pod, so a pod + # filter hides exactly the line that says whether it ran and what it returned. + echo "----- kubelet ${node} credential provider decisions -----" + docker exec -i "${node}" journalctl -u kubelet --no-pager 2>&1 \ + | grep -E 'exec plugin|image credentials|k8s-image-cred|without credentials|zot\.production\.other|[Ss]ervice account' \ + | tail -40 || true + done + echo "===== END ${job} =====" + set -x +} + +# Same treatment as dump_job. zot logs at debug, and its rejection reason for a bearer +# token only appears there, so take the whole log rather than a tail. +dump_zot() { + set +x + echo "===== BEGIN zot =====" + kubectl get pods -n zot -o wide 2>&1 || true + kubectl describe pod -n zot -l app.kubernetes.io/name=zot 2>&1 || true + kubectl logs -n zot -l app.kubernetes.io/name=zot --all-containers --prefix 2>&1 || true + echo "===== END zot =====" + set -x +} + +# Whether kubelet was configured with the image credential provider at all, and whether it +# ran it. The kubeadm patch landing is the whole question when a pull comes back anonymous. +dump_kubelet() { + local node="$1" + set +x + echo "===== BEGIN kubelet ${node} =====" + docker exec -i "${node}" cat /var/lib/kubelet/kubeadm-flags.env 2>&1 || true + docker exec -i "${node}" ps ax 2>&1 | grep '[k]ubelet' || true + docker exec -i "${node}" ls -l /credential-plugins /etc/kubernetes/credential-provider-config.yaml 2>&1 || true + # The plugin reaches the workload API through this bridge. If the socket is absent the + # plugin fails immediately, kubelet falls back to anonymous, and the pull 401s. + docker exec -i "${node}" ls -l /var/run/spire/agent/sockets/main/public/ 2>&1 || true + # Only the registry we care about. A broad credential grep is pure noise at v=4, which + # logs a provider line for every image pull on the node. + docker exec -i "${node}" journalctl -u kubelet --no-pager 2>&1 \ + | grep -E 'zot\.production\.other' -A3 | tail -60 || true + echo "===== END kubelet ${node} =====" + set -x +} + +dump_kubelet_all() { + for NODE in $(kubectl get nodes -o name 2>/dev/null | cut -d/ -f2); do + dump_kubelet "${NODE}" + done +} + +# kubectl wait --for=condition=complete blocks the full timeout when a job has already +# failed, which makes every failure look like a hang and delays the dump by minutes. Poll +# both terminal conditions instead. +wait_for_job() { + local job="$1" + local timeout="${2:-120}" + local count=0 + while [ "$count" -lt "$timeout" ]; do + if kubectl get job "$job" -o jsonpath='{.status.conditions[?(@.type=="Complete")].status}' 2>/dev/null | grep -q True; then + return 0 + fi + if kubectl get job "$job" -o jsonpath='{.status.conditions[?(@.type=="Failed")].status}' 2>/dev/null | grep -q True; then + echo "job/$job failed" + dump_job "$job" + return 1 + fi + sleep 1 + ((count++)) || true + done + echo "job/$job did not finish within ${timeout}s" + dump_job "$job" + return 1 +} + +wait_for_socket() { + local socket="$1" + local timeout=30 + local count=0 + while [ "$count" -lt "$timeout" ]; do + if [ -S "$socket" ]; then + return 0 + fi + sleep 1 + ((count++)) || true + done + return 1 +} + wait_for_jwt() { local socket="$1" local timeout=30 @@ -199,11 +327,19 @@ run_federation_test_job() { # Get the package repo and install the packages sudo curl -s -o /etc/apt/sources.list.d/spire-examples.list https://raw.githubusercontent.com/spiffe/spire-examples/refs/heads/main/examples/debs/amd64/spire-examples.list sudo apt-get update -sudo apt-get install -y spire-common spire-agent spire-server spire-controller-manager spiffe-socat-unix socat spire-trust-sync spiffe-helper +sudo apt-get install -y spire-common spire-agent spire-server spire-controller-manager spiffe-socat-unix socat spire-trust-sync spiffe-helper spire-ha-agent # Set our testing trust domain sudo sed -i 's/example.org/production.other/' /etc/spiffe/default-trust-domain.env +# A trust domain has one OIDC discovery endpoint, but the packaged root server config +# advertises oidc-discovery-provider. while the charts advertise +# oidc-discovery.. The identity exchange checks the iss claim by exact string, +# so a JWT-SVID minted by a root server, which is what kubelet's credential provider +# presents, is rejected unless the two agree. Align the roots with the charts. +sudo sed -i 's|jwt_issuer = "https://oidc-discovery-provider\.|jwt_issuer = "https://oidc-discovery.|' /etc/spire/server/default.conf +grep jwt_issuer /etc/spire/server/default.conf + if [ "${BROKER}" -eq 1 ]; then # Pull the federation test job images out of the charts so they always sync up. AGENT_IMAGE=$(helm template t charts/spire -s charts/spire-agent/templates/daemonset.yaml --values "${COMMON_TEST_YOUR_VALUES}" --set spire-agent.enabled=true | yq e 'select(.kind=="DaemonSet") | .spec.template.spec.containers[] | select(.name=="spire-agent") | .image' -) @@ -317,6 +453,36 @@ wait_for_jwt /var/run/spiffe/socat/unix/k8s-spire-agent-3-b/public/api.sock wait_for_jwt /var/run/spiffe/socat/unix/k8s-spire-agent-4-a/public/api.sock wait_for_jwt /var/run/spiffe/socat/unix/k8s-spire-agent-4-b/public/api.sock +# Start the host spire-ha-agent. It merges the two root agents into one workload API, which is +# what lets host services keep their identity when a single root server goes away. The compiled +# in defaults already point at /var/run/spire/agent/sockets/{a,b}/private/admin.sock and listen +# on the main instance socket, and the packaged agent config already lists the ha-agent in its +# authorized_delegates, so no configuration is needed. +sudo systemctl start spire-ha-agent@main +# Not wait_for_healthcheck: that calls the grpc.health.v1 service, which the ha-agent does +# not serve, so it always reports "unable to determine health". Not wait_for_jwt either: +# the ha-agent attests callers by pid, and the cli invoking it is in no registered unit. +# Readiness is proven through the bridges below, where the caller does have an entry. +wait_for_socket /var/run/spire/agent/sockets/main/public/api.sock + +# Bridge the merged workload API into each virtual node for kubelet's image credential provider. +# A real deployment runs one ha-agent per host and kubelet talks to it directly. Here a single VM +# backs three virtual nodes, so we put one socat instance in front of the shared ha-agent per +# node. The ha-agent attests each caller by pid, so every bridge resolves to its own entry and +# each node still gets a distinct identity. Each bridge is mounted into its node at +# /var/run/spire/agent/sockets/main/public, where a package installed ha-agent listens, so +# kubelet's configuration inside the node is the same one a real host would use. +sudo /bin/bash -c "echo SPIFFE_INSTANCE=main > /etc/spiffe/socat/unix/k8s-kubelet-2.conf" +sudo /bin/bash -c "echo SPIFFE_INSTANCE=main > /etc/spiffe/socat/unix/k8s-kubelet-3.conf" +sudo /bin/bash -c "echo SPIFFE_INSTANCE=main > /etc/spiffe/socat/unix/k8s-kubelet-4.conf" +sudo systemctl start spiffe-socat-unix@k8s-kubelet-2 spiffe-socat-unix@k8s-kubelet-3 spiffe-socat-unix@k8s-kubelet-4 +# These front the ha-agent rather than a spire-agent, so healthcheck does not apply here +# either. Fetching an svid is the real signal: it exercises the bridge, the ha-agent and +# whichever root agent answered. +wait_for_jwt /var/run/spiffe/socat/unix/k8s-kubelet-2/public/api.sock +wait_for_jwt /var/run/spiffe/socat/unix/k8s-kubelet-3/public/api.sock +wait_for_jwt /var/run/spiffe/socat/unix/k8s-kubelet-4/public/api.sock + # Deploy an ingress controller IP=$(kubectl get nodes chart-testing-control-plane -o go-template='{{ range .status.addresses }}{{ if eq .type "InternalIP" }}{{ .address }}{{ end }}{{ end }}') helm upgrade --install ingress-nginx ingress-nginx --version "$VERSION_INGRESS_NGINX" --repo "$HELM_REPO_INGRESS_NGINX" \ @@ -332,7 +498,7 @@ common_test_url "$IP" # Get the host IP And add spire-server-[ab].${trust_domain} records to it so the spire-servers can talk back to root servers running on the host HOSTIP=$(ip addr show docker0 | grep 'inet ' | awk '{print $2}' | cut -d/ -f1) kubectl get configmap -n kube-system coredns -o yaml | grep hosts || kubectl get configmap -n kube-system coredns -o yaml | sed "/ready/a\ hosts {\n fallthrough\n }" | kubectl apply -f - -kubectl get configmap -n kube-system coredns -o yaml | grep production.other || kubectl get configmap -n kube-system coredns -o yaml | sed "/hosts/a\ $HOSTIP spire-server-a.production.other\n $IP oidc-discovery.production.other\n $HOSTIP spire-server-b.production.other\n 127.0.0.1 $FEDERATION_ENDPOINT_HOST\n" | kubectl apply -f - +kubectl get configmap -n kube-system coredns -o yaml | grep production.other || kubectl get configmap -n kube-system coredns -o yaml | sed "/hosts/a\ $HOSTIP spire-server-a.production.other\n $IP oidc-discovery.production.other\n $HOSTIP spire-server-b.production.other\n $IP zot.production.other\n $IP spire-identity-exchange-rest.production.other\n 127.0.0.1 $FEDERATION_ENDPOINT_HOST\n" | kubectl apply -f - kubectl rollout restart -n kube-system deployment/coredns kubectl rollout status -n kube-system -w --timeout=1m deploy/coredns @@ -343,6 +509,10 @@ helm upgrade --install --create-namespace --namespace spire-mgmt --values "${COM --set "global.spire.namespaces.create=true" \ --set "global.spire.ingressControllerType=ingress-nginx" \ --set "spiffe-oidc-discovery-provider.ingress.enabled=true" \ + --set "spireIdentityExchange.tls.rest.enabled=true" \ + --set "spireIdentityExchange.tls.rest.ingress.enabled=true" \ + --set "spireIdentityExchange.spiffe.rest.enabled=true" \ + --set "spireIdentityExchange.spiffe.rest.ingress.enabled=true" \ "${BROKER_MODE_ARGS[@]}" # Create spire-identity-exchange cert for testing. @@ -422,6 +592,80 @@ TOKEN=$(kubectl logs job/test) curl --fail-with-body -H "Authorization: Bearer ${TOKEN}" -X POST --resolve "spire-identity-exchange-a-rest.production.other:443:$IP" "https://spire-identity-exchange-a-rest.production.other/api/v1/svid/k8s_psat/x509" -k -sS -q curl --fail-with-body -H "Authorization: Bearer ${TOKEN}" -X POST --resolve "spire-identity-exchange-b-rest.production.other:443:$IP" "https://spire-identity-exchange-b-rest.production.other/api/v1/svid/k8s_psat/x509" -k -sS -q +# Registry image pull. zot serves a SPIRE issued certificate, an in cluster job pushes an +# image with an identity minted by the exchange, and kubelet pulls it back through the +# image credential provider staged on every node by .github/scripts. + +# Nodes are not cluster DNS clients, so coredns does nothing for containerd. Give each +# node the name directly, and a hosts.toml so it trusts the registry's SPIRE certificate. +# The bundle has to carry both roots: after a failover the certificate is issued by the +# other side's chain. +sudo spire-server bundle show -socketPath /run/spire/server/sockets/a/private/api.sock | sudo tee /tmp/zot-ca.pem > /dev/null +sudo spire-server bundle show -socketPath /run/spire/server/sockets/b/private/api.sock | sudo tee -a /tmp/zot-ca.pem > /dev/null +for NODE in $(kubectl get nodes -o name | cut -d/ -f2); do + # The credential provider runs on the node, not in a pod, so it resolves the registry + # and the exchange here rather than through coredns. + docker exec -i "${NODE}" /bin/bash -c "grep -q zot.production.other /etc/hosts || echo '$IP zot.production.other spire-identity-exchange-rest-spiffe.production.other' >> /etc/hosts" + docker exec -i "${NODE}" /bin/bash -c "mkdir -p /etc/containerd/certs.d/zot.production.other" + docker exec -i "${NODE}" /bin/bash -c "cat > /etc/containerd/certs.d/zot.production.other/zot-ca.pem" < /tmp/zot-ca.pem + docker exec -i "${NODE}" /bin/bash -c "cat > /etc/containerd/certs.d/zot.production.other/hosts.toml" < "${rendered}" + # Fail loudly rather than applying a half substituted manifest. + if grep -q 'IMAGE_BUSYBOX\|IMAGE_SPIRE_AGENT\|IMAGE_TOOLKIT' "${rendered}"; then + echo "unsubstituted image placeholder left in ${rendered}" + exit 1 + fi + kubectl apply -f "${rendered}" +} + +# Push with the writer identity. +apply_registry_job "${SCRIPTPATH}/image-push-job.yaml" +wait_for_job image-push + +# Pull it back. Nothing in the job fetches a credential; kubelet runs the plugin, which is +# the whole point of the test. +kubectl apply -f "${SCRIPTPATH}/image-pull-job.yaml" +wait_for_job image-pull +# Completing at all is the assertion: zot grants no anonymous access, so the image only +# comes down if kubelet ran the plugin and the exchange minted a token zot accepted. The +# kubelet log line naming the plugin needs -v=4, which is not worth turning on for every +# example, so teardown prints it as a diagnostic rather than asserting on it. +kubectl logs job/image-pull | grep IMAGE-PULL-OK + +# The pull identity is read only. This job completes only when zot refuses the write. +apply_registry_job "${SCRIPTPATH}/image-push-denied-job.yaml" +wait_for_job image-push-denied +kubectl logs job/image-push-denied | grep PUSH-DENIED-OK + if [ "${BROKER}" -eq 1 ]; then # Verify a workload on the ha-agent socket receives the other.invalid federated trust bundles, # x509 and jwt, merged from both sides. @@ -441,3 +685,12 @@ if [ "${BROKER}" -eq 1 ]; then run_federation_test_job fi +# The image pull path has to survive losing a side too. Everything it depends on is HA: +# the node's identity comes from the host spire-ha-agent, and the credential provider +# talks to the combined exchange endpoint rather than either side directly. Delete the +# job first so this is a genuine second pull rather than a cached result. +kubectl delete job image-pull +kubectl apply -f "${SCRIPTPATH}/image-pull-job.yaml" +wait_for_job image-pull +kubectl logs job/image-pull | grep IMAGE-PULL-OK + diff --git a/examples/bottom-turtle-ha/spire-identity-exchange-values.yaml b/examples/bottom-turtle-ha/spire-identity-exchange-values.yaml index d6a93f9..5c7e0f2 100644 --- a/examples/bottom-turtle-ha/spire-identity-exchange-values.yaml +++ b/examples/bottom-turtle-ha/spire-identity-exchange-values.yaml @@ -1,6 +1,14 @@ internal-spire-server-bottom-turtle-ha-a: &server controllerManager: identities: + clusterSPIFFEIDs: + zot: + spiffeIDTemplate: spiffe://{{ .TrustDomain }}/zot + podSelector: + matchLabels: + app.kubernetes.io/name: zot + dnsNameTemplates: + - zot.{{ .TrustDomain }} clusterStaticEntries: test: parentID: spiffe://production.other/spire-identity-exchange @@ -8,6 +16,33 @@ internal-spire-server-bottom-turtle-ha-a: &server selectors: - k8s_psat:namespace:default - k8s_psat:service_account_name:default + + image-push: + parentID: spiffe://production.other/spire-identity-exchange + spiffeID: spiffe://production.other/image-push + selectors: + - k8s_psat:namespace:default + - k8s_psat:service_account_name:zot-push + - spiffe:source_path:/ns/default/sa/zot-push + - spire_identity_exchange:stack:name:image_pull + + image-pull: + parentID: spiffe://production.other/spire-identity-exchange + spiffeID: spiffe://production.other/image-pull + selectors: + - k8s_psat:namespace:default + - k8s_psat:service_account_name:zot-pull + - spiffe:source_path:/kubelet + - spire_identity_exchange:stack:name:image_pull + + image-pull-from-pod: + parentID: spiffe://production.other/spire-identity-exchange + spiffeID: spiffe://production.other/image-pull + selectors: + - k8s_psat:namespace:default + - k8s_psat:service_account_name:zot-pull + - spiffe:source_path:/ns/default/sa/zot-pull + - spire_identity_exchange:stack:name:image_pull spireIdentityExchange: enabled: true @@ -30,6 +65,14 @@ spire-identity-exchange-bottom-turtle-ha-a: &six config: allowedServiceAccounts: - default/default + - default/zot-push + - default/zot-pull + spiffe: + config: + pathPatterns: + - "^/kubelet$" + - "^/ns/default/sa/zot-push$" + - "^/ns/default/sa/zot-pull$" #Set the same settings on the B side spire-identity-exchange-bottom-turtle-ha-b: *six diff --git a/examples/bottom-turtle-ha/zot-values.yaml b/examples/bottom-turtle-ha/zot-values.yaml new file mode 100644 index 0000000..953b8f4 --- /dev/null +++ b/examples/bottom-turtle-ha/zot-values.yaml @@ -0,0 +1,154 @@ +# zot registry with a SPIRE issued serving certificate. + +image: + tag: v2.1.20 + +initContainers: + - name: spiffe-helper-init + image: ghcr.io/spiffe/spiffe-helper:0.11.0 + args: ["-config", "/etc/spiffe-helper.conf", "-daemon-mode=false"] + securityContext: + allowPrivilegeEscalation: false + runAsNonRoot: true + readOnlyRootFilesystem: true + capabilities: + drop: [ALL] + seccompProfile: + type: RuntimeDefault + volumeMounts: + - name: spiffe-workload-api + mountPath: /spiffe-workload-api + readOnly: true + - name: zot-config + mountPath: /etc/spiffe-helper.conf + subPath: spiffe-helper.conf + readOnly: true + - name: spire-svid + mountPath: /svid + +extraContainers: + - name: spiffe-helper + image: ghcr.io/spiffe/spiffe-helper:0.11.0 + args: ["-config", "/etc/spiffe-helper.conf"] + securityContext: + allowPrivilegeEscalation: false + runAsNonRoot: true + readOnlyRootFilesystem: true + capabilities: + drop: [ALL] + seccompProfile: + type: RuntimeDefault + volumeMounts: + - name: spiffe-workload-api + mountPath: /spiffe-workload-api + readOnly: true + - name: zot-config + mountPath: /etc/spiffe-helper.conf + subPath: spiffe-helper.conf + readOnly: true + - name: spire-svid + mountPath: /svid + +extraVolumes: + - name: spire-svid + emptyDir: {} + - name: spiffe-workload-api + csi: + driver: csi.spiffe.io + readOnly: true + +extraVolumeMounts: + - name: spire-svid + mountPath: /svid + readOnly: true + +podSecurityContext: + fsGroupChangePolicy: OnRootMismatch + runAsUser: 1000 + runAsGroup: 1000 + fsGroup: 1000 + +securityContext: + allowPrivilegeEscalation: false + runAsNonRoot: true + capabilities: + drop: [ALL] + seccompProfile: + type: RuntimeDefault + +resources: {} + +httpGet: + scheme: HTTPS + port: 5000 + +startupProbe: + initialDelaySeconds: 5 + periodSeconds: 10 + failureThreshold: 30 + +mountConfig: true +configFiles: + spiffe-helper.conf: |- + agent_address = "/spiffe-workload-api/spire-agent.sock" + cert_dir = "/svid" + svid_file_name = "tls.crt" + svid_key_file_name = "tls.key" + svid_bundle_file_name = "ca.pem" + config.json: |- + { + "storage": { "rootDirectory": "/var/lib/registry" }, + "http": { + "address": "0.0.0.0", + "port": "5000", + "compat": ["docker2s2"], + "tls": { "cert": "/svid/tls.crt", "key": "/svid/tls.key" }, + "realm": "zot", + "auth": { + "bearer": { + "realm": "https://zot.production.other/zot/auth/token", + "service": "https://zot.production.other", + "oidc": [ + { + "issuer": "https://oidc-discovery.production.other", + "audiences": ["zot"], + "certificateAuthorityFile": "/svid/ca.pem", + "claimMapping": { "username": "claims.sub" } + } + ] + } + }, + "accessControl": { + "repositories": { + "**": { + "policies": [ + { + "users": ["spiffe://production.other/image-push"], + "actions": ["read", "create", "update", "delete"] + }, + { + "users": ["spiffe://production.other/image-pull"], + "actions": ["read"] + } + ] + } + } + } + }, + "log": { "level": "debug" } + } + +service: + type: ClusterIP + +ingress: + enabled: true + className: nginx + pathtype: Prefix + annotations: + nginx.ingress.kubernetes.io/ssl-passthrough: "true" + nginx.ingress.kubernetes.io/backend-protocol: "HTTPS" + hosts: + - host: zot.production.other + paths: + - path: /