Bottom turtle ha registry tests (#929)

* Bottom turtle ha registry tests

Signed-off-by: Kevin Fox <[email protected]>

* Fix test because of spire-ha-agent

Signed-off-by: Kevin Fox <[email protected]>

* Fix config options

Signed-off-by: Kevin Fox <[email protected]>

* Fix yq issue and log better

Signed-off-by: Kevin Fox <[email protected]>

* Fix some things

Signed-off-by: Kevin Fox <[email protected]>

* Change timeout and fix image

Signed-off-by: Kevin Fox <[email protected]>

* Bump up zot version.

Signed-off-by: Kevin Fox <[email protected]>

* More debugging stuff

Signed-off-by: Kevin Fox <[email protected]>

* More debugging stuff

Signed-off-by: Kevin Fox <[email protected]>

* More fixes

Signed-off-by: Kevin Fox <[email protected]>

* More logging

Signed-off-by: Kevin Fox <[email protected]>

* More logging

Signed-off-by: Kevin Fox <[email protected]>

* More logging

Signed-off-by: Kevin Fox <[email protected]>

* More logging

Signed-off-by: Kevin Fox <[email protected]>

* More logging

Signed-off-by: Kevin Fox <[email protected]>

* Fix test

Signed-off-by: Kevin Fox <[email protected]>

* Log less

Signed-off-by: Kevin Fox <[email protected]>

* Cleanup

Signed-off-by: Kevin Fox <[email protected]>

* Make test less different then real deployment

Signed-off-by: Kevin Fox <[email protected]>

---------

Signed-off-by: Kevin Fox <[email protected]>
Signed-off-by: kfox1111 <[email protected]>
This commit is contained in:
kfox1111
2026-09-04 12:50:34 -07:00
committed by GitHub
parent bb4c73febf
commit 2a8c1b63c4
23 changed files with 1082 additions and 28 deletions
+255 -2
View File
@@ -74,6 +74,10 @@ teardown() {
sudo systemctl status spire-server@other || true
kubectl describe job federation-test || true
kubectl logs job/federation-test || true
sudo systemctl status spire-ha-agent@main || true
sudo systemctl status spiffe-socat-unix@k8s-kubelet-2 || true
sudo systemctl status spiffe-socat-unix@k8s-kubelet-3 || true
sudo systemctl status spiffe-socat-unix@k8s-kubelet-4 || true
sudo spire-server entry show -instance a || true
sudo spire-server entry show -instance b || true
sudo systemctl status spire-controller-manager@a || true
@@ -107,10 +111,20 @@ teardown() {
if [[ "$1" -ne 0 ]]; then
get_namespace_details spire-server spire-system
kubectl describe pod -n spire-system
# Only on failure: these are verbose, and the kubelet journal in particular is what
# tells you why an image pull came back anonymous rather than using the plugin.
for JOB in image-push image-pull image-push-denied; do
dump_job "${JOB}"
done
dump_zot
dump_kubelet_all
fi
if [ "${CLEANUP}" -eq 1 ]; then
kubectl delete job federation-test 2>/dev/null || true
kubectl delete job image-push image-pull image-push-denied 2>/dev/null || true
helm uninstall --namespace zot zot 2>/dev/null || true
kubectl delete ns zot 2>/dev/null || true
helm uninstall --namespace spire-mgmt spire-b 2>/dev/null || true
helm uninstall --namespace spire-mgmt spire-a 2>/dev/null || true
helm uninstall --namespace spire-mgmt spire 2>/dev/null || true
@@ -154,6 +168,120 @@ wait_for_trust_sync() {
return 1
}
# Dump everything useful about a job in one block. The trace goes to stderr while command
# output goes to stdout, and the two are separate streams in the CI log, so anything printed
# here can interleave or drop. Suspend the trace, merge each command's streams, and bracket
# the whole thing so it stays readable.
dump_job() {
local job="$1"
set +x
echo "===== BEGIN ${job} ====="
kubectl get pods -l "job-name=${job}" -o wide 2>&1 || true
kubectl describe job "${job}" 2>&1 || true
# Pod events are where image pull and volume failures show up; the job has none of this.
kubectl describe pod -l "job-name=${job}" 2>&1 || true
# --prefix labels each line with its container. These jobs have three init containers and
# the interesting output is rarely the last one.
kubectl logs "job/${job}" --all-containers --prefix 2>&1 || true
# Everything kubelet said about this pod on the node that ran it. Filtering on the image
# misses the credential provider path, which names the pod and service account instead,
# and only the first pull attempt carries the real error; the retries are all backoff.
local pod node
for pod in $(kubectl get pods -l "job-name=${job}" -o name 2>/dev/null | cut -d/ -f2); do
node="$(kubectl get pod "${pod}" -o jsonpath='{.spec.nodeName}' 2>/dev/null)"
[ -n "${node}" ] || continue
echo "----- kubelet ${node} for pod ${pod} -----"
# Only this pod. A broader filter matches every provider line since boot and the
# window fills long before the pull happens.
docker exec -i "${node}" journalctl -u kubelet --no-pager 2>&1 \
| grep -F "${pod}" | head -60 || true
# The plugin exec is logged against the image and plugin name, not the pod, so a pod
# filter hides exactly the line that says whether it ran and what it returned.
echo "----- kubelet ${node} credential provider decisions -----"
docker exec -i "${node}" journalctl -u kubelet --no-pager 2>&1 \
| grep -E 'exec plugin|image credentials|k8s-image-cred|without credentials|zot\.production\.other|[Ss]ervice account' \
| tail -40 || true
done
echo "===== END ${job} ====="
set -x
}
# Same treatment as dump_job. zot logs at debug, and its rejection reason for a bearer
# token only appears there, so take the whole log rather than a tail.
dump_zot() {
set +x
echo "===== BEGIN zot ====="
kubectl get pods -n zot -o wide 2>&1 || true
kubectl describe pod -n zot -l app.kubernetes.io/name=zot 2>&1 || true
kubectl logs -n zot -l app.kubernetes.io/name=zot --all-containers --prefix 2>&1 || true
echo "===== END zot ====="
set -x
}
# Whether kubelet was configured with the image credential provider at all, and whether it
# ran it. The kubeadm patch landing is the whole question when a pull comes back anonymous.
dump_kubelet() {
local node="$1"
set +x
echo "===== BEGIN kubelet ${node} ====="
docker exec -i "${node}" cat /var/lib/kubelet/kubeadm-flags.env 2>&1 || true
docker exec -i "${node}" ps ax 2>&1 | grep '[k]ubelet' || true
docker exec -i "${node}" ls -l /credential-plugins /etc/kubernetes/credential-provider-config.yaml 2>&1 || true
# The plugin reaches the workload API through this bridge. If the socket is absent the
# plugin fails immediately, kubelet falls back to anonymous, and the pull 401s.
docker exec -i "${node}" ls -l /var/run/spire/agent/sockets/main/public/ 2>&1 || true
# Only the registry we care about. A broad credential grep is pure noise at v=4, which
# logs a provider line for every image pull on the node.
docker exec -i "${node}" journalctl -u kubelet --no-pager 2>&1 \
| grep -E 'zot\.production\.other' -A3 | tail -60 || true
echo "===== END kubelet ${node} ====="
set -x
}
dump_kubelet_all() {
for NODE in $(kubectl get nodes -o name 2>/dev/null | cut -d/ -f2); do
dump_kubelet "${NODE}"
done
}
# kubectl wait --for=condition=complete blocks the full timeout when a job has already
# failed, which makes every failure look like a hang and delays the dump by minutes. Poll
# both terminal conditions instead.
wait_for_job() {
local job="$1"
local timeout="${2:-120}"
local count=0
while [ "$count" -lt "$timeout" ]; do
if kubectl get job "$job" -o jsonpath='{.status.conditions[?(@.type=="Complete")].status}' 2>/dev/null | grep -q True; then
return 0
fi
if kubectl get job "$job" -o jsonpath='{.status.conditions[?(@.type=="Failed")].status}' 2>/dev/null | grep -q True; then
echo "job/$job failed"
dump_job "$job"
return 1
fi
sleep 1
((count++)) || true
done
echo "job/$job did not finish within ${timeout}s"
dump_job "$job"
return 1
}
wait_for_socket() {
local socket="$1"
local timeout=30
local count=0
while [ "$count" -lt "$timeout" ]; do
if [ -S "$socket" ]; then
return 0
fi
sleep 1
((count++)) || true
done
return 1
}
wait_for_jwt() {
local socket="$1"
local timeout=30
@@ -199,11 +327,19 @@ run_federation_test_job() {
# Get the package repo and install the packages
sudo curl -s -o /etc/apt/sources.list.d/spire-examples.list https://raw.githubusercontent.com/spiffe/spire-examples/refs/heads/main/examples/debs/amd64/spire-examples.list
sudo apt-get update
sudo apt-get install -y spire-common spire-agent spire-server spire-controller-manager spiffe-socat-unix socat spire-trust-sync spiffe-helper
sudo apt-get install -y spire-common spire-agent spire-server spire-controller-manager spiffe-socat-unix socat spire-trust-sync spiffe-helper spire-ha-agent
# Set our testing trust domain
sudo sed -i 's/example.org/production.other/' /etc/spiffe/default-trust-domain.env
# A trust domain has one OIDC discovery endpoint, but the packaged root server config
# advertises oidc-discovery-provider.<trust domain> while the charts advertise
# oidc-discovery.<trust domain>. The identity exchange checks the iss claim by exact string,
# so a JWT-SVID minted by a root server, which is what kubelet's credential provider
# presents, is rejected unless the two agree. Align the roots with the charts.
sudo sed -i 's|jwt_issuer = "https://oidc-discovery-provider\.|jwt_issuer = "https://oidc-discovery.|' /etc/spire/server/default.conf
grep jwt_issuer /etc/spire/server/default.conf
if [ "${BROKER}" -eq 1 ]; then
# Pull the federation test job images out of the charts so they always sync up.
AGENT_IMAGE=$(helm template t charts/spire -s charts/spire-agent/templates/daemonset.yaml --values "${COMMON_TEST_YOUR_VALUES}" --set spire-agent.enabled=true | yq e 'select(.kind=="DaemonSet") | .spec.template.spec.containers[] | select(.name=="spire-agent") | .image' -)
@@ -317,6 +453,36 @@ wait_for_jwt /var/run/spiffe/socat/unix/k8s-spire-agent-3-b/public/api.sock
wait_for_jwt /var/run/spiffe/socat/unix/k8s-spire-agent-4-a/public/api.sock
wait_for_jwt /var/run/spiffe/socat/unix/k8s-spire-agent-4-b/public/api.sock
# Start the host spire-ha-agent. It merges the two root agents into one workload API, which is
# what lets host services keep their identity when a single root server goes away. The compiled
# in defaults already point at /var/run/spire/agent/sockets/{a,b}/private/admin.sock and listen
# on the main instance socket, and the packaged agent config already lists the ha-agent in its
# authorized_delegates, so no configuration is needed.
sudo systemctl start spire-ha-agent@main
# Not wait_for_healthcheck: that calls the grpc.health.v1 service, which the ha-agent does
# not serve, so it always reports "unable to determine health". Not wait_for_jwt either:
# the ha-agent attests callers by pid, and the cli invoking it is in no registered unit.
# Readiness is proven through the bridges below, where the caller does have an entry.
wait_for_socket /var/run/spire/agent/sockets/main/public/api.sock
# Bridge the merged workload API into each virtual node for kubelet's image credential provider.
# A real deployment runs one ha-agent per host and kubelet talks to it directly. Here a single VM
# backs three virtual nodes, so we put one socat instance in front of the shared ha-agent per
# node. The ha-agent attests each caller by pid, so every bridge resolves to its own entry and
# each node still gets a distinct identity. Each bridge is mounted into its node at
# /var/run/spire/agent/sockets/main/public, where a package installed ha-agent listens, so
# kubelet's configuration inside the node is the same one a real host would use.
sudo /bin/bash -c "echo SPIFFE_INSTANCE=main > /etc/spiffe/socat/unix/k8s-kubelet-2.conf"
sudo /bin/bash -c "echo SPIFFE_INSTANCE=main > /etc/spiffe/socat/unix/k8s-kubelet-3.conf"
sudo /bin/bash -c "echo SPIFFE_INSTANCE=main > /etc/spiffe/socat/unix/k8s-kubelet-4.conf"
sudo systemctl start spiffe-socat-unix@k8s-kubelet-2 spiffe-socat-unix@k8s-kubelet-3 spiffe-socat-unix@k8s-kubelet-4
# These front the ha-agent rather than a spire-agent, so healthcheck does not apply here
# either. Fetching an svid is the real signal: it exercises the bridge, the ha-agent and
# whichever root agent answered.
wait_for_jwt /var/run/spiffe/socat/unix/k8s-kubelet-2/public/api.sock
wait_for_jwt /var/run/spiffe/socat/unix/k8s-kubelet-3/public/api.sock
wait_for_jwt /var/run/spiffe/socat/unix/k8s-kubelet-4/public/api.sock
# Deploy an ingress controller
IP=$(kubectl get nodes chart-testing-control-plane -o go-template='{{ range .status.addresses }}{{ if eq .type "InternalIP" }}{{ .address }}{{ end }}{{ end }}')
helm upgrade --install ingress-nginx ingress-nginx --version "$VERSION_INGRESS_NGINX" --repo "$HELM_REPO_INGRESS_NGINX" \
@@ -332,7 +498,7 @@ common_test_url "$IP"
# Get the host IP And add spire-server-[ab].${trust_domain} records to it so the spire-servers can talk back to root servers running on the host
HOSTIP=$(ip addr show docker0 | grep 'inet ' | awk '{print $2}' | cut -d/ -f1)
kubectl get configmap -n kube-system coredns -o yaml | grep hosts || kubectl get configmap -n kube-system coredns -o yaml | sed "/ready/a\ hosts {\n fallthrough\n }" | kubectl apply -f -
kubectl get configmap -n kube-system coredns -o yaml | grep production.other || kubectl get configmap -n kube-system coredns -o yaml | sed "/hosts/a\ $HOSTIP spire-server-a.production.other\n $IP oidc-discovery.production.other\n $HOSTIP spire-server-b.production.other\n 127.0.0.1 $FEDERATION_ENDPOINT_HOST\n" | kubectl apply -f -
kubectl get configmap -n kube-system coredns -o yaml | grep production.other || kubectl get configmap -n kube-system coredns -o yaml | sed "/hosts/a\ $HOSTIP spire-server-a.production.other\n $IP oidc-discovery.production.other\n $HOSTIP spire-server-b.production.other\n $IP zot.production.other\n $IP spire-identity-exchange-rest.production.other\n 127.0.0.1 $FEDERATION_ENDPOINT_HOST\n" | kubectl apply -f -
kubectl rollout restart -n kube-system deployment/coredns
kubectl rollout status -n kube-system -w --timeout=1m deploy/coredns
@@ -343,6 +509,10 @@ helm upgrade --install --create-namespace --namespace spire-mgmt --values "${COM
--set "global.spire.namespaces.create=true" \
--set "global.spire.ingressControllerType=ingress-nginx" \
--set "spiffe-oidc-discovery-provider.ingress.enabled=true" \
--set "spireIdentityExchange.tls.rest.enabled=true" \
--set "spireIdentityExchange.tls.rest.ingress.enabled=true" \
--set "spireIdentityExchange.spiffe.rest.enabled=true" \
--set "spireIdentityExchange.spiffe.rest.ingress.enabled=true" \
"${BROKER_MODE_ARGS[@]}"
# Create spire-identity-exchange cert for testing.
@@ -422,6 +592,80 @@ TOKEN=$(kubectl logs job/test)
curl --fail-with-body -H "Authorization: Bearer ${TOKEN}" -X POST --resolve "spire-identity-exchange-a-rest.production.other:443:$IP" "https://spire-identity-exchange-a-rest.production.other/api/v1/svid/k8s_psat/x509" -k -sS -q
curl --fail-with-body -H "Authorization: Bearer ${TOKEN}" -X POST --resolve "spire-identity-exchange-b-rest.production.other:443:$IP" "https://spire-identity-exchange-b-rest.production.other/api/v1/svid/k8s_psat/x509" -k -sS -q
# Registry image pull. zot serves a SPIRE issued certificate, an in cluster job pushes an
# image with an identity minted by the exchange, and kubelet pulls it back through the
# image credential provider staged on every node by .github/scripts.
# Nodes are not cluster DNS clients, so coredns does nothing for containerd. Give each
# node the name directly, and a hosts.toml so it trusts the registry's SPIRE certificate.
# The bundle has to carry both roots: after a failover the certificate is issued by the
# other side's chain.
sudo spire-server bundle show -socketPath /run/spire/server/sockets/a/private/api.sock | sudo tee /tmp/zot-ca.pem > /dev/null
sudo spire-server bundle show -socketPath /run/spire/server/sockets/b/private/api.sock | sudo tee -a /tmp/zot-ca.pem > /dev/null
for NODE in $(kubectl get nodes -o name | cut -d/ -f2); do
# The credential provider runs on the node, not in a pod, so it resolves the registry
# and the exchange here rather than through coredns.
docker exec -i "${NODE}" /bin/bash -c "grep -q zot.production.other /etc/hosts || echo '$IP zot.production.other spire-identity-exchange-rest-spiffe.production.other' >> /etc/hosts"
docker exec -i "${NODE}" /bin/bash -c "mkdir -p /etc/containerd/certs.d/zot.production.other"
docker exec -i "${NODE}" /bin/bash -c "cat > /etc/containerd/certs.d/zot.production.other/zot-ca.pem" < /tmp/zot-ca.pem
docker exec -i "${NODE}" /bin/bash -c "cat > /etc/containerd/certs.d/zot.production.other/hosts.toml" <<EOF
server = "https://zot.production.other"
[host."https://zot.production.other"]
capabilities = ["pull", "resolve"]
ca = "/etc/containerd/certs.d/zot.production.other/zot-ca.pem"
EOF
done
helm upgrade --install zot zot --version "$VERSION_ZOT" --repo "$HELM_REPO_ZOT" \
--namespace zot --create-namespace \
--values "${SCRIPTPATH}/zot-values.yaml" \
--wait --timeout 5m
# Pull the job images out of the charts so they always sync up.
BUSYBOX_IMAGE=$(helm template t charts/spire -s charts/spiffe-oidc-discovery-provider/templates/tests/test-keys.yaml --values "${COMMON_TEST_YOUR_VALUES}" --set spiffe-oidc-discovery-provider.enabled=true | yq e 'select(.kind=="Pod") | .spec.initContainers[] | select(.name=="static-busybox") | .image' -)
AGENT_IMAGE=$(helm template t charts/spire -s charts/spire-agent/templates/daemonset.yaml --values "${COMMON_TEST_YOUR_VALUES}" --set spire-agent.enabled=true | yq e 'select(.kind=="DaemonSet") | .spec.template.spec.containers[] | select(.name=="spire-agent") | .image' -)
TOOLKIT_IMAGE=$(helm template t charts/spire -s charts/spiffe-oidc-discovery-provider/templates/tests/test-keys.yaml --values "${COMMON_TEST_YOUR_VALUES}" --set spiffe-oidc-discovery-provider.enabled=true | yq e 'select(.kind=="Pod") | .spec.containers[] | select(.name=="verify-keys") | .image' -)
echo "image pull job images: ${BUSYBOX_IMAGE} ${AGENT_IMAGE} ${TOOLKIT_IMAGE}"
# Substitute rather than rewrite. yq edits the document structure, which is version
# dependent and on a multi document file can graft fields onto the wrong document; sed
# cannot restructure anything. Writing to a file first also keeps the applied manifest
# inspectable and avoids the pipeline trap, since this script sets -e but not -o pipefail.
apply_registry_job() {
local rendered
rendered="/tmp/$(basename "$1")"
sed -e "s|IMAGE_BUSYBOX|${BUSYBOX_IMAGE}|g" \
-e "s|IMAGE_SPIRE_AGENT|${AGENT_IMAGE}|g" \
-e "s|IMAGE_TOOLKIT|${TOOLKIT_IMAGE}|g" \
"$1" > "${rendered}"
# Fail loudly rather than applying a half substituted manifest.
if grep -q 'IMAGE_BUSYBOX\|IMAGE_SPIRE_AGENT\|IMAGE_TOOLKIT' "${rendered}"; then
echo "unsubstituted image placeholder left in ${rendered}"
exit 1
fi
kubectl apply -f "${rendered}"
}
# Push with the writer identity.
apply_registry_job "${SCRIPTPATH}/image-push-job.yaml"
wait_for_job image-push
# Pull it back. Nothing in the job fetches a credential; kubelet runs the plugin, which is
# the whole point of the test.
kubectl apply -f "${SCRIPTPATH}/image-pull-job.yaml"
wait_for_job image-pull
# Completing at all is the assertion: zot grants no anonymous access, so the image only
# comes down if kubelet ran the plugin and the exchange minted a token zot accepted. The
# kubelet log line naming the plugin needs -v=4, which is not worth turning on for every
# example, so teardown prints it as a diagnostic rather than asserting on it.
kubectl logs job/image-pull | grep IMAGE-PULL-OK
# The pull identity is read only. This job completes only when zot refuses the write.
apply_registry_job "${SCRIPTPATH}/image-push-denied-job.yaml"
wait_for_job image-push-denied
kubectl logs job/image-push-denied | grep PUSH-DENIED-OK
if [ "${BROKER}" -eq 1 ]; then
# Verify a workload on the ha-agent socket receives the other.invalid federated trust bundles,
# x509 and jwt, merged from both sides.
@@ -441,3 +685,12 @@ if [ "${BROKER}" -eq 1 ]; then
run_federation_test_job
fi
# The image pull path has to survive losing a side too. Everything it depends on is HA:
# the node's identity comes from the host spire-ha-agent, and the credential provider
# talks to the combined exchange endpoint rather than either side directly. Delete the
# job first so this is a genuine second pull rather than a cached result.
kubectl delete job image-pull
kubectl apply -f "${SCRIPTPATH}/image-pull-job.yaml"
wait_for_job image-pull
kubectl logs job/image-pull | grep IMAGE-PULL-OK