panxiao81/helm-charts-hardened
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add broker suport to the spire-ha-agent (#884)

Browse Source 
* Add broker suport to the spire-ha-agent

Signed-off-by: Kevin Fox <[email protected]>

---------

Signed-off-by: Kevin Fox <[email protected]>
Signed-off-by: kfox1111 <[email protected]>

---------

Signed-off-by: Kevin Fox <[email protected]>
Signed-off-by: kfox1111 <[email protected]>
Co-authored-by: Faisal Memon <[email protected]>
This commit is contained in:
kfox1111
2026-07-30 13:15:55 -07:00
committed by GitHub
co-authored by Faisal Memon
parent 3cfefb72ba
commit f4c7df239e
10 changed files with 322 additions and 15 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/workflows/helm-chart-ci.yaml
+1 -1
View File
@@ -28,7 +28,7 @@ concurrency:
env: env:
HELM_VERSION: v3.16.2 HELM_VERSION: v3.16.2
PYTHON_VERSION: 3.11.3 PYTHON_VERSION: 3.11.3
KIND_VERSION: v0.24.0 KIND_VERSION: v0.32.0
CHART_TESTING_VERSION: v3.8.0 CHART_TESTING_VERSION: v3.8.0
jobs: jobs:
charts/spire-ha-agent/Chart.yaml
+1 -1
View File
@@ -3,7 +3,7 @@ name: spire-ha-agent
description: A Helm chart to install the SPIRE HA agent. description: A Helm chart to install the SPIRE HA agent.
type: application type: application
version: 0.2.0 version: 0.2.0
appVersion: "0.0.17" appVersion: "0.2.0"
keywords: ["spiffe", "spire-ha-agent"] keywords: ["spiffe", "spire-ha-agent"]
home: https://github.com/spiffe/helm-charts-hardened/tree/main/charts/spire-ha-agent home: https://github.com/spiffe/helm-charts-hardened/tree/main/charts/spire-ha-agent
sources: sources:
charts/spire-ha-agent/README.md
+10 -3
View File
@@ -30,10 +30,17 @@ A Helm chart to install the SPIRE HA agent.
| `image.repository` | The repository within the registry | `spiffe/spire-ha-agent` | | `image.repository` | The repository within the registry | `spiffe/spire-ha-agent` |
| `image.pullPolicy` | The image pull policy | `IfNotPresent` | | `image.pullPolicy` | The image pull policy | `IfNotPresent` |
| `image.tag` | Overrides the image tag whose default is the chart appVersion | `""` | | `image.tag` | Overrides the image tag whose default is the chart appVersion | `""` |
| `mode` | If the spire-ha-agent will run in delegated or broker mode | `delegated` |
| `singleSocket` | If in singleSocket mode, only one driver is used | `false` | | `singleSocket` | If in singleSocket mode, only one driver is used | `false` |
| `sockets.single.admin.hostPath` | Where the sockets are on disk when in single socket mode | `/var/run/spire/agent/sockets/main/csi.spiffe.io/admin` | | `sockets.single.admin.hostPath` | Where the admin socket is on disk when in single socket mode | `/var/run/spire/agent/sockets/main/csi.spiffe.io/admin` |
| `sockets.a.admin.hostPath` | Where the sockets are on disk | `/var/run/spire/agent/sockets/a/csi.spiffe.io/admin` | | `sockets.a.admin.hostPath` | Where the a admin socket is on disk | `/var/run/spire/agent/sockets/a/csi.spiffe.io/admin` |
| `sockets.b.admin.hostPath` | Where the sockets are on disk | `/var/run/spire/agent/sockets/b/csi.spiffe.io/admin` | | `sockets.b.admin.hostPath` | Where the b admin sockets is on disk | `/var/run/spire/agent/sockets/b/csi.spiffe.io/admin` |
| `sockets.single.broker.hostPath` | Where the broker socket is on disk when in single socket mode | `/var/run/spire/agent/sockets/main/csi.spiffe.io/broker` |
| `sockets.a.broker.hostPath` | Where the a broker socket is on disk | `/var/run/spire/agent/sockets/a/csi.spiffe.io/broker` |
| `sockets.b.broker.hostPath` | Where the b broker socket is on disk | `/var/run/spire/agent/sockets/b/csi.spiffe.io/broker` |
| `sockets.single.workload.hostPath` | Where the broker socket is on disk when in single socket mode | `/var/run/spire/agent-sockets` |
| `sockets.a.workload.hostPath` | Where the a workload socket is on disk | `/var/run/spire/agent/sockets/a/csi.spiffe.io/public` |
| `sockets.b.workload.hostPath` | Where the b workload socket is on disk | `/var/run/spire/agent/sockets/b/csi.spiffe.io/public` |
| `vsock` | Use a vsockets to expose the service rather then a unix socket | `false` | | `vsock` | Use a vsockets to expose the service rather then a unix socket | `false` |
| `port` | Port number to listen on | `999` | | `port` | Port number to listen on | `999` |
| `imagePullSecrets` | Pull secrets for images | `[]` | | `imagePullSecrets` | Pull secrets for images | `[]` |
charts/spire-ha-agent/templates/daemonset.yaml
+55
View File
@@ -119,6 +119,9 @@ spec:
- name: {{ .Chart.Name | quote }} - name: {{ .Chart.Name | quote }}
image: {{ template "spire-lib.image" (dict "appVersion" $.Chart.AppVersion "image" .Values.image "global" .Values.global) }} image: {{ template "spire-lib.image" (dict "appVersion" $.Chart.AppVersion "image" .Values.image "global" .Values.global) }}
imagePullPolicy: {{ .Values.image.pullPolicy | quote }} imagePullPolicy: {{ .Values.image.pullPolicy | quote }}
args:
- "-mode"
- {{ .Values.mode | quote }}
securityContext: securityContext:
privileged: true privileged: true
#FIXME read permission to api socket #FIXME read permission to api socket
@@ -131,13 +134,31 @@ spec:
{{- if .Values.singleSocket }} {{- if .Values.singleSocket }}
- name: SPIRE_HA_AGENT_SINGLE - name: SPIRE_HA_AGENT_SINGLE
value: enabled value: enabled
{{- if eq .Values.mode "delegated" }}
- name: SPIRE_HA_AGENT_SOCKET - name: SPIRE_HA_AGENT_SOCKET
value: unix:///var/run/spire/agent/sockets/a/csi.spiffe.io/admin/admin.sock value: unix:///var/run/spire/agent/sockets/a/csi.spiffe.io/admin/admin.sock
{{- else }} {{- else }}
- name: SPIRE_HA_AGENT_BROKER
value: unix:///var/run/spire/agent/sockets/a/csi.spiffe.io/broker/broker.sock
- name: SPIRE_HA_AGENT_WORKLOAD_SOCKET
value: unix:///var/run/spire/agent/sockets/a/csi.spiffe.io/public/spire-agent.sock
{{- end }}
{{- else }}
{{- if eq .Values.mode "delegated" }}
- name: SPIRE_HA_AGENT_SOCKET_A - name: SPIRE_HA_AGENT_SOCKET_A
value: unix:///var/run/spire/agent/sockets/a/csi.spiffe.io/admin/admin.sock value: unix:///var/run/spire/agent/sockets/a/csi.spiffe.io/admin/admin.sock
- name: SPIRE_HA_AGENT_SOCKET_B - name: SPIRE_HA_AGENT_SOCKET_B
value: unix:///var/run/spire/agent/sockets/b/csi.spiffe.io/admin/admin.sock value: unix:///var/run/spire/agent/sockets/b/csi.spiffe.io/admin/admin.sock
{{- else }}
- name: SPIRE_HA_AGENT_BROKER_A
value: unix:///var/run/spire/agent/sockets/a/csi.spiffe.io/broker/broker.sock
- name: SPIRE_HA_AGENT_BROKER_B
value: unix:///var/run/spire/agent/sockets/b/csi.spiffe.io/broker/broker.sock
- name: SPIRE_HA_AGENT_WORKLOAD_SOCKET_A
value: unix:///var/run/spire/agent/sockets/a/csi.spiffe.io/public/spire-agent.sock
- name: SPIRE_HA_AGENT_WORKLOAD_SOCKET_B
value: unix:///var/run/spire/agent/sockets/b/csi.spiffe.io/public/spire-agent.sock
{{- end }}
{{- end }} {{- end }}
{{- if .Values.vsock }} {{- if .Values.vsock }}
- name: SPIRE_HA_AGENT_VSOCK - name: SPIRE_HA_AGENT_VSOCK
@@ -161,12 +182,25 @@ spec:
mountPath: /tmp/spire-ha-agent/public mountPath: /tmp/spire-ha-agent/public
readOnly: false readOnly: false
{{- end }} {{- end }}
{{- if eq .Values.mode "delegated" }}
- name: spire-ha-admin-socket-dir-upstream-a - name: spire-ha-admin-socket-dir-upstream-a
mountPath: /var/run/spire/agent/sockets/a/csi.spiffe.io/admin mountPath: /var/run/spire/agent/sockets/a/csi.spiffe.io/admin
{{- if not .Values.singleSocket }} {{- if not .Values.singleSocket }}
- name: spire-ha-admin-socket-dir-upstream-b - name: spire-ha-admin-socket-dir-upstream-b
mountPath: /var/run/spire/agent/sockets/b/csi.spiffe.io/admin mountPath: /var/run/spire/agent/sockets/b/csi.spiffe.io/admin
{{- end }} {{- end }}
{{- else }}
- name: spire-ha-broker-socket-dir-upstream-a
mountPath: /var/run/spire/agent/sockets/a/csi.spiffe.io/broker
- name: spire-ha-workload-socket-dir-upstream-a
mountPath: /var/run/spire/agent/sockets/a/csi.spiffe.io/public
{{- if not .Values.singleSocket }}
- name: spire-ha-broker-socket-dir-upstream-b
mountPath: /var/run/spire/agent/sockets/b/csi.spiffe.io/broker
- name: spire-ha-workload-socket-dir-upstream-b
mountPath: /var/run/spire/agent/sockets/b/csi.spiffe.io/public
{{- end }}
{{- end }}
- name: dev - name: dev
mountPath: /dev mountPath: /dev
{{- if gt (len .Values.extraVolumeMounts) 0 }} {{- if gt (len .Values.extraVolumeMounts) 0 }}
@@ -196,6 +230,7 @@ spec:
path: {{ $socketPath | dir }} path: {{ $socketPath | dir }}
type: DirectoryOrCreate type: DirectoryOrCreate
{{- end }} {{- end }}
{{- if eq .Values.mode "delegated" }}
- name: spire-ha-admin-socket-dir-upstream-a - name: spire-ha-admin-socket-dir-upstream-a
hostPath: hostPath:
path: {{ if .Values.singleSocket }}{{ .Values.sockets.single.admin.hostPath }}{{ else }}{{ .Values.sockets.a.admin.hostPath }}{{ end }} path: {{ if .Values.singleSocket }}{{ .Values.sockets.single.admin.hostPath }}{{ else }}{{ .Values.sockets.a.admin.hostPath }}{{ end }}
@@ -206,6 +241,26 @@ spec:
path: {{ .Values.sockets.b.admin.hostPath }} path: {{ .Values.sockets.b.admin.hostPath }}
type: DirectoryOrCreate type: DirectoryOrCreate
{{- end }} {{- end }}
{{- else }}
- name: spire-ha-broker-socket-dir-upstream-a
hostPath:
path: {{ if .Values.singleSocket }}{{ .Values.sockets.single.broker.hostPath }}{{ else }}{{ .Values.sockets.a.broker.hostPath }}{{ end }}
type: DirectoryOrCreate
- name: spire-ha-workload-socket-dir-upstream-a
hostPath:
path: {{ if .Values.singleSocket }}{{ .Values.sockets.single.workload.hostPath }}{{ else }}{{ .Values.sockets.a.workload.hostPath }}{{ end }}
type: DirectoryOrCreate
{{- if not .Values.singleSocket }}
- name: spire-ha-broker-socket-dir-upstream-b
hostPath:
path: {{ .Values.sockets.b.broker.hostPath }}
type: DirectoryOrCreate
- name: spire-ha-workload-socket-dir-upstream-b
hostPath:
path: {{ .Values.sockets.b.workload.hostPath }}
type: DirectoryOrCreate
{{- end }}
{{- end }}
- name: dev - name: dev
hostPath: hostPath:
path: /dev path: /dev
charts/spire-ha-agent/values.yaml
+24 -3
View File
@@ -17,22 +17,43 @@ image:
pullPolicy: IfNotPresent pullPolicy: IfNotPresent
tag: "" tag: ""
## @param mode If the spire-ha-agent will run in delegated or broker mode
mode: delegated
## @param singleSocket If in singleSocket mode, only one driver is used ## @param singleSocket If in singleSocket mode, only one driver is used
singleSocket: false singleSocket: false
## @param sockets.single.admin.hostPath Where the sockets are on disk when in single socket mode ## @param sockets.single.admin.hostPath Where the admin socket is on disk when in single socket mode
## @param sockets.a.admin.hostPath Where the sockets are on disk ## @param sockets.a.admin.hostPath Where the a admin socket is on disk
## @param sockets.b.admin.hostPath Where the sockets are on disk ## @param sockets.b.admin.hostPath Where the b admin sockets is on disk
## @param sockets.single.broker.hostPath Where the broker socket is on disk when in single socket mode
## @param sockets.a.broker.hostPath Where the a broker socket is on disk
## @param sockets.b.broker.hostPath Where the b broker socket is on disk
## @param sockets.single.workload.hostPath Where the broker socket is on disk when in single socket mode
## @param sockets.a.workload.hostPath Where the a workload socket is on disk
## @param sockets.b.workload.hostPath Where the b workload socket is on disk
sockets: sockets:
single: single:
admin: admin:
hostPath: /var/run/spire/agent/sockets/main/csi.spiffe.io/admin hostPath: /var/run/spire/agent/sockets/main/csi.spiffe.io/admin
broker:
hostPath: /var/run/spire/agent/sockets/main/csi.spiffe.io/broker
workload:
hostPath: /var/run/spire/agent-sockets
a: a:
admin: admin:
hostPath: /var/run/spire/agent/sockets/a/csi.spiffe.io/admin hostPath: /var/run/spire/agent/sockets/a/csi.spiffe.io/admin
broker:
hostPath: /var/run/spire/agent/sockets/a/csi.spiffe.io/broker
workload:
hostPath: /var/run/spire/agent/sockets/a/csi.spiffe.io/public
b: b:
admin: admin:
hostPath: /var/run/spire/agent/sockets/b/csi.spiffe.io/admin hostPath: /var/run/spire/agent/sockets/b/csi.spiffe.io/admin
broker:
hostPath: /var/run/spire/agent/sockets/b/csi.spiffe.io/broker
workload:
hostPath: /var/run/spire/agent/sockets/b/csi.spiffe.io/public
## @param vsock Use a vsockets to expose the service rather then a unix socket ## @param vsock Use a vsockets to expose the service rather then a unix socket
vsock: false vsock: false
charts/spire-nested/README.md
+6
View File
@@ -466,6 +466,9 @@ Now you can interact with the Spire agent socket from your own application. The
| `downstream-spire-agent-bottom-turtle-ha-a.sockets.admin.enabled` | Enable admin socket | `true` | | `downstream-spire-agent-bottom-turtle-ha-a.sockets.admin.enabled` | Enable admin socket | `true` |
| `downstream-spire-agent-bottom-turtle-ha-a.sockets.admin.mountOnHost` | Mount admin socket on host | `true` | | `downstream-spire-agent-bottom-turtle-ha-a.sockets.admin.mountOnHost` | Mount admin socket on host | `true` |
| `downstream-spire-agent-bottom-turtle-ha-a.authorizedDelegates` | List of workloads able to use the delegation api | `["/spire-ha-agent"]` | | `downstream-spire-agent-bottom-turtle-ha-a.authorizedDelegates` | List of workloads able to use the delegation api | `["/spire-ha-agent"]` |
| `downstream-spire-agent-bottom-turtle-ha-a.brokerAPI.brokers.spire-ha-agent.enabled` | Enable the spire-ha-agent by default | `true` |
| `downstream-spire-agent-bottom-turtle-ha-a.workloadAttestors.k8s.brokerAPI.accessPolicy` | The default accessPolicy | `permissive` |
| `downstream-spire-agent-bottom-turtle-ha-a.workloadAttestors.k8s.brokerAPI.brokers.spire-ha-agent.enabled` | Enable the spire-ha-agent by default | `true` |
| `downstream-spire-agent-bottom-turtle-ha-b.nameOverride` | Overrides the name of Spire agent pods | `agent-downstream` | | `downstream-spire-agent-bottom-turtle-ha-b.nameOverride` | Overrides the name of Spire agent pods | `agent-downstream` |
| `downstream-spire-agent-bottom-turtle-ha-b.server.nameOverride` | The name override setting of the internal SPIRE server | `internal-server` | | `downstream-spire-agent-bottom-turtle-ha-b.server.nameOverride` | The name override setting of the internal SPIRE server | `internal-server` |
| `downstream-spire-agent-bottom-turtle-ha-b.bundleConfigMap` | The name of the configmap that contains the downstream bundle | `spire-server-b-bundle` | | `downstream-spire-agent-bottom-turtle-ha-b.bundleConfigMap` | The name of the configmap that contains the downstream bundle | `spire-server-b-bundle` |
@@ -485,6 +488,9 @@ Now you can interact with the Spire agent socket from your own application. The
| `downstream-spire-agent-bottom-turtle-ha-b.sockets.admin.enabled` | Enable admin socket | `true` | | `downstream-spire-agent-bottom-turtle-ha-b.sockets.admin.enabled` | Enable admin socket | `true` |
| `downstream-spire-agent-bottom-turtle-ha-b.sockets.admin.mountOnHost` | Mount admin socket on host | `true` | | `downstream-spire-agent-bottom-turtle-ha-b.sockets.admin.mountOnHost` | Mount admin socket on host | `true` |
| `downstream-spire-agent-bottom-turtle-ha-b.authorizedDelegates` | List of workloads able to use the delegation api | `["/spire-ha-agent"]` | | `downstream-spire-agent-bottom-turtle-ha-b.authorizedDelegates` | List of workloads able to use the delegation api | `["/spire-ha-agent"]` |
| `downstream-spire-agent-bottom-turtle-ha-b.brokerAPI.brokers.spire-ha-agent.enabled` | Enable the spire-ha-agent by default | `true` |
| `downstream-spire-agent-bottom-turtle-ha-b.workloadAttestors.k8s.brokerAPI.accessPolicy` | The default accessPolicy | `permissive` |
| `downstream-spire-agent-bottom-turtle-ha-b.workloadAttestors.k8s.brokerAPI.brokers.spire-ha-agent.enabled` | Enable the spire-ha-agent by default | `true` |
| `downstream-spiffe-csi-driver-bottom-turtle-ha-a.fullnameOverride` | Fullname override | `spiffe-csi-driver-downstream-a` | | `downstream-spiffe-csi-driver-bottom-turtle-ha-a.fullnameOverride` | Fullname override | `spiffe-csi-driver-downstream-a` |
| `downstream-spiffe-csi-driver-bottom-turtle-ha-a.agentSocketPath` | path to agent socket | `/var/run/spire/agent/sockets/a/csi.spiffe.io/public/spire-agent.sock` | | `downstream-spiffe-csi-driver-bottom-turtle-ha-a.agentSocketPath` | path to agent socket | `/var/run/spire/agent/sockets/a/csi.spiffe.io/public/spire-agent.sock` |
| `downstream-spiffe-csi-driver-bottom-turtle-ha-a.pluginName` | The name of the plugin instance | `a.csi.spiffe.io` | | `downstream-spiffe-csi-driver-bottom-turtle-ha-a.pluginName` | The name of the plugin instance | `a.csi.spiffe.io` |
charts/spire-nested/values.yaml
+28
View File
@@ -680,6 +680,20 @@ downstream-spire-agent-bottom-turtle-ha-a:
## @param downstream-spire-agent-bottom-turtle-ha-a.authorizedDelegates List of workloads able to use the delegation api ## @param downstream-spire-agent-bottom-turtle-ha-a.authorizedDelegates List of workloads able to use the delegation api
authorizedDelegates: authorizedDelegates:
- /spire-ha-agent - /spire-ha-agent
brokerAPI:
brokers:
spire-ha-agent:
## @param downstream-spire-agent-bottom-turtle-ha-a.brokerAPI.brokers.spire-ha-agent.enabled Enable the spire-ha-agent by default
enabled: true
workloadAttestors:
k8s:
brokerAPI:
## @param downstream-spire-agent-bottom-turtle-ha-a.workloadAttestors.k8s.brokerAPI.accessPolicy The default accessPolicy
accessPolicy: permissive
brokers:
spire-ha-agent:
## @param downstream-spire-agent-bottom-turtle-ha-a.workloadAttestors.k8s.brokerAPI.brokers.spire-ha-agent.enabled Enable the spire-ha-agent by default
enabled: true
# Used with tags [bottomTurtleHAB] # Used with tags [bottomTurtleHAB]
downstream-spire-agent-bottom-turtle-ha-b: downstream-spire-agent-bottom-turtle-ha-b:
@@ -737,6 +751,20 @@ downstream-spire-agent-bottom-turtle-ha-b:
## @param downstream-spire-agent-bottom-turtle-ha-b.authorizedDelegates List of workloads able to use the delegation api ## @param downstream-spire-agent-bottom-turtle-ha-b.authorizedDelegates List of workloads able to use the delegation api
authorizedDelegates: authorizedDelegates:
- /spire-ha-agent - /spire-ha-agent
brokerAPI:
brokers:
spire-ha-agent:
## @param downstream-spire-agent-bottom-turtle-ha-b.brokerAPI.brokers.spire-ha-agent.enabled Enable the spire-ha-agent by default
enabled: true
workloadAttestors:
k8s:
brokerAPI:
## @param downstream-spire-agent-bottom-turtle-ha-b.workloadAttestors.k8s.brokerAPI.accessPolicy The default accessPolicy
accessPolicy: permissive
brokers:
spire-ha-agent:
## @param downstream-spire-agent-bottom-turtle-ha-b.workloadAttestors.k8s.brokerAPI.brokers.spire-ha-agent.enabled Enable the spire-ha-agent by default
enabled: true
# Used with tags [bottomTurtleHAA] # Used with tags [bottomTurtleHAA]
downstream-spiffe-csi-driver-bottom-turtle-ha-a: downstream-spiffe-csi-driver-bottom-turtle-ha-a:
examples/bottom-turtle-ha/federation-test-job.yaml
+74
View File
@@ -0,0 +1,74 @@
apiVersion: batch/v1
kind: Job
metadata:
name: federation-test
spec:
backoffLimit: 0
template:
metadata:
labels:
app: federation-test
spec:
restartPolicy: Never
initContainers:
- name: static-busybox
# Replaced by run-tests.sh with the image from the spiffe-oidc-discovery-provider chart
image: replaced-by-run-tests.sh
command:
- sh
- -c
- |
cp /bin/busybox /data/busybox
chmod +x /data/busybox
volumeMounts:
- name: data-volume
mountPath: /data
containers:
- name: main
# Replaced by run-tests.sh with the image from the spire-agent chart
image: replaced-by-run-tests.sh
command:
- /data/busybox
- sh
- -xc
- |
SOCK=/spire-agent/spire-agent.sock
i=0
while [ "$i" -lt 60 ]; do
if XOUT=$(/opt/spire/bin/spire-agent api fetch x509 -socketPath "$SOCK" -write /data -timeout 5s 2>&1) &&
echo "$XOUT" | /data/busybox grep -q "for trust domain other.org" &&
JOUT=$(/opt/spire/bin/spire-agent api fetch jwt -audience test -socketPath "$SOCK" -timeout 5s 2>&1) &&
echo "$JOUT" | /data/busybox grep -q "bundle(other.org)"; then
# The other.org bundle was statically set to the same single CA on both sides,
# so every federated bundle delivered must contain exactly one certificate.
for f in /data/federated_bundle.*.pem; do
COUNT=$(/data/busybox grep -c "BEGIN CERTIFICATE" "$f")
if [ "$COUNT" != "1" ]; then
echo "expected 1 CA in $f, got $COUNT"
exit 1
fi
done
echo "$XOUT"
echo "$JOUT"
echo FEDERATION-OK
exit 0
fi
i=$((i+1))
/data/busybox sleep 2
done
echo "$XOUT"
echo "$JOUT"
exit 1
volumeMounts:
- name: data-volume
mountPath: /data
- name: spire-api
mountPath: /spire-agent
readOnly: true
volumes:
- name: data-volume
emptyDir: {}
- name: spire-api
csi:
driver: csi.spiffe.io
readOnly: true
examples/bottom-turtle-ha/run-tests.sh
+112 -5
View File
@@ -14,6 +14,7 @@ source "${SCRIPTPATH}/../../.github/scripts/parse-versions.sh"
source "${TESTDIR}/common.sh" source "${TESTDIR}/common.sh"
CLEANUP=1 CLEANUP=1
BROKER=0
for i in "$@"; do for i in "$@"; do
case $i in case $i in
@@ -21,9 +22,38 @@ for i in "$@"; do
CLEANUP=0 CLEANUP=0
shift # past argument=value shift # past argument=value
;; ;;
-b)
BROKER=1
shift # past argument=value
;;
esac esac
done done
# With -b, test the spire-ha-agent broker api instead of the delegated api.
# Broker mode also supports federated trust bundles, so federate the ha-agent's own entry and a
# dedicated federation-test workload entry with the other.org trust domain on both sides. Delegated
# mode only tolerates the local and spire-ha bundles, so none of this may apply without -b.
BROKER_MODE_ARGS=()
BROKER_SOCKET_ARGS_A=()
BROKER_SOCKET_ARGS_B=()
if [ "${BROKER}" -eq 1 ]; then
BROKER_MODE_ARGS=(--set "spire-ha-agent.mode=broker")
BROKER_SOCKET_ARGS_A=(
--set downstream-spire-agent-bottom-turtle-ha-a.sockets.broker.enabled=true
--set downstream-spire-agent-bottom-turtle-ha-a.sockets.broker.mountOnHost=true
--set 'internal-spire-server-bottom-turtle-ha-a.controllerManager.identities.clusterSPIFFEIDs.spire-ha-agent.federatesWith={spire-ha,other.org}'
--set 'internal-spire-server-bottom-turtle-ha-a.controllerManager.identities.clusterSPIFFEIDs.federation-test.federatesWith={other.org}'
--set 'internal-spire-server-bottom-turtle-ha-a.controllerManager.identities.clusterSPIFFEIDs.federation-test.podSelector.matchLabels.app=federation-test'
)
BROKER_SOCKET_ARGS_B=(
--set downstream-spire-agent-bottom-turtle-ha-b.sockets.broker.enabled=true
--set downstream-spire-agent-bottom-turtle-ha-b.sockets.broker.mountOnHost=true
--set 'internal-spire-server-bottom-turtle-ha-b.controllerManager.identities.clusterSPIFFEIDs.spire-ha-agent.federatesWith={spire-ha,other.org}'
--set 'internal-spire-server-bottom-turtle-ha-b.controllerManager.identities.clusterSPIFFEIDs.federation-test.federatesWith={other.org}'
--set 'internal-spire-server-bottom-turtle-ha-b.controllerManager.identities.clusterSPIFFEIDs.federation-test.podSelector.matchLabels.app=federation-test'
)
fi
if [ "x${GITHUB_JOB}" != "x" ]; then if [ "x${GITHUB_JOB}" != "x" ]; then
echo "Running in GitHub" echo "Running in GitHub"
else else
@@ -36,6 +66,9 @@ teardown() {
docker exec -i chart-testing-worker /bin/bash -c "more /var/lib/kubelet/pods/*/volumes/kubernetes.io~empty-dir/disk-keymanager/keys.json /var/lib/kubelet/pods/*/volumes/kubernetes.io~empty-dir/spire-agent-persistence/agent-data.json | cat" docker exec -i chart-testing-worker /bin/bash -c "more /var/lib/kubelet/pods/*/volumes/kubernetes.io~empty-dir/disk-keymanager/keys.json /var/lib/kubelet/pods/*/volumes/kubernetes.io~empty-dir/spire-agent-persistence/agent-data.json | cat"
sudo systemctl status spire-server@a || true sudo systemctl status spire-server@a || true
sudo systemctl status spire-server@b || true sudo systemctl status spire-server@b || true
sudo systemctl status spire-server@other || true
kubectl describe job federation-test || true
kubectl logs job/federation-test || true
sudo spire-server entry show -instance a || true sudo spire-server entry show -instance a || true
sudo spire-server entry show -instance b || true sudo spire-server entry show -instance b || true
sudo systemctl status spire-controller-manager@a || true sudo systemctl status spire-controller-manager@a || true
@@ -59,6 +92,9 @@ teardown() {
kubectl exec -i -n spire-server spire-a-internal-server-0 -- spire-server agent list -output json | yq e . - -P || true kubectl exec -i -n spire-server spire-a-internal-server-0 -- spire-server agent list -output json | yq e . - -P || true
kubectl exec -i -n spire-server spire-b-internal-server-0 -- spire-server agent list -output json | yq e . - -P || true kubectl exec -i -n spire-server spire-b-internal-server-0 -- spire-server agent list -output json | yq e . - -P || true
kubectl get pods -A -o wide || true kubectl get pods -A -o wide || true
kubectl describe daemonset pods -n spire-system || true
kubectl get configmap -n spire-system || true
kubectl get configmap -n spire-system spire-a-agent-downstream -o yaml || true
print_helm_releases print_helm_releases
@@ -68,6 +104,7 @@ teardown() {
fi fi
if [ "${CLEANUP}" -eq 1 ]; then if [ "${CLEANUP}" -eq 1 ]; then
kubectl delete job federation-test 2>/dev/null || true
helm uninstall --namespace spire-mgmt spire-b 2>/dev/null || true helm uninstall --namespace spire-mgmt spire-b 2>/dev/null || true
helm uninstall --namespace spire-mgmt spire-a 2>/dev/null || true helm uninstall --namespace spire-mgmt spire-a 2>/dev/null || true
helm uninstall --namespace spire-mgmt spire 2>/dev/null || true helm uninstall --namespace spire-mgmt spire 2>/dev/null || true
@@ -127,6 +164,30 @@ wait_for_jwt() {
return 1 return 1
} }
wait_for_entry_federation() {
local pod="$1"
local trustdomain="$2"
local timeout=60
local count=0
while [ "$count" -lt "$timeout" ]; do
if kubectl exec -i -n spire-server "$pod" -- spire-server entry show -spiffeID spiffe://production.other/spire-ha-agent | grep -q "$trustdomain"; then
return 0
fi
sleep 2
((count++)) || true
done
return 1
}
run_federation_test_job() {
kubectl delete job federation-test 2>/dev/null || true
# Inject the images from the charts into the job so they always sync up
yq e "(.spec.template.spec.initContainers[] | select(.name == \"static-busybox\") | .image) = \"${BUSYBOX_IMAGE}\" | (.spec.template.spec.containers[] | select(.name == \"main\") | .image) = \"${AGENT_IMAGE}\"" \
"${SCRIPTPATH}/federation-test-job.yaml" | kubectl apply -f -
kubectl wait --for=condition=complete --timeout=240s job/federation-test
kubectl logs job/federation-test | grep FEDERATION-OK
}
"${SCRIPTPATH}/../../.github/scripts/prepare-local-chart-deps.sh" "${SCRIPTPATH}/../../.github/scripts/prepare-local-chart-deps.sh"
# Get the package repo and install the packages # Get the package repo and install the packages
@@ -137,6 +198,25 @@ sudo apt-get install -y spire-common spire-agent spire-server spire-controller-m
# Set our testing trust domain # Set our testing trust domain
sudo sed -i 's/example.org/production.other/' /etc/spiffe/default-trust-domain.env sudo sed -i 's/example.org/production.other/' /etc/spiffe/default-trust-domain.env
if [ "${BROKER}" -eq 1 ]; then
# Pull the federation test job images out of the charts so they always sync up.
AGENT_IMAGE=$(helm template t charts/spire -s charts/spire-agent/templates/daemonset.yaml --values "${COMMON_TEST_YOUR_VALUES}" --set spire-agent.enabled=true | yq e 'select(.kind=="DaemonSet") | .spec.template.spec.containers[] | select(.name=="spire-agent") | .image' -)
BUSYBOX_IMAGE=$(helm template t charts/spire -s charts/spiffe-oidc-discovery-provider/templates/tests/test-keys.yaml --values "${COMMON_TEST_YOUR_VALUES}" --set spiffe-oidc-discovery-provider.enabled=true | yq e 'select(.kind=="Pod") | .spec.initContainers[] | select(.name=="static-busybox") | .image' -)
echo "federation test job images: ${AGENT_IMAGE} ${BUSYBOX_IMAGE}"
# Mint a trust bundle for a foreign trust domain (other.org) to test federated trust bundle
# support. A throwaway third spire-server instance produces a genuine spiffe format bundle
# carrying both x509 and jwt authorities. The instance env file overrides the global trust
# domain since systemd applies later EnvironmentFiles last.
sudo /bin/bash -c '(echo SPIFFE_TRUST_DOMAIN=other.org; echo SPIRE_BIND_PORT=8083) > /etc/spire/server/other.env'
sudo systemctl start spire-server@other
wait_for_healthcheck spire-server /run/spire/server/sockets/other/private/api.sock
sudo spire-server bundle show -format spiffe -socketPath /run/spire/server/sockets/other/private/api.sock | sudo tee /tmp/other-org-bundle.json > /dev/null
sudo systemctl stop spire-server@other
grep -q '"x509-svid"' /tmp/other-org-bundle.json
grep -q '"jwt-svid"' /tmp/other-org-bundle.json
fi
# register some workloads with the spire server using manifests # register some workloads with the spire server using manifests
sudo mkdir -p /etc/spire/server/a/manifests/ /etc/spire/server/b/manifests/ sudo mkdir -p /etc/spire/server/a/manifests/ /etc/spire/server/b/manifests/
sudo cp "${SCRIPTPATH}/example-manifests"/* /etc/spire/server/a/manifests/ sudo cp "${SCRIPTPATH}/example-manifests"/* /etc/spire/server/a/manifests/
@@ -235,7 +315,8 @@ helm upgrade --install --create-namespace --namespace spire-mgmt --values "${COM
--set tags.haAgentCommon=true \ --set tags.haAgentCommon=true \
--set "global.spire.namespaces.create=true" \ --set "global.spire.namespaces.create=true" \
--set "global.spire.ingressControllerType=ingress-nginx" \ --set "global.spire.ingressControllerType=ingress-nginx" \
--set "spiffe-oidc-discovery-provider.ingress.enabled=true" --set "spiffe-oidc-discovery-provider.ingress.enabled=true" \
"${BROKER_MODE_ARGS[@]}"
# Create spire-identity-exchange cert for testing. # Create spire-identity-exchange cert for testing.
mkdir -p certs mkdir -p certs
@@ -253,14 +334,21 @@ helm upgrade --install --namespace spire-mgmt --values "${COMMON_TEST_YOUR_VALUE
--set tags.bottomTurtleHAA=true \ --set tags.bottomTurtleHAA=true \
--values "${SCRIPTPATH}/spire-identity-exchange-values.yaml" \ --values "${SCRIPTPATH}/spire-identity-exchange-values.yaml" \
--set "spire-identity-exchange-bottom-turtle-ha-a.enabled=true" \ --set "spire-identity-exchange-bottom-turtle-ha-a.enabled=true" \
--set "global.spire.ingressControllerType=ingress-nginx" --set "global.spire.ingressControllerType=ingress-nginx" \
"${BROKER_SOCKET_ARGS_A[@]}"
if [ "${BROKER}" -eq 1 ]; then
# Install the other.org bundle so the controller manager can create the entries that federate
# with it. It retries any entries that failed with "unable to find federated bundle".
kubectl exec -i -n spire-server spire-a-internal-server-0 -- spire-server bundle set -format spiffe -id spiffe://other.org < /tmp/other-org-bundle.json
fi
docker exec -i chart-testing-worker /bin/bash -c "more /var/lib/kubelet/pods/*/volumes/kubernetes.io~empty-dir/disk-keymanager/keys.json /var/lib/kubelet/pods/*/volumes/kubernetes.io~empty-dir/spire-agent-persistence/agent-data.json | cat" docker exec -i chart-testing-worker /bin/bash -c "more /var/lib/kubelet/pods/*/volumes/kubernetes.io~empty-dir/disk-keymanager/keys.json /var/lib/kubelet/pods/*/volumes/kubernetes.io~empty-dir/spire-agent-persistence/agent-data.json | cat"
# Rollout just to sped up the tests # Rollout just to sped up the tests
kubectl patch deployment spiffe-oidc-discovery-provider -n spire-server --type='strategic' -p '{"spec": {"strategy": {"type": "Recreate", "rollingUpdate": null}}}' kubectl patch deployment spiffe-oidc-discovery-provider -n spire-server --type='strategic' -p '{"spec": {"strategy": {"type": "Recreate", "rollingUpdate": null}}}'
kubectl rollout restart daemonset -n spire-system spire-ha-agent kubectl rollout restart daemonset -n spire-system spire-ha-agent
kubectl rollout status daemonset -n spire-system spire-ha-agent kubectl rollout status daemonset -n spire-system spire-ha-agent --timeout=1m
kubectl rollout restart deployment -n spire-server spiffe-oidc-discovery-provider kubectl rollout restart deployment -n spire-server spiffe-oidc-discovery-provider
kubectl rollout status deployment -n spire-server spiffe-oidc-discovery-provider --timeout=1m kubectl rollout status deployment -n spire-server spiffe-oidc-discovery-provider --timeout=1m
kubectl wait -n spire-server --for=condition=ready pod -l "app.kubernetes.io/name=spiffe-oidc-discovery-provider" --field-selector=status.phase=Running --timeout=90s kubectl wait -n spire-server --for=condition=ready pod -l "app.kubernetes.io/name=spiffe-oidc-discovery-provider" --field-selector=status.phase=Running --timeout=90s
@@ -273,7 +361,15 @@ helm upgrade --install --namespace spire-mgmt --values "${COMMON_TEST_YOUR_VALUE
--set internal-spire-server-bottom-turtle-ha-b.upstreamAuthority.spire.server.port=8082 \ --set internal-spire-server-bottom-turtle-ha-b.upstreamAuthority.spire.server.port=8082 \
--values "${SCRIPTPATH}/spire-identity-exchange-values.yaml" \ --values "${SCRIPTPATH}/spire-identity-exchange-values.yaml" \
--set "spire-identity-exchange-bottom-turtle-ha-b.enabled=true" \ --set "spire-identity-exchange-bottom-turtle-ha-b.enabled=true" \
--set "global.spire.ingressControllerType=ingress-nginx" --set "global.spire.ingressControllerType=ingress-nginx" \
"${BROKER_SOCKET_ARGS_B[@]}"
if [ "${BROKER}" -eq 1 ]; then
kubectl exec -i -n spire-server spire-b-internal-server-0 -- spire-server bundle set -format spiffe -id spiffe://other.org < /tmp/other-org-bundle.json
# Both sides' spire-ha-agent entries must federate with other.org before the workload test.
wait_for_entry_federation spire-a-internal-server-0 other.org
wait_for_entry_federation spire-b-internal-server-0 other.org
fi
docker ps docker ps
docker exec -i chart-testing-worker /bin/bash -c "more /var/lib/kubelet/pods/*/volumes/kubernetes.io~empty-dir/disk-keymanager/keys.json /var/lib/kubelet/pods/*/volumes/kubernetes.io~empty-dir/spire-agent-persistence/agent-data.json | cat" docker exec -i chart-testing-worker /bin/bash -c "more /var/lib/kubelet/pods/*/volumes/kubernetes.io~empty-dir/disk-keymanager/keys.json /var/lib/kubelet/pods/*/volumes/kubernetes.io~empty-dir/spire-agent-persistence/agent-data.json | cat"
@@ -305,11 +401,22 @@ TOKEN=$(kubectl logs job/test)
curl -f -H "Authorization: Bearer ${TOKEN}" -X POST --resolve "spire-identity-exchange-a-rest.production.other:443:$IP" "https://spire-identity-exchange-a-rest.production.other/api/v1/svid/k8s_psat/x509" -k -sS -q curl -f -H "Authorization: Bearer ${TOKEN}" -X POST --resolve "spire-identity-exchange-a-rest.production.other:443:$IP" "https://spire-identity-exchange-a-rest.production.other/api/v1/svid/k8s_psat/x509" -k -sS -q
curl -f -H "Authorization: Bearer ${TOKEN}" -X POST --resolve "spire-identity-exchange-b-rest.production.other:443:$IP" "https://spire-identity-exchange-b-rest.production.other/api/v1/svid/k8s_psat/x509" -k -sS -q curl -f -H "Authorization: Bearer ${TOKEN}" -X POST --resolve "spire-identity-exchange-b-rest.production.other:443:$IP" "https://spire-identity-exchange-b-rest.production.other/api/v1/svid/k8s_psat/x509" -k -sS -q
if [ "${BROKER}" -eq 1 ]; then
# Verify a workload on the ha-agent socket receives the other.org federated trust bundles,
# x509 and jwt, merged from both sides.
run_federation_test_job
fi
#Test out running only on side b since we know already only both servers work together, and that only side a works if we made it this far. #Test out running only on side b since we know already only both servers work together, and that only side a works if we made it this far.
helm delete -n spire-mgmt spire-a helm delete -n spire-mgmt spire-a
kubectl rollout restart daemonset -n spire-system spire-ha-agent kubectl rollout restart daemonset -n spire-system spire-ha-agent
kubectl rollout status daemonset -n spire-system spire-ha-agent kubectl rollout status daemonset -n spire-system spire-ha-agent
kubectl rollout restart deployment -n spire-server spiffe-oidc-discovery-provider kubectl rollout restart deployment -n spire-server spiffe-oidc-discovery-provider
kubectl rollout status deployment -n spire-server spiffe-oidc-discovery-provider --timeout=1m kubectl rollout status deployment -n spire-server spiffe-oidc-discovery-provider --timeout=5m
curl -k --resolve "oidc-discovery.production.other:443:$IP" "https://oidc-discovery.production.other/.well-known/openid-configuration" -s --fail curl -k --resolve "oidc-discovery.production.other:443:$IP" "https://oidc-discovery.production.other/.well-known/openid-configuration" -s --fail
if [ "${BROKER}" -eq 1 ]; then
# Verify the other.org federated trust bundles still serve with only side b running.
run_federation_test_job
fi
tests/integration/bottom-turtle-ha-broker/run-tests.sh
Executable
+9
View File
@@ -0,0 +1,9 @@
#!/usr/bin/env bash
set -xe
SCRIPT="$(readlink -f "$0")"
SCRIPTPATH="$(dirname "${SCRIPT}")"
# Run the bottom-turtle-ha example test with the spire-ha-agent broker api enabled.
exec "${SCRIPTPATH}/../../../examples/bottom-turtle-ha/run-tests.sh" -b "$@"