Add broker suport to the spire-ha-agent (#884)
* Add broker suport to the spire-ha-agent Signed-off-by: Kevin Fox <[email protected]> --------- Signed-off-by: Kevin Fox <[email protected]> Signed-off-by: kfox1111 <[email protected]> --------- Signed-off-by: Kevin Fox <[email protected]> Signed-off-by: kfox1111 <[email protected]> Co-authored-by: Faisal Memon <[email protected]>
This commit is contained in:
kfox1111
@@ -28,7 +28,7 @@ concurrency:
|
||||
env:
|
||||
HELM_VERSION: v3.16.2
|
||||
PYTHON_VERSION: 3.11.3
|
||||
KIND_VERSION: v0.24.0
|
||||
KIND_VERSION: v0.32.0
|
||||
CHART_TESTING_VERSION: v3.8.0
|
||||
|
||||
jobs:
|
||||
|
||||
@@ -3,7 +3,7 @@ name: spire-ha-agent
|
||||
description: A Helm chart to install the SPIRE HA agent.
|
||||
type: application
|
||||
version: 0.2.0
|
||||
appVersion: "0.0.17"
|
||||
appVersion: "0.2.0"
|
||||
keywords: ["spiffe", "spire-ha-agent"]
|
||||
home: https://github.com/spiffe/helm-charts-hardened/tree/main/charts/spire-ha-agent
|
||||
sources:
|
||||
|
||||
@@ -30,10 +30,17 @@ A Helm chart to install the SPIRE HA agent.
|
||||
| `image.repository` | The repository within the registry | `spiffe/spire-ha-agent` |
|
||||
| `image.pullPolicy` | The image pull policy | `IfNotPresent` |
|
||||
| `image.tag` | Overrides the image tag whose default is the chart appVersion | `""` |
|
||||
| `mode` | If the spire-ha-agent will run in delegated or broker mode | `delegated` |
|
||||
| `singleSocket` | If in singleSocket mode, only one driver is used | `false` |
|
||||
| `sockets.single.admin.hostPath` | Where the sockets are on disk when in single socket mode | `/var/run/spire/agent/sockets/main/csi.spiffe.io/admin` |
|
||||
| `sockets.a.admin.hostPath` | Where the sockets are on disk | `/var/run/spire/agent/sockets/a/csi.spiffe.io/admin` |
|
||||
| `sockets.b.admin.hostPath` | Where the sockets are on disk | `/var/run/spire/agent/sockets/b/csi.spiffe.io/admin` |
|
||||
| `sockets.single.admin.hostPath` | Where the admin socket is on disk when in single socket mode | `/var/run/spire/agent/sockets/main/csi.spiffe.io/admin` |
|
||||
| `sockets.a.admin.hostPath` | Where the a admin socket is on disk | `/var/run/spire/agent/sockets/a/csi.spiffe.io/admin` |
|
||||
| `sockets.b.admin.hostPath` | Where the b admin sockets is on disk | `/var/run/spire/agent/sockets/b/csi.spiffe.io/admin` |
|
||||
| `sockets.single.broker.hostPath` | Where the broker socket is on disk when in single socket mode | `/var/run/spire/agent/sockets/main/csi.spiffe.io/broker` |
|
||||
| `sockets.a.broker.hostPath` | Where the a broker socket is on disk | `/var/run/spire/agent/sockets/a/csi.spiffe.io/broker` |
|
||||
| `sockets.b.broker.hostPath` | Where the b broker socket is on disk | `/var/run/spire/agent/sockets/b/csi.spiffe.io/broker` |
|
||||
| `sockets.single.workload.hostPath` | Where the broker socket is on disk when in single socket mode | `/var/run/spire/agent-sockets` |
|
||||
| `sockets.a.workload.hostPath` | Where the a workload socket is on disk | `/var/run/spire/agent/sockets/a/csi.spiffe.io/public` |
|
||||
| `sockets.b.workload.hostPath` | Where the b workload socket is on disk | `/var/run/spire/agent/sockets/b/csi.spiffe.io/public` |
|
||||
| `vsock` | Use a vsockets to expose the service rather then a unix socket | `false` |
|
||||
| `port` | Port number to listen on | `999` |
|
||||
| `imagePullSecrets` | Pull secrets for images | `[]` |
|
||||
|
||||
@@ -119,6 +119,9 @@ spec:
|
||||
- name: {{ .Chart.Name | quote }}
|
||||
image: {{ template "spire-lib.image" (dict "appVersion" $.Chart.AppVersion "image" .Values.image "global" .Values.global) }}
|
||||
imagePullPolicy: {{ .Values.image.pullPolicy | quote }}
|
||||
args:
|
||||
- "-mode"
|
||||
- {{ .Values.mode | quote }}
|
||||
securityContext:
|
||||
privileged: true
|
||||
#FIXME read permission to api socket
|
||||
@@ -131,13 +134,31 @@ spec:
|
||||
{{- if .Values.singleSocket }}
|
||||
- name: SPIRE_HA_AGENT_SINGLE
|
||||
value: enabled
|
||||
{{- if eq .Values.mode "delegated" }}
|
||||
- name: SPIRE_HA_AGENT_SOCKET
|
||||
value: unix:///var/run/spire/agent/sockets/a/csi.spiffe.io/admin/admin.sock
|
||||
{{- else }}
|
||||
- name: SPIRE_HA_AGENT_BROKER
|
||||
value: unix:///var/run/spire/agent/sockets/a/csi.spiffe.io/broker/broker.sock
|
||||
- name: SPIRE_HA_AGENT_WORKLOAD_SOCKET
|
||||
value: unix:///var/run/spire/agent/sockets/a/csi.spiffe.io/public/spire-agent.sock
|
||||
{{- end }}
|
||||
{{- else }}
|
||||
{{- if eq .Values.mode "delegated" }}
|
||||
- name: SPIRE_HA_AGENT_SOCKET_A
|
||||
value: unix:///var/run/spire/agent/sockets/a/csi.spiffe.io/admin/admin.sock
|
||||
- name: SPIRE_HA_AGENT_SOCKET_B
|
||||
value: unix:///var/run/spire/agent/sockets/b/csi.spiffe.io/admin/admin.sock
|
||||
{{- else }}
|
||||
- name: SPIRE_HA_AGENT_BROKER_A
|
||||
value: unix:///var/run/spire/agent/sockets/a/csi.spiffe.io/broker/broker.sock
|
||||
- name: SPIRE_HA_AGENT_BROKER_B
|
||||
value: unix:///var/run/spire/agent/sockets/b/csi.spiffe.io/broker/broker.sock
|
||||
- name: SPIRE_HA_AGENT_WORKLOAD_SOCKET_A
|
||||
value: unix:///var/run/spire/agent/sockets/a/csi.spiffe.io/public/spire-agent.sock
|
||||
- name: SPIRE_HA_AGENT_WORKLOAD_SOCKET_B
|
||||
value: unix:///var/run/spire/agent/sockets/b/csi.spiffe.io/public/spire-agent.sock
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- if .Values.vsock }}
|
||||
- name: SPIRE_HA_AGENT_VSOCK
|
||||
@@ -161,12 +182,25 @@ spec:
|
||||
mountPath: /tmp/spire-ha-agent/public
|
||||
readOnly: false
|
||||
{{- end }}
|
||||
{{- if eq .Values.mode "delegated" }}
|
||||
- name: spire-ha-admin-socket-dir-upstream-a
|
||||
mountPath: /var/run/spire/agent/sockets/a/csi.spiffe.io/admin
|
||||
{{- if not .Values.singleSocket }}
|
||||
- name: spire-ha-admin-socket-dir-upstream-b
|
||||
mountPath: /var/run/spire/agent/sockets/b/csi.spiffe.io/admin
|
||||
{{- end }}
|
||||
{{- else }}
|
||||
- name: spire-ha-broker-socket-dir-upstream-a
|
||||
mountPath: /var/run/spire/agent/sockets/a/csi.spiffe.io/broker
|
||||
- name: spire-ha-workload-socket-dir-upstream-a
|
||||
mountPath: /var/run/spire/agent/sockets/a/csi.spiffe.io/public
|
||||
{{- if not .Values.singleSocket }}
|
||||
- name: spire-ha-broker-socket-dir-upstream-b
|
||||
mountPath: /var/run/spire/agent/sockets/b/csi.spiffe.io/broker
|
||||
- name: spire-ha-workload-socket-dir-upstream-b
|
||||
mountPath: /var/run/spire/agent/sockets/b/csi.spiffe.io/public
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
- name: dev
|
||||
mountPath: /dev
|
||||
{{- if gt (len .Values.extraVolumeMounts) 0 }}
|
||||
@@ -196,6 +230,7 @@ spec:
|
||||
path: {{ $socketPath | dir }}
|
||||
type: DirectoryOrCreate
|
||||
{{- end }}
|
||||
{{- if eq .Values.mode "delegated" }}
|
||||
- name: spire-ha-admin-socket-dir-upstream-a
|
||||
hostPath:
|
||||
path: {{ if .Values.singleSocket }}{{ .Values.sockets.single.admin.hostPath }}{{ else }}{{ .Values.sockets.a.admin.hostPath }}{{ end }}
|
||||
@@ -206,6 +241,26 @@ spec:
|
||||
path: {{ .Values.sockets.b.admin.hostPath }}
|
||||
type: DirectoryOrCreate
|
||||
{{- end }}
|
||||
{{- else }}
|
||||
- name: spire-ha-broker-socket-dir-upstream-a
|
||||
hostPath:
|
||||
path: {{ if .Values.singleSocket }}{{ .Values.sockets.single.broker.hostPath }}{{ else }}{{ .Values.sockets.a.broker.hostPath }}{{ end }}
|
||||
type: DirectoryOrCreate
|
||||
- name: spire-ha-workload-socket-dir-upstream-a
|
||||
hostPath:
|
||||
path: {{ if .Values.singleSocket }}{{ .Values.sockets.single.workload.hostPath }}{{ else }}{{ .Values.sockets.a.workload.hostPath }}{{ end }}
|
||||
type: DirectoryOrCreate
|
||||
{{- if not .Values.singleSocket }}
|
||||
- name: spire-ha-broker-socket-dir-upstream-b
|
||||
hostPath:
|
||||
path: {{ .Values.sockets.b.broker.hostPath }}
|
||||
type: DirectoryOrCreate
|
||||
- name: spire-ha-workload-socket-dir-upstream-b
|
||||
hostPath:
|
||||
path: {{ .Values.sockets.b.workload.hostPath }}
|
||||
type: DirectoryOrCreate
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
- name: dev
|
||||
hostPath:
|
||||
path: /dev
|
||||
|
||||
@@ -17,22 +17,43 @@ image:
|
||||
pullPolicy: IfNotPresent
|
||||
tag: ""
|
||||
|
||||
## @param mode If the spire-ha-agent will run in delegated or broker mode
|
||||
mode: delegated
|
||||
|
||||
## @param singleSocket If in singleSocket mode, only one driver is used
|
||||
singleSocket: false
|
||||
|
||||
## @param sockets.single.admin.hostPath Where the sockets are on disk when in single socket mode
|
||||
## @param sockets.a.admin.hostPath Where the sockets are on disk
|
||||
## @param sockets.b.admin.hostPath Where the sockets are on disk
|
||||
## @param sockets.single.admin.hostPath Where the admin socket is on disk when in single socket mode
|
||||
## @param sockets.a.admin.hostPath Where the a admin socket is on disk
|
||||
## @param sockets.b.admin.hostPath Where the b admin sockets is on disk
|
||||
## @param sockets.single.broker.hostPath Where the broker socket is on disk when in single socket mode
|
||||
## @param sockets.a.broker.hostPath Where the a broker socket is on disk
|
||||
## @param sockets.b.broker.hostPath Where the b broker socket is on disk
|
||||
## @param sockets.single.workload.hostPath Where the broker socket is on disk when in single socket mode
|
||||
## @param sockets.a.workload.hostPath Where the a workload socket is on disk
|
||||
## @param sockets.b.workload.hostPath Where the b workload socket is on disk
|
||||
sockets:
|
||||
single:
|
||||
admin:
|
||||
hostPath: /var/run/spire/agent/sockets/main/csi.spiffe.io/admin
|
||||
broker:
|
||||
hostPath: /var/run/spire/agent/sockets/main/csi.spiffe.io/broker
|
||||
workload:
|
||||
hostPath: /var/run/spire/agent-sockets
|
||||
a:
|
||||
admin:
|
||||
hostPath: /var/run/spire/agent/sockets/a/csi.spiffe.io/admin
|
||||
broker:
|
||||
hostPath: /var/run/spire/agent/sockets/a/csi.spiffe.io/broker
|
||||
workload:
|
||||
hostPath: /var/run/spire/agent/sockets/a/csi.spiffe.io/public
|
||||
b:
|
||||
admin:
|
||||
hostPath: /var/run/spire/agent/sockets/b/csi.spiffe.io/admin
|
||||
broker:
|
||||
hostPath: /var/run/spire/agent/sockets/b/csi.spiffe.io/broker
|
||||
workload:
|
||||
hostPath: /var/run/spire/agent/sockets/b/csi.spiffe.io/public
|
||||
|
||||
## @param vsock Use a vsockets to expose the service rather then a unix socket
|
||||
vsock: false
|
||||
|
||||
@@ -466,6 +466,9 @@ Now you can interact with the Spire agent socket from your own application. The
|
||||
| `downstream-spire-agent-bottom-turtle-ha-a.sockets.admin.enabled` | Enable admin socket | `true` |
|
||||
| `downstream-spire-agent-bottom-turtle-ha-a.sockets.admin.mountOnHost` | Mount admin socket on host | `true` |
|
||||
| `downstream-spire-agent-bottom-turtle-ha-a.authorizedDelegates` | List of workloads able to use the delegation api | `["/spire-ha-agent"]` |
|
||||
| `downstream-spire-agent-bottom-turtle-ha-a.brokerAPI.brokers.spire-ha-agent.enabled` | Enable the spire-ha-agent by default | `true` |
|
||||
| `downstream-spire-agent-bottom-turtle-ha-a.workloadAttestors.k8s.brokerAPI.accessPolicy` | The default accessPolicy | `permissive` |
|
||||
| `downstream-spire-agent-bottom-turtle-ha-a.workloadAttestors.k8s.brokerAPI.brokers.spire-ha-agent.enabled` | Enable the spire-ha-agent by default | `true` |
|
||||
| `downstream-spire-agent-bottom-turtle-ha-b.nameOverride` | Overrides the name of Spire agent pods | `agent-downstream` |
|
||||
| `downstream-spire-agent-bottom-turtle-ha-b.server.nameOverride` | The name override setting of the internal SPIRE server | `internal-server` |
|
||||
| `downstream-spire-agent-bottom-turtle-ha-b.bundleConfigMap` | The name of the configmap that contains the downstream bundle | `spire-server-b-bundle` |
|
||||
@@ -485,6 +488,9 @@ Now you can interact with the Spire agent socket from your own application. The
|
||||
| `downstream-spire-agent-bottom-turtle-ha-b.sockets.admin.enabled` | Enable admin socket | `true` |
|
||||
| `downstream-spire-agent-bottom-turtle-ha-b.sockets.admin.mountOnHost` | Mount admin socket on host | `true` |
|
||||
| `downstream-spire-agent-bottom-turtle-ha-b.authorizedDelegates` | List of workloads able to use the delegation api | `["/spire-ha-agent"]` |
|
||||
| `downstream-spire-agent-bottom-turtle-ha-b.brokerAPI.brokers.spire-ha-agent.enabled` | Enable the spire-ha-agent by default | `true` |
|
||||
| `downstream-spire-agent-bottom-turtle-ha-b.workloadAttestors.k8s.brokerAPI.accessPolicy` | The default accessPolicy | `permissive` |
|
||||
| `downstream-spire-agent-bottom-turtle-ha-b.workloadAttestors.k8s.brokerAPI.brokers.spire-ha-agent.enabled` | Enable the spire-ha-agent by default | `true` |
|
||||
| `downstream-spiffe-csi-driver-bottom-turtle-ha-a.fullnameOverride` | Fullname override | `spiffe-csi-driver-downstream-a` |
|
||||
| `downstream-spiffe-csi-driver-bottom-turtle-ha-a.agentSocketPath` | path to agent socket | `/var/run/spire/agent/sockets/a/csi.spiffe.io/public/spire-agent.sock` |
|
||||
| `downstream-spiffe-csi-driver-bottom-turtle-ha-a.pluginName` | The name of the plugin instance | `a.csi.spiffe.io` |
|
||||
|
||||
@@ -680,6 +680,20 @@ downstream-spire-agent-bottom-turtle-ha-a:
|
||||
## @param downstream-spire-agent-bottom-turtle-ha-a.authorizedDelegates List of workloads able to use the delegation api
|
||||
authorizedDelegates:
|
||||
- /spire-ha-agent
|
||||
brokerAPI:
|
||||
brokers:
|
||||
spire-ha-agent:
|
||||
## @param downstream-spire-agent-bottom-turtle-ha-a.brokerAPI.brokers.spire-ha-agent.enabled Enable the spire-ha-agent by default
|
||||
enabled: true
|
||||
workloadAttestors:
|
||||
k8s:
|
||||
brokerAPI:
|
||||
## @param downstream-spire-agent-bottom-turtle-ha-a.workloadAttestors.k8s.brokerAPI.accessPolicy The default accessPolicy
|
||||
accessPolicy: permissive
|
||||
brokers:
|
||||
spire-ha-agent:
|
||||
## @param downstream-spire-agent-bottom-turtle-ha-a.workloadAttestors.k8s.brokerAPI.brokers.spire-ha-agent.enabled Enable the spire-ha-agent by default
|
||||
enabled: true
|
||||
|
||||
# Used with tags [bottomTurtleHAB]
|
||||
downstream-spire-agent-bottom-turtle-ha-b:
|
||||
@@ -737,6 +751,20 @@ downstream-spire-agent-bottom-turtle-ha-b:
|
||||
## @param downstream-spire-agent-bottom-turtle-ha-b.authorizedDelegates List of workloads able to use the delegation api
|
||||
authorizedDelegates:
|
||||
- /spire-ha-agent
|
||||
brokerAPI:
|
||||
brokers:
|
||||
spire-ha-agent:
|
||||
## @param downstream-spire-agent-bottom-turtle-ha-b.brokerAPI.brokers.spire-ha-agent.enabled Enable the spire-ha-agent by default
|
||||
enabled: true
|
||||
workloadAttestors:
|
||||
k8s:
|
||||
brokerAPI:
|
||||
## @param downstream-spire-agent-bottom-turtle-ha-b.workloadAttestors.k8s.brokerAPI.accessPolicy The default accessPolicy
|
||||
accessPolicy: permissive
|
||||
brokers:
|
||||
spire-ha-agent:
|
||||
## @param downstream-spire-agent-bottom-turtle-ha-b.workloadAttestors.k8s.brokerAPI.brokers.spire-ha-agent.enabled Enable the spire-ha-agent by default
|
||||
enabled: true
|
||||
|
||||
# Used with tags [bottomTurtleHAA]
|
||||
downstream-spiffe-csi-driver-bottom-turtle-ha-a:
|
||||
|
||||
@@ -0,0 +1,74 @@
|
||||
apiVersion: batch/v1
|
||||
kind: Job
|
||||
metadata:
|
||||
name: federation-test
|
||||
spec:
|
||||
backoffLimit: 0
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: federation-test
|
||||
spec:
|
||||
restartPolicy: Never
|
||||
initContainers:
|
||||
- name: static-busybox
|
||||
# Replaced by run-tests.sh with the image from the spiffe-oidc-discovery-provider chart
|
||||
image: replaced-by-run-tests.sh
|
||||
command:
|
||||
- sh
|
||||
- -c
|
||||
- |
|
||||
cp /bin/busybox /data/busybox
|
||||
chmod +x /data/busybox
|
||||
volumeMounts:
|
||||
- name: data-volume
|
||||
mountPath: /data
|
||||
containers:
|
||||
- name: main
|
||||
# Replaced by run-tests.sh with the image from the spire-agent chart
|
||||
image: replaced-by-run-tests.sh
|
||||
command:
|
||||
- /data/busybox
|
||||
- sh
|
||||
- -xc
|
||||
- |
|
||||
SOCK=/spire-agent/spire-agent.sock
|
||||
i=0
|
||||
while [ "$i" -lt 60 ]; do
|
||||
if XOUT=$(/opt/spire/bin/spire-agent api fetch x509 -socketPath "$SOCK" -write /data -timeout 5s 2>&1) &&
|
||||
echo "$XOUT" | /data/busybox grep -q "for trust domain other.org" &&
|
||||
JOUT=$(/opt/spire/bin/spire-agent api fetch jwt -audience test -socketPath "$SOCK" -timeout 5s 2>&1) &&
|
||||
echo "$JOUT" | /data/busybox grep -q "bundle(other.org)"; then
|
||||
# The other.org bundle was statically set to the same single CA on both sides,
|
||||
# so every federated bundle delivered must contain exactly one certificate.
|
||||
for f in /data/federated_bundle.*.pem; do
|
||||
COUNT=$(/data/busybox grep -c "BEGIN CERTIFICATE" "$f")
|
||||
if [ "$COUNT" != "1" ]; then
|
||||
echo "expected 1 CA in $f, got $COUNT"
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
echo "$XOUT"
|
||||
echo "$JOUT"
|
||||
echo FEDERATION-OK
|
||||
exit 0
|
||||
fi
|
||||
i=$((i+1))
|
||||
/data/busybox sleep 2
|
||||
done
|
||||
echo "$XOUT"
|
||||
echo "$JOUT"
|
||||
exit 1
|
||||
volumeMounts:
|
||||
- name: data-volume
|
||||
mountPath: /data
|
||||
- name: spire-api
|
||||
mountPath: /spire-agent
|
||||
readOnly: true
|
||||
volumes:
|
||||
- name: data-volume
|
||||
emptyDir: {}
|
||||
- name: spire-api
|
||||
csi:
|
||||
driver: csi.spiffe.io
|
||||
readOnly: true
|
||||
@@ -14,6 +14,7 @@ source "${SCRIPTPATH}/../../.github/scripts/parse-versions.sh"
|
||||
source "${TESTDIR}/common.sh"
|
||||
|
||||
CLEANUP=1
|
||||
BROKER=0
|
||||
|
||||
for i in "$@"; do
|
||||
case $i in
|
||||
@@ -21,9 +22,38 @@ for i in "$@"; do
|
||||
CLEANUP=0
|
||||
shift # past argument=value
|
||||
;;
|
||||
-b)
|
||||
BROKER=1
|
||||
shift # past argument=value
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
# With -b, test the spire-ha-agent broker api instead of the delegated api.
|
||||
# Broker mode also supports federated trust bundles, so federate the ha-agent's own entry and a
|
||||
# dedicated federation-test workload entry with the other.org trust domain on both sides. Delegated
|
||||
# mode only tolerates the local and spire-ha bundles, so none of this may apply without -b.
|
||||
BROKER_MODE_ARGS=()
|
||||
BROKER_SOCKET_ARGS_A=()
|
||||
BROKER_SOCKET_ARGS_B=()
|
||||
if [ "${BROKER}" -eq 1 ]; then
|
||||
BROKER_MODE_ARGS=(--set "spire-ha-agent.mode=broker")
|
||||
BROKER_SOCKET_ARGS_A=(
|
||||
--set downstream-spire-agent-bottom-turtle-ha-a.sockets.broker.enabled=true
|
||||
--set downstream-spire-agent-bottom-turtle-ha-a.sockets.broker.mountOnHost=true
|
||||
--set 'internal-spire-server-bottom-turtle-ha-a.controllerManager.identities.clusterSPIFFEIDs.spire-ha-agent.federatesWith={spire-ha,other.org}'
|
||||
--set 'internal-spire-server-bottom-turtle-ha-a.controllerManager.identities.clusterSPIFFEIDs.federation-test.federatesWith={other.org}'
|
||||
--set 'internal-spire-server-bottom-turtle-ha-a.controllerManager.identities.clusterSPIFFEIDs.federation-test.podSelector.matchLabels.app=federation-test'
|
||||
)
|
||||
BROKER_SOCKET_ARGS_B=(
|
||||
--set downstream-spire-agent-bottom-turtle-ha-b.sockets.broker.enabled=true
|
||||
--set downstream-spire-agent-bottom-turtle-ha-b.sockets.broker.mountOnHost=true
|
||||
--set 'internal-spire-server-bottom-turtle-ha-b.controllerManager.identities.clusterSPIFFEIDs.spire-ha-agent.federatesWith={spire-ha,other.org}'
|
||||
--set 'internal-spire-server-bottom-turtle-ha-b.controllerManager.identities.clusterSPIFFEIDs.federation-test.federatesWith={other.org}'
|
||||
--set 'internal-spire-server-bottom-turtle-ha-b.controllerManager.identities.clusterSPIFFEIDs.federation-test.podSelector.matchLabels.app=federation-test'
|
||||
)
|
||||
fi
|
||||
|
||||
if [ "x${GITHUB_JOB}" != "x" ]; then
|
||||
echo "Running in GitHub"
|
||||
else
|
||||
@@ -36,6 +66,9 @@ teardown() {
|
||||
docker exec -i chart-testing-worker /bin/bash -c "more /var/lib/kubelet/pods/*/volumes/kubernetes.io~empty-dir/disk-keymanager/keys.json /var/lib/kubelet/pods/*/volumes/kubernetes.io~empty-dir/spire-agent-persistence/agent-data.json | cat"
|
||||
sudo systemctl status spire-server@a || true
|
||||
sudo systemctl status spire-server@b || true
|
||||
sudo systemctl status spire-server@other || true
|
||||
kubectl describe job federation-test || true
|
||||
kubectl logs job/federation-test || true
|
||||
sudo spire-server entry show -instance a || true
|
||||
sudo spire-server entry show -instance b || true
|
||||
sudo systemctl status spire-controller-manager@a || true
|
||||
@@ -59,6 +92,9 @@ teardown() {
|
||||
kubectl exec -i -n spire-server spire-a-internal-server-0 -- spire-server agent list -output json | yq e . - -P || true
|
||||
kubectl exec -i -n spire-server spire-b-internal-server-0 -- spire-server agent list -output json | yq e . - -P || true
|
||||
kubectl get pods -A -o wide || true
|
||||
kubectl describe daemonset pods -n spire-system || true
|
||||
kubectl get configmap -n spire-system || true
|
||||
kubectl get configmap -n spire-system spire-a-agent-downstream -o yaml || true
|
||||
|
||||
print_helm_releases
|
||||
|
||||
@@ -68,6 +104,7 @@ teardown() {
|
||||
fi
|
||||
|
||||
if [ "${CLEANUP}" -eq 1 ]; then
|
||||
kubectl delete job federation-test 2>/dev/null || true
|
||||
helm uninstall --namespace spire-mgmt spire-b 2>/dev/null || true
|
||||
helm uninstall --namespace spire-mgmt spire-a 2>/dev/null || true
|
||||
helm uninstall --namespace spire-mgmt spire 2>/dev/null || true
|
||||
@@ -127,6 +164,30 @@ wait_for_jwt() {
|
||||
return 1
|
||||
}
|
||||
|
||||
wait_for_entry_federation() {
|
||||
local pod="$1"
|
||||
local trustdomain="$2"
|
||||
local timeout=60
|
||||
local count=0
|
||||
while [ "$count" -lt "$timeout" ]; do
|
||||
if kubectl exec -i -n spire-server "$pod" -- spire-server entry show -spiffeID spiffe://production.other/spire-ha-agent | grep -q "$trustdomain"; then
|
||||
return 0
|
||||
fi
|
||||
sleep 2
|
||||
((count++)) || true
|
||||
done
|
||||
return 1
|
||||
}
|
||||
|
||||
run_federation_test_job() {
|
||||
kubectl delete job federation-test 2>/dev/null || true
|
||||
# Inject the images from the charts into the job so they always sync up
|
||||
yq e "(.spec.template.spec.initContainers[] | select(.name == \"static-busybox\") | .image) = \"${BUSYBOX_IMAGE}\" | (.spec.template.spec.containers[] | select(.name == \"main\") | .image) = \"${AGENT_IMAGE}\"" \
|
||||
"${SCRIPTPATH}/federation-test-job.yaml" | kubectl apply -f -
|
||||
kubectl wait --for=condition=complete --timeout=240s job/federation-test
|
||||
kubectl logs job/federation-test | grep FEDERATION-OK
|
||||
}
|
||||
|
||||
"${SCRIPTPATH}/../../.github/scripts/prepare-local-chart-deps.sh"
|
||||
|
||||
# Get the package repo and install the packages
|
||||
@@ -137,6 +198,25 @@ sudo apt-get install -y spire-common spire-agent spire-server spire-controller-m
|
||||
# Set our testing trust domain
|
||||
sudo sed -i 's/example.org/production.other/' /etc/spiffe/default-trust-domain.env
|
||||
|
||||
if [ "${BROKER}" -eq 1 ]; then
|
||||
# Pull the federation test job images out of the charts so they always sync up.
|
||||
AGENT_IMAGE=$(helm template t charts/spire -s charts/spire-agent/templates/daemonset.yaml --values "${COMMON_TEST_YOUR_VALUES}" --set spire-agent.enabled=true | yq e 'select(.kind=="DaemonSet") | .spec.template.spec.containers[] | select(.name=="spire-agent") | .image' -)
|
||||
BUSYBOX_IMAGE=$(helm template t charts/spire -s charts/spiffe-oidc-discovery-provider/templates/tests/test-keys.yaml --values "${COMMON_TEST_YOUR_VALUES}" --set spiffe-oidc-discovery-provider.enabled=true | yq e 'select(.kind=="Pod") | .spec.initContainers[] | select(.name=="static-busybox") | .image' -)
|
||||
echo "federation test job images: ${AGENT_IMAGE} ${BUSYBOX_IMAGE}"
|
||||
|
||||
# Mint a trust bundle for a foreign trust domain (other.org) to test federated trust bundle
|
||||
# support. A throwaway third spire-server instance produces a genuine spiffe format bundle
|
||||
# carrying both x509 and jwt authorities. The instance env file overrides the global trust
|
||||
# domain since systemd applies later EnvironmentFiles last.
|
||||
sudo /bin/bash -c '(echo SPIFFE_TRUST_DOMAIN=other.org; echo SPIRE_BIND_PORT=8083) > /etc/spire/server/other.env'
|
||||
sudo systemctl start spire-server@other
|
||||
wait_for_healthcheck spire-server /run/spire/server/sockets/other/private/api.sock
|
||||
sudo spire-server bundle show -format spiffe -socketPath /run/spire/server/sockets/other/private/api.sock | sudo tee /tmp/other-org-bundle.json > /dev/null
|
||||
sudo systemctl stop spire-server@other
|
||||
grep -q '"x509-svid"' /tmp/other-org-bundle.json
|
||||
grep -q '"jwt-svid"' /tmp/other-org-bundle.json
|
||||
fi
|
||||
|
||||
# register some workloads with the spire server using manifests
|
||||
sudo mkdir -p /etc/spire/server/a/manifests/ /etc/spire/server/b/manifests/
|
||||
sudo cp "${SCRIPTPATH}/example-manifests"/* /etc/spire/server/a/manifests/
|
||||
@@ -235,7 +315,8 @@ helm upgrade --install --create-namespace --namespace spire-mgmt --values "${COM
|
||||
--set tags.haAgentCommon=true \
|
||||
--set "global.spire.namespaces.create=true" \
|
||||
--set "global.spire.ingressControllerType=ingress-nginx" \
|
||||
--set "spiffe-oidc-discovery-provider.ingress.enabled=true"
|
||||
--set "spiffe-oidc-discovery-provider.ingress.enabled=true" \
|
||||
"${BROKER_MODE_ARGS[@]}"
|
||||
|
||||
# Create spire-identity-exchange cert for testing.
|
||||
mkdir -p certs
|
||||
@@ -253,14 +334,21 @@ helm upgrade --install --namespace spire-mgmt --values "${COMMON_TEST_YOUR_VALUE
|
||||
--set tags.bottomTurtleHAA=true \
|
||||
--values "${SCRIPTPATH}/spire-identity-exchange-values.yaml" \
|
||||
--set "spire-identity-exchange-bottom-turtle-ha-a.enabled=true" \
|
||||
--set "global.spire.ingressControllerType=ingress-nginx"
|
||||
--set "global.spire.ingressControllerType=ingress-nginx" \
|
||||
"${BROKER_SOCKET_ARGS_A[@]}"
|
||||
|
||||
if [ "${BROKER}" -eq 1 ]; then
|
||||
# Install the other.org bundle so the controller manager can create the entries that federate
|
||||
# with it. It retries any entries that failed with "unable to find federated bundle".
|
||||
kubectl exec -i -n spire-server spire-a-internal-server-0 -- spire-server bundle set -format spiffe -id spiffe://other.org < /tmp/other-org-bundle.json
|
||||
fi
|
||||
|
||||
docker exec -i chart-testing-worker /bin/bash -c "more /var/lib/kubelet/pods/*/volumes/kubernetes.io~empty-dir/disk-keymanager/keys.json /var/lib/kubelet/pods/*/volumes/kubernetes.io~empty-dir/spire-agent-persistence/agent-data.json | cat"
|
||||
|
||||
# Rollout just to sped up the tests
|
||||
kubectl patch deployment spiffe-oidc-discovery-provider -n spire-server --type='strategic' -p '{"spec": {"strategy": {"type": "Recreate", "rollingUpdate": null}}}'
|
||||
kubectl rollout restart daemonset -n spire-system spire-ha-agent
|
||||
kubectl rollout status daemonset -n spire-system spire-ha-agent
|
||||
kubectl rollout status daemonset -n spire-system spire-ha-agent --timeout=1m
|
||||
kubectl rollout restart deployment -n spire-server spiffe-oidc-discovery-provider
|
||||
kubectl rollout status deployment -n spire-server spiffe-oidc-discovery-provider --timeout=1m
|
||||
kubectl wait -n spire-server --for=condition=ready pod -l "app.kubernetes.io/name=spiffe-oidc-discovery-provider" --field-selector=status.phase=Running --timeout=90s
|
||||
@@ -273,7 +361,15 @@ helm upgrade --install --namespace spire-mgmt --values "${COMMON_TEST_YOUR_VALUE
|
||||
--set internal-spire-server-bottom-turtle-ha-b.upstreamAuthority.spire.server.port=8082 \
|
||||
--values "${SCRIPTPATH}/spire-identity-exchange-values.yaml" \
|
||||
--set "spire-identity-exchange-bottom-turtle-ha-b.enabled=true" \
|
||||
--set "global.spire.ingressControllerType=ingress-nginx"
|
||||
--set "global.spire.ingressControllerType=ingress-nginx" \
|
||||
"${BROKER_SOCKET_ARGS_B[@]}"
|
||||
|
||||
if [ "${BROKER}" -eq 1 ]; then
|
||||
kubectl exec -i -n spire-server spire-b-internal-server-0 -- spire-server bundle set -format spiffe -id spiffe://other.org < /tmp/other-org-bundle.json
|
||||
# Both sides' spire-ha-agent entries must federate with other.org before the workload test.
|
||||
wait_for_entry_federation spire-a-internal-server-0 other.org
|
||||
wait_for_entry_federation spire-b-internal-server-0 other.org
|
||||
fi
|
||||
|
||||
docker ps
|
||||
docker exec -i chart-testing-worker /bin/bash -c "more /var/lib/kubelet/pods/*/volumes/kubernetes.io~empty-dir/disk-keymanager/keys.json /var/lib/kubelet/pods/*/volumes/kubernetes.io~empty-dir/spire-agent-persistence/agent-data.json | cat"
|
||||
@@ -305,11 +401,22 @@ TOKEN=$(kubectl logs job/test)
|
||||
curl -f -H "Authorization: Bearer ${TOKEN}" -X POST --resolve "spire-identity-exchange-a-rest.production.other:443:$IP" "https://spire-identity-exchange-a-rest.production.other/api/v1/svid/k8s_psat/x509" -k -sS -q
|
||||
curl -f -H "Authorization: Bearer ${TOKEN}" -X POST --resolve "spire-identity-exchange-b-rest.production.other:443:$IP" "https://spire-identity-exchange-b-rest.production.other/api/v1/svid/k8s_psat/x509" -k -sS -q
|
||||
|
||||
if [ "${BROKER}" -eq 1 ]; then
|
||||
# Verify a workload on the ha-agent socket receives the other.org federated trust bundles,
|
||||
# x509 and jwt, merged from both sides.
|
||||
run_federation_test_job
|
||||
fi
|
||||
|
||||
#Test out running only on side b since we know already only both servers work together, and that only side a works if we made it this far.
|
||||
helm delete -n spire-mgmt spire-a
|
||||
kubectl rollout restart daemonset -n spire-system spire-ha-agent
|
||||
kubectl rollout status daemonset -n spire-system spire-ha-agent
|
||||
kubectl rollout restart deployment -n spire-server spiffe-oidc-discovery-provider
|
||||
kubectl rollout status deployment -n spire-server spiffe-oidc-discovery-provider --timeout=1m
|
||||
kubectl rollout status deployment -n spire-server spiffe-oidc-discovery-provider --timeout=5m
|
||||
curl -k --resolve "oidc-discovery.production.other:443:$IP" "https://oidc-discovery.production.other/.well-known/openid-configuration" -s --fail
|
||||
|
||||
if [ "${BROKER}" -eq 1 ]; then
|
||||
# Verify the other.org federated trust bundles still serve with only side b running.
|
||||
run_federation_test_job
|
||||
fi
|
||||
|
||||
|
||||
@@ -0,0 +1,9 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
set -xe
|
||||
|
||||
SCRIPT="$(readlink -f "$0")"
|
||||
SCRIPTPATH="$(dirname "${SCRIPT}")"
|
||||
|
||||
# Run the bottom-turtle-ha example test with the spire-ha-agent broker api enabled.
|
||||
exec "${SCRIPTPATH}/../../../examples/bottom-turtle-ha/run-tests.sh" -b "$@"
|
||||
Reference in New Issue
Block a user