From f8327657cdc673a3bcd9d08c325ebbe60a9e5d6d Mon Sep 17 00:00:00 2001 From: kfox1111 Date: Wed, 17 Jan 2024 15:09:32 -0800 Subject: [PATCH] Add a test and example for federation support (#169) * SPIFFE OIDC Discovery Provider Rework Fixes: https://github.com/spiffe/helm-charts-hardened/issues/151 Signed-off-by: Kevin Fox * Enhance clusterspiffeid's so the discovery provider is independently configurable Signed-off-by: Kevin Fox * Fix tests Signed-off-by: Kevin Fox * More fix tests Signed-off-by: Kevin Fox * More fix tests Signed-off-by: Kevin Fox * Undo Signed-off-by: Kevin Fox * Fix logging Signed-off-by: Kevin Fox * Try to get output Signed-off-by: Kevin Fox * Try and get error code Signed-off-by: Kevin Fox * Fix more logging. Switch port used. Signed-off-by: Kevin Fox * Fix logging Signed-off-by: Kevin Fox * Fix port Signed-off-by: Kevin Fox * Fix up logs for nested test and fix values Signed-off-by: Kevin Fox * Make consistent Signed-off-by: Kevin Fox * Fix nested test Signed-off-by: Kevin Fox * Fix insecure mode and test. Signed-off-by: Kevin Fox * Fix test. Signed-off-by: Kevin Fox * Fix var scoping issue Signed-off-by: Kevin Fox * Set the right flags for ingress Signed-off-by: Kevin Fox * Update dns template Signed-off-by: Kevin Fox * Use more standard port Signed-off-by: Kevin Fox * Fix test logging Signed-off-by: Kevin Fox * Allow reencrypt. Signed-off-by: Kevin Fox * Remove testing changes Signed-off-by: Kevin Fox * Fix formatting Signed-off-by: Kevin Fox * Add LetsEncrypt/ACME/cert-manager support. Remove broken ACME support. Signed-off-by: Kevin Fox * Use spiffe-helper as a sidecar. Significant space savings and read only cert dir Signed-off-by: Kevin Fox * Fix the nested test Signed-off-by: Kevin Fox * Fix merge issue Signed-off-by: Kevin Fox * Remove 1.29.0 until deps catch up. Related issue: https://github.com/rancher/kubectl/pull/94 Signed-off-by: Kevin Fox * Add more error checking Signed-off-by: Kevin Fox * Remove testing code Signed-off-by: Kevin Fox * Simplify the ids. Fix docs Signed-off-by: Kevin Fox * Fix logic Signed-off-by: Kevin Fox * Fix var Signed-off-by: Kevin Fox * Make cert-manager bits more readable Signed-off-by: Kevin Fox * Fix template Signed-off-by: Kevin Fox * Fix openshift ingress Signed-off-by: Kevin Fox * Incorperate feedback Signed-off-by: Kevin Fox * Update docs Signed-off-by: Kevin Fox * Add resource spec Signed-off-by: Kevin Fox * Remove parts that cant merge yet Signed-off-by: Kevin Fox * Fix tests Signed-off-by: Kevin Fox * Incorperate feedback Signed-off-by: Kevin Fox * Incorperate feedback Signed-off-by: Kevin Fox * Fix test Signed-off-by: Kevin Fox * Apply suggestions from code review Co-authored-by: Faisal Memon Signed-off-by: kfox1111 * Fix docs Signed-off-by: Kevin Fox * Apply suggestions from code review Co-authored-by: Faisal Memon Signed-off-by: kfox1111 * Incorperate feedback Signed-off-by: Kevin Fox * Incorperate feedback Signed-off-by: Kevin Fox * Fix test Signed-off-by: Kevin Fox * Fix test Signed-off-by: Kevin Fox * Apply suggestions from code review Co-authored-by: Faisal Memon Signed-off-by: kfox1111 * Fix docs Signed-off-by: Kevin Fox * Incorperate feedback Signed-off-by: Kevin Fox * Add missing configurable for the discovery providers csi driver Signed-off-by: Kevin Fox * Add a test and example for federation support Signed-off-by: Kevin Fox * Fix test Signed-off-by: Kevin Fox * Fix test Signed-off-by: Kevin Fox * Fix test Signed-off-by: Kevin Fox * Fix ingress annotations for federation bundle endpoint Signed-off-by: Kevin Fox * Add cert-manager support to the federation bundle endpoint and fix up bundle endpoint ingress annotations Signed-off-by: Kevin Fox * Add external secret too Signed-off-by: Kevin Fox * Add forgotten files Signed-off-by: Kevin Fox * Apply suggestions from code review Signed-off-by: kfox1111 --------- Signed-off-by: Kevin Fox Signed-off-by: kfox1111 Co-authored-by: Faisal Memon --- .github/workflows/helm-chart-ci.yaml | 9 +++- examples/federation/a-values.yaml | 51 +++++++++++++++++++ examples/federation/b-values.yaml | 58 +++++++++++++++++++++ examples/federation/client-pod.yaml | 44 ++++++++++++++++ examples/federation/run-tests.sh | 75 ++++++++++++++++++++++++++++ examples/federation/server-pod.yaml | 50 +++++++++++++++++++ examples/federation/server-svc.yaml | 11 ++++ 7 files changed, 296 insertions(+), 2 deletions(-) create mode 100644 examples/federation/a-values.yaml create mode 100644 examples/federation/b-values.yaml create mode 100644 examples/federation/client-pod.yaml create mode 100755 examples/federation/run-tests.sh create mode 100644 examples/federation/server-pod.yaml create mode 100644 examples/federation/server-svc.yaml diff --git a/.github/workflows/helm-chart-ci.yaml b/.github/workflows/helm-chart-ci.yaml index 7e8a5c8..a0d8659 100644 --- a/.github/workflows/helm-chart-ci.yaml +++ b/.github/workflows/helm-chart-ci.yaml @@ -240,8 +240,13 @@ jobs: - name: Install and test example run: | - kubectl create namespace spire-server - helm install -n spire-server spire-crds charts/spire-crds + if [ "${{ matrix.example }}" = "examples/federation" ]; then + kubectl create namespace spire-mgmt + helm install -n spire-mgmt spire-crds charts/spire-crds + else + kubectl create namespace spire-server + helm install -n spire-server spire-crds charts/spire-crds + fi ${{ matrix.example }}/run-tests.sh upgrade-test: diff --git a/examples/federation/a-values.yaml b/examples/federation/a-values.yaml new file mode 100644 index 0000000..6ca171f --- /dev/null +++ b/examples/federation/a-values.yaml @@ -0,0 +1,51 @@ +global: + openshift: false + spire: + recommendations: + enabled: true + namespaces: + create: true + ingressControllerType: ingress-nginx + clusterName: a + trustDomain: a-org.local + bundleConfigMap: spire-bundle-a + +spire-server: + ca_subject: + country: US + organization: A + common_name: a.local + federation: + enabled: true + ingress: + enabled: true + controllerManager: + identities: + clusterSPIFFEIDs: + default: + federatesWith: + - b-org.local + clusterFederatedTrustDomains: + b: + bundleEndpointProfile: + endpointSPIFFEID: spiffe://b-org.local/spire/server + type: https_spiffe + bundleEndpointURL: https://spire-server-federation.b-org.local + trustDomain: b-org.local + +spiffe-oidc-discovery-provider: + enabled: true + tls: + spire: + enabled: false + csiDriverName: a.csi.spiffe.io + ingress: + enabled: true + +spire-agent: + socketPath: /run/spire/agent-sockets-a/spire-agent.sock + +spiffe-csi-driver: + pluginName: a.csi.spiffe.io + agentSocketPath: /run/spire/agent-sockets-a/spire-agent.sock + diff --git a/examples/federation/b-values.yaml b/examples/federation/b-values.yaml new file mode 100644 index 0000000..b76ec6b --- /dev/null +++ b/examples/federation/b-values.yaml @@ -0,0 +1,58 @@ +global: + openshift: false + spire: + recommendations: + enabled: true + namespaces: + create: false + ingressControllerType: ingress-nginx + clusterName: b + trustDomain: b-org.local + bundleConfigMap: spire-bundle-b + +spire-server: + ca_subject: + country: US + organization: B + common_name: b.local + federation: + enabled: true + ingress: + enabled: true + controllerManager: + identities: + clusterSPIFFEIDs: + default: + federatesWith: + - a-org.local + clusterFederatedTrustDomains: + a: + bundleEndpointProfile: + endpointSPIFFEID: spiffe://a-org.local/spire/server + type: https_spiffe + bundleEndpointURL: https://spire-server-federation.a-org.local + trustDomain: a-org.local + +spiffe-oidc-discovery-provider: + enabled: true + tls: + spire: + enabled: false + csiDriverName: b.csi.spiffe.io + ingress: + enabled: true + +spire-agent: + healthChecks: + port: 9983 + telemetry: + prometheus: + port: 9990 + socketPath: /run/spire/agent-sockets-b/spire-agent.sock + +spiffe-csi-driver: + healthChecks: + port: 9811 + pluginName: b.csi.spiffe.io + agentSocketPath: /run/spire/agent-sockets-b/spire-agent.sock + diff --git a/examples/federation/client-pod.yaml b/examples/federation/client-pod.yaml new file mode 100644 index 0000000..dc003f8 --- /dev/null +++ b/examples/federation/client-pod.yaml @@ -0,0 +1,44 @@ +apiVersion: v1 +kind: Pod +metadata: + name: client +spec: + terminationGracePeriodSeconds: 1 + containers: + - name: busybox + image: busybox + command: + - /bin/sh + - -c + - 'while true; do wget -q -O - localhost:8080 && touch /tmp/healthy; sleep 1; done' + readinessProbe: + exec: + command: + - cat + - /tmp/healthy + initialDelaySeconds: 5 + periodSeconds: 5 + - name: ghostunnel + image: ghostunnel/ghostunnel:v1.7.2 + command: + - ghostunnel + - client + - --use-workload-api-addr + - unix:/spire-workload-api/spire-agent.sock + - --listen + - localhost:8080 + - --target + - server:8443 + - --verify-uri + - spiffe://b-org.local/ns/default/sa/default + volumeMounts: + - name: agent + mountPath: /spire-workload-api + readOnly: true + ports: + - containerPort: 8443 + volumes: + - name: agent + csi: + driver: a.csi.spiffe.io + readOnly: true diff --git a/examples/federation/run-tests.sh b/examples/federation/run-tests.sh new file mode 100755 index 0000000..f320e4c --- /dev/null +++ b/examples/federation/run-tests.sh @@ -0,0 +1,75 @@ +#!/usr/bin/env bash + +set -xe + +SCRIPT="$(readlink -f "$0")" +SCRIPTPATH="$(dirname "${SCRIPT}")" +TESTDIR="${SCRIPTPATH}/../../.github/tests" + +# shellcheck source=/dev/null +source "${SCRIPTPATH}/../../.github/scripts/parse-versions.sh" +# shellcheck source=/dev/null +source "${TESTDIR}/common.sh" + +CLEANUP=1 + +for i in "$@"; do + case $i in + -c) + CLEANUP=0 + shift # past argument=value + ;; + esac +done + +teardown() { + print_helm_releases + print_spire_workload_status spire-server spire-system + + if [[ "$1" -ne 0 ]]; then + get_namespace_details spire-server spire-system + fi + + if [ "${CLEANUP}" -eq 1 ]; then + helm uninstall --namespace spire-mgmt spire-b 2>/dev/null || true + helm uninstall --namespace spire-mgmt spire-a 2>/dev/null || true + kubectl delete ns spire-mgmt 2>/dev/null || true + fi +} + +trap 'EC=$? && trap - SIGTERM && teardown $EC' SIGINT SIGTERM EXIT + +helm upgrade --install ingress-nginx ingress-nginx --version "$VERSION_INGRESS_NGINX" --repo "$HELM_REPO_INGRESS_NGINX" \ + --namespace ingress-nginx \ + --create-namespace \ + --set controller.extraArgs.enable-ssl-passthrough=,controller.admissionWebhooks.enabled=false,controller.service.type=ClusterIP \ + --set controller.ingressClassResource.default=true \ + --wait + +IP=$(kubectl get svc -n ingress-nginx ingress-nginx-controller -o yaml | yq e .spec.clusterIPs[0] -) +kubectl get configmap -n kube-system coredns -o yaml | grep hosts || kubectl get configmap -n kube-system coredns -o yaml | sed "/ready/a\ hosts {\n fallthrough\n }" | kubectl apply -f - +kubectl get configmap -n kube-system coredns -o yaml | grep a-org || kubectl get configmap -n kube-system coredns -o yaml | sed "/hosts/a\ $IP spire-server-federation.a-org.local\n $IP spire-server-federation.b-org.local\n" | kubectl apply -f - + +kubectl rollout restart -n kube-system deployment/coredns +kubectl rollout status -w --timeout=1m deploy/coredns -n kube-system + +kubectl create namespace spire-mgmt --dry-run=client -o yaml | kubectl apply -f - +kubectl label namespace spire-mgmt pod-security.kubernetes.io/enforce=restricted || true + +helm upgrade --install --namespace spire-mgmt --values "${SCRIPTPATH}/a-values.yaml" \ + --wait spire-a charts/spire + +helm upgrade --install --namespace spire-mgmt --values "${SCRIPTPATH}/b-values.yaml" \ + --wait spire-b charts/spire + +kubectl exec -it -n spire-server spire-a-server-0 -c spire-server -- spire-server bundle show -format spiffe | kubectl exec -i -n spire-server spire-b-server-0 -c spire-server -- spire-server bundle set -format spiffe -id spiffe://a-org.local +kubectl exec -it -n spire-server spire-b-server-0 -c spire-server -- spire-server bundle show -format spiffe | kubectl exec -i -n spire-server spire-a-server-0 -c spire-server -- spire-server bundle set -format spiffe -id spiffe://b-org.local + +kubectl exec -it -n spire-server spire-b-server-0 -c spire-server -- spire-server bundle list +kubectl exec -it -n spire-server spire-a-server-0 -c spire-server -- spire-server bundle list + +kubectl apply -f "${SCRIPTPATH}/server-svc.yaml" +kubectl apply -f "${SCRIPTPATH}/server-pod.yaml" +kubectl apply -f "${SCRIPTPATH}/client-pod.yaml" + +kubectl wait --for=condition=Ready pod/client --timeout 5m diff --git a/examples/federation/server-pod.yaml b/examples/federation/server-pod.yaml new file mode 100644 index 0000000..db992f4 --- /dev/null +++ b/examples/federation/server-pod.yaml @@ -0,0 +1,50 @@ +apiVersion: v1 +kind: Pod +metadata: + name: server + labels: + app.kubernetes.io/name: server +spec: + containers: + - name: nginx + image: nginx + command: + - /bin/sh + - -c + - | + cat > /etc/nginx/conf.d/default.conf < /usr/share/nginx/html/index.html + exec nginx -g "daemon off;" + - name: ghostunnel + image: ghostunnel/ghostunnel:v1.7.2 + command: + - ghostunnel + - server + - --use-workload-api-addr + - unix:/spire-workload-api/spire-agent.sock + - --listen + - 0.0.0.0:8443 + - --target + - localhost:8080 + - --allow-uri + - spiffe://a-org.local/ns/default/sa/default + volumeMounts: + - name: agent + mountPath: /spire-workload-api + readOnly: true + ports: + - containerPort: 8443 + volumes: + - name: agent + csi: + driver: b.csi.spiffe.io + readOnly: true diff --git a/examples/federation/server-svc.yaml b/examples/federation/server-svc.yaml new file mode 100644 index 0000000..caef658 --- /dev/null +++ b/examples/federation/server-svc.yaml @@ -0,0 +1,11 @@ +apiVersion: v1 +kind: Service +metadata: + name: server +spec: + selector: + app.kubernetes.io/name: server + ports: + - protocol: TCP + port: 8443 + targetPort: 8443