From ef610df51f1813922994096b8cfe3c9dfb58df4e Mon Sep 17 00:00:00 2001 From: kfox1111 Date: Wed, 13 Mar 2024 13:36:19 -0700 Subject: [PATCH] Complete Server K8S PSAT support (#242) * Complete Server K8S PSAT support Add all the SPIRE Server supported options for the K8S PSAT attestor. This retains the ease of use for configuring local cluster support while adding the ability to configure multiple/external clusters as well. Kubeconfig support is added in its own config block as it will be used/shared with spire-controller-manager support in the future. Signed-off-by: Kevin Fox * Fix merge conflict Signed-off-by: Kevin Fox * Add support for integration tests in the tests/integration dir Signed-off-by: Kevin Fox * Fix split issue and typo Signed-off-by: Kevin Fox * Add basic psat test Signed-off-by: Kevin Fox * Fix linter Signed-off-by: Kevin Fox * Fix up test Signed-off-by: Kevin Fox * Add missing file Signed-off-by: Kevin Fox * Better encode config Signed-off-by: Kevin Fox * Update charts/spire/charts/spire-server/values.yaml Co-authored-by: Faisal Memon Signed-off-by: kfox1111 * Update docs Signed-off-by: Kevin Fox * Apply suggestions from code review Co-authored-by: Faisal Memon Signed-off-by: kfox1111 * Fix docs Signed-off-by: Kevin Fox * Update charts/spire/charts/spire-server/values.yaml Co-authored-by: Faisal Memon Signed-off-by: kfox1111 * Fix docs Signed-off-by: Kevin Fox * Update default Signed-off-by: Kevin Fox * Fix config file layout. Incorperate feedback. Signed-off-by: Kevin Fox * Incorperate feedback Signed-off-by: Kevin Fox * Fix up kind Signed-off-by: Kevin Fox --------- Signed-off-by: Kevin Fox Signed-off-by: kfox1111 Co-authored-by: Faisal Memon --- charts/spire/charts/spire-server/README.md | 10 +++ .../spire-server/templates/configmap.yaml | 40 +++++++++-- .../templates/kubeconfig-secret.yaml | 19 ++++++ .../spire-server/templates/statefulset.yaml | 10 +++ charts/spire/charts/spire-server/values.yaml | 33 ++++++++++ tests/integration/psat/kind-config.yaml | 7 ++ tests/integration/psat/run-tests.sh | 66 +++++++++++++++++++ tests/integration/psat/values.yaml | 12 ++++ 8 files changed, 192 insertions(+), 5 deletions(-) create mode 100644 charts/spire/charts/spire-server/templates/kubeconfig-secret.yaml create mode 100644 tests/integration/psat/kind-config.yaml create mode 100755 tests/integration/psat/run-tests.sh create mode 100644 tests/integration/psat/values.yaml diff --git a/charts/spire/charts/spire-server/README.md b/charts/spire/charts/spire-server/README.md index b233852..c04ec19 100644 --- a/charts/spire/charts/spire-server/README.md +++ b/charts/spire/charts/spire-server/README.md @@ -296,6 +296,15 @@ In order to run Tornjak with simple HTTP Connection only, make sure you don't cr | `defaultJwtSvidTTL` | TTL for JWT Svids | `1h` | | `nodeAttestor.k8sPsat.enabled` | Enable Psat k8s nodeattestor | `true` | | `nodeAttestor.k8sPsat.serviceAccountAllowList` | Allowed service accounts for Psat nodeattestor | `[]` | +| `nodeAttestor.k8sPsat.audience` | Audience for token validation. If set to [] (empty array), Kubernetes API server audience is used | `[]` | +| `nodeAttestor.k8sPsat.allowedNodeLabelKeys` | Node label keys considered for selectors | `[]` | +| `nodeAttestor.k8sPsat.allowedPodLabelKeys` | Pod label keys considered for selectors | `[]` | +| `nodeAttestor.externalK8sPsat.enabled` | Enable PSAT k8s nodeattestor for external Kubernetes clusters | `true` | +| `nodeAttestor.externalK8sPsat.defaults.serviceAccountAllowList` | Allowed service accounts for PSAT node attestor | `[]` | +| `nodeAttestor.externalK8sPsat.defaults.audience` | Audience for token validation. If it is set to an empty array ([]), Kubernetes API server audience is used | `[]` | +| `nodeAttestor.externalK8sPsat.defaults.allowedNodeLabelKeys` | Node label keys considered for selectors | `[]` | +| `nodeAttestor.externalK8sPsat.defaults.allowedPodLabelKeys` | Pod label keys considered for selectors | `[]` | +| `nodeAttestor.externalK8sPsat.clusters` | A dictionary of clusters to add with optional overrides. If empty, all clusters defined in kubeConfigs will be used. | `{}` | | `nodeAttestor.joinToken.enabled` | Enable the join_token nodeattestor | `false` | | `nodeAttestor.tpmDirect.enabled` | Enable the direct TPM node attestor, a 3rd party plugin by Boxboat. This plugin is experimental. | `false` | | `nodeAttestor.tpmDirect.image.registry` | The OCI registry to pull the image from | `docker.io` | @@ -358,3 +367,4 @@ In order to run Tornjak with simple HTTP Connection only, make sure you don't cr | `tests.bash.image.repository` | The repository within the registry | `chainguard/bash` | | `tests.bash.image.pullPolicy` | The image pull policy | `IfNotPresent` | | `tests.bash.image.tag` | Overrides the image tag whose default is the chart appVersion | `latest@sha256:e4e1f63802396154706a44017f23bd3bfba4f8684374c4c981ba7567636a948e` | +| `kubeConfigs` | Manage additional kubeconfig files to talk to external Kubernetes clusters | `{}` | diff --git a/charts/spire/charts/spire-server/templates/configmap.yaml b/charts/spire/charts/spire-server/templates/configmap.yaml index f211981..743657d 100644 --- a/charts/spire/charts/spire-server/templates/configmap.yaml +++ b/charts/spire/charts/spire-server/templates/configmap.yaml @@ -95,16 +95,46 @@ plugins: plugin_data: {{ include "spire-server.datastore-config" . | nindent 10 }} - {{- if or .Values.nodeAttestor.k8sPsat.enabled .Values.nodeAttestor.joinToken.enabled .Values.nodeAttestor.tpmDirect.enabled }} + {{- if or .Values.nodeAttestor.k8sPsat.enabled .Values.nodeAttestor.externalK8sPsat.enabled .Values.nodeAttestor.joinToken.enabled .Values.nodeAttestor.tpmDirect.enabled }} NodeAttestor: - {{- with .Values.nodeAttestor.k8sPsat }} - {{- if eq (.enabled | toString) "true" }} + {{- if or (eq (.Values.nodeAttestor.k8sPsat.enabled | toString) "true") (eq (.Values.nodeAttestor.externalK8sPsat.enabled | toString) "true") }} k8s_psat: plugin_data: clusters: - {{ include "spire-lib.cluster-name" $root }}: + {{- with .Values.nodeAttestor.k8sPsat }} + {{- if eq (.enabled | toString) "true" }} + - {{ include "spire-lib.cluster-name" $root }}: service_account_allow_list: {{ include "spire-server.serviceAccountAllowedList" $root | trim }} - {{- end }} + audience: {{ .audience }} + allowed_node_label_keys: + {{ toYaml .allowedNodeLabelKeys | nindent 14 }} + allowed_pod_label_keys: + {{ toYaml .allowedPodLabelKeys | nindent 14 }} + {{- end }} + {{- end }} + {{- if eq (.Values.nodeAttestor.externalK8sPsat.enabled | toString) "true" }} + {{- $clusters := default .Values.kubeConfigs .Values.nodeAttestor.externalK8sPsat.clusters }} + {{- $clusterDefaults := .Values.nodeAttestor.externalK8sPsat.defaults }} + {{- range $name, $_ := $clusters }} + {{- $clusterSettings := dict }} + {{- if hasKey $root.Values.nodeAttestor.externalK8sPsat.clusters $name }} + {{- $clusterSettings = index $root.Values.nodeAttestor.externalK8sPsat.clusters $name }} + {{- end }} + - {{ $name }}: + {{- if hasKey $clusterSettings "kubeConfigName" }} + kube_config_file: /kubeconfigs/{{ $clusterSettings.kubeConfigName }} + {{- else }} + kube_config_file: /kubeconfigs/{{ $name }} + {{- end }} + service_account_allow_list: + {{ if hasKey $clusterSettings "serviceAccountAllowList" }}{{ toYaml $clusterSettings.serviceAccountAllowList | nindent 14 }}{{ else }}{{ toYaml $clusterDefaults.serviceAccountAllowList | nindent 14 }}{{ end }} + audience: {{ if hasKey $clusterSettings "audience" }}{{ $clusterSettings.audience }}{{ else }}{{ $clusterDefaults.audience }}{{ end }} + allowed_node_label_keys: + {{ if hasKey $clusterSettings "allowedNodeLabelKeys" }}{{ toYaml $clusterSettings.allowedNodeLabelKeys | nindent 14 }}{{ else }}{{ toYaml $clusterDefaults.allowedNodeLabelKeys | nindent 14 }}{{ end }} + allowed_pod_label_keys: + {{ if hasKey $clusterSettings "allowedPodLabelKeys" }}{{ toYaml $clusterSettings.allowedPodLabelKeys | nindent 14 }}{{ else }}{{ toYaml $clusterDefaults.allowedPodLabelKeys | nindent 14 }}{{ end }} + {{- end }} + {{- end }} {{- end }} {{- with .Values.nodeAttestor.joinToken }} {{- if eq (.enabled | toString) "true" }} diff --git a/charts/spire/charts/spire-server/templates/kubeconfig-secret.yaml b/charts/spire/charts/spire-server/templates/kubeconfig-secret.yaml new file mode 100644 index 0000000..1a553d7 --- /dev/null +++ b/charts/spire/charts/spire-server/templates/kubeconfig-secret.yaml @@ -0,0 +1,19 @@ +{{- $root := . }} +{{- with .Values.kubeConfigs }} +apiVersion: v1 +kind: Secret +metadata: + name: {{ include "spire-server.fullname" $root }}-kubeconfigs + namespace: {{ include "spire-server.namespace" $root }} +data: + {{- range $name, $value := . }} + {{- if and (hasKey . "kubeConfig") (hasKey . "kubeConfigBase64") }} + {{- fail "You can not use both kubeConfig and kubeConfigBase64" }} + {{- end }} + {{- if (hasKey . "kubeConfig") }} + {{ $name }}: {{ .kubeConfig | b64enc }} + {{- else }} + {{ $name }}: {{ .kubeConfigBase64 | nospace }} + {{- end }} + {{- end }} +{{- end }} diff --git a/charts/spire/charts/spire-server/templates/statefulset.yaml b/charts/spire/charts/spire-server/templates/statefulset.yaml index 47f073a..2d3f48a 100644 --- a/charts/spire/charts/spire-server/templates/statefulset.yaml +++ b/charts/spire/charts/spire-server/templates/statefulset.yaml @@ -181,6 +181,11 @@ spec: - name: spire-data mountPath: /run/spire/data readOnly: false + {{- with .Values.kubeConfigs }} + - name: kubeconfigs + mountPath: /kubeconfigs + readOnly: true + {{- end }} {{- if .Values.nodeAttestor.tpmDirect.enabled }} - name: tpm-direct mountPath: /tpm @@ -362,6 +367,11 @@ spec: emptyDir: {} - name: spire-controller-manager-tmp emptyDir: {} + {{- if gt (len .Values.kubeConfigs) 0 }} + - name: kubeconfigs + secret: + secretName: {{ include "spire-server.fullname" . }}-kubeconfigs + {{- end }} {{- if .Values.nodeAttestor.tpmDirect.enabled }} - name: tpm-direct emptyDir: {} diff --git a/charts/spire/charts/spire-server/values.yaml b/charts/spire/charts/spire-server/values.yaml index 78ffd33..c3fffe8 100644 --- a/charts/spire/charts/spire-server/values.yaml +++ b/charts/spire/charts/spire-server/values.yaml @@ -700,6 +700,30 @@ nodeAttestor: enabled: true ## @param nodeAttestor.k8sPsat.serviceAccountAllowList [array] Allowed service accounts for Psat nodeattestor serviceAccountAllowList: [] + ## @param nodeAttestor.k8sPsat.audience [array] Audience for token validation. If set to [] (empty array), Kubernetes API server audience is used + audience: ["spire-server"] + ## @param nodeAttestor.k8sPsat.allowedNodeLabelKeys [array] Node label keys considered for selectors + allowedNodeLabelKeys: [] + ## @param nodeAttestor.k8sPsat.allowedPodLabelKeys [array] Pod label keys considered for selectors + allowedPodLabelKeys: [] + externalK8sPsat: + ## @param nodeAttestor.externalK8sPsat.enabled Enable PSAT k8s nodeattestor for external Kubernetes clusters + enabled: true + defaults: + ## @param nodeAttestor.externalK8sPsat.defaults.serviceAccountAllowList [array] Allowed service accounts for PSAT node attestor + serviceAccountAllowList: ["spire-system:spire-agent-upstream"] + ## @param nodeAttestor.externalK8sPsat.defaults.audience [array] Audience for token validation. If it is set to an empty array ([]), Kubernetes API server audience is used + audience: ["spire-server"] + ## @param nodeAttestor.externalK8sPsat.defaults.allowedNodeLabelKeys [array] Node label keys considered for selectors + allowedNodeLabelKeys: [] + ## @param nodeAttestor.externalK8sPsat.defaults.allowedPodLabelKeys [array] Pod label keys considered for selectors + allowedPodLabelKeys: [] + ## @param nodeAttestor.externalK8sPsat.clusters [object] A dictionary of clusters to add with optional overrides. If empty, all clusters defined in kubeConfigs will be used. + clusters: {} + # clustera: + # kubeConfigName: foo + # serviceAccountAllowList: ["other-ns:other-agent"] + # clusterb: {} joinToken: ## @param nodeAttestor.joinToken.enabled Enable the join_token nodeattestor enabled: false @@ -887,3 +911,12 @@ tests: repository: chainguard/bash pullPolicy: IfNotPresent tag: latest@sha256:e4e1f63802396154706a44017f23bd3bfba4f8684374c4c981ba7567636a948e + +## @param kubeConfigs [object] Manage additional kubeconfig files to talk to external Kubernetes clusters +kubeConfigs: {} +# clustera: +# kubeConfig: | +# xxxxx +# xxxxx +# clusterb: +# kubeConfigBase64: eXl5Cnl5eQo= diff --git a/tests/integration/psat/kind-config.yaml b/tests/integration/psat/kind-config.yaml new file mode 100644 index 0000000..d85992a --- /dev/null +++ b/tests/integration/psat/kind-config.yaml @@ -0,0 +1,7 @@ +kind: Cluster +apiVersion: kind.x-k8s.io/v1alpha4 +networking: + apiServerAddress: "172.17.0.1" + apiServerPort: 7443 + podSubnet: "10.245.0.0/16" + serviceSubnet: "10.97.0.0/12" diff --git a/tests/integration/psat/run-tests.sh b/tests/integration/psat/run-tests.sh new file mode 100755 index 0000000..bbcc22d --- /dev/null +++ b/tests/integration/psat/run-tests.sh @@ -0,0 +1,66 @@ +#!/usr/bin/env bash + +set -xe + +SCRIPT="$(readlink -f "$0")" +SCRIPTPATH="$(dirname "${SCRIPT}")" +TESTDIR="${SCRIPTPATH}/../../../.github/tests" +DEPS="${TESTDIR}/dependencies" + +# shellcheck source=/dev/null +source "${SCRIPTPATH}/../../../.github/scripts/parse-versions.sh" +# shellcheck source=/dev/null +source "${TESTDIR}/common.sh" + +CLEANUP=1 + +for i in "$@"; do + case $i in + -c) + CLEANUP=0 + shift # past argument=value + ;; + esac +done + +teardown() { + print_helm_releases + print_spire_workload_status spire-root-server + print_spire_workload_status spire-server spire-system + + if [[ "$1" -ne 0 ]]; then + get_namespace_details spire-root-server + get_namespace_details spire-server spire-system + fi + + if [ "${CLEANUP}" -eq 1 ]; then + helm uninstall --namespace spire-server spire 2>/dev/null || true + kubectl delete ns spire-server 2>/dev/null || true + kubectl delete ns spire-system 2>/dev/null || true + + helm uninstall --namespace mysql spire-root-server 2>/dev/null || true + kubectl delete ns spire-root-server 2>/dev/null || true + fi +} + +trap 'EC=$? && trap - SIGTERM && teardown $EC' SIGINT SIGTERM EXIT + +kubectl create namespace spire-system --dry-run=client -o yaml | kubectl apply -f - +kubectl label namespace spire-system pod-security.kubernetes.io/enforce=privileged || true +kubectl create namespace spire-server --dry-run=client -o yaml | kubectl apply -f - +kubectl label namespace spire-server pod-security.kubernetes.io/enforce=restricted || true + +helm upgrade --install --create-namespace spire charts/spire \ + --namespace spire-root-server \ + --values "${DEPS}/spire-root-server-values.yaml" \ + --wait + +kind create cluster --name other --kubeconfig "${SCRIPTPATH}/kubeconfig" --config "${SCRIPTPATH}/kind-config.yaml" +md5sum "${SCRIPTPATH}/kubeconfig" +wc -l "${SCRIPTPATH}/kubeconfig" +KCB64="$(base64 < "${SCRIPTPATH}/kubeconfig" | tr '\n' ' ' | sed 's/ //g')" + +helm upgrade --install --create-namespace --namespace spire-server --values "${SCRIPTPATH}/values.yaml" \ + --wait spire charts/spire --set "spire-server.kubeConfigs.other.kubeConfigBase64=$KCB64" +helm test --namespace spire-server spire + diff --git a/tests/integration/psat/values.yaml b/tests/integration/psat/values.yaml new file mode 100644 index 0000000..fbb7b55 --- /dev/null +++ b/tests/integration/psat/values.yaml @@ -0,0 +1,12 @@ +global: + spire: + recommendations: + enabled: true + clusterName: production + trustDomain: production.other + +spire-server: + ca_subject: + country: US + organization: Production + common_name: production.other