diff --git a/charts/spire/charts/spire-server/README.md b/charts/spire/charts/spire-server/README.md index b233852..c04ec19 100644 --- a/charts/spire/charts/spire-server/README.md +++ b/charts/spire/charts/spire-server/README.md @@ -296,6 +296,15 @@ In order to run Tornjak with simple HTTP Connection only, make sure you don't cr | `defaultJwtSvidTTL` | TTL for JWT Svids | `1h` | | `nodeAttestor.k8sPsat.enabled` | Enable Psat k8s nodeattestor | `true` | | `nodeAttestor.k8sPsat.serviceAccountAllowList` | Allowed service accounts for Psat nodeattestor | `[]` | +| `nodeAttestor.k8sPsat.audience` | Audience for token validation. If set to [] (empty array), Kubernetes API server audience is used | `[]` | +| `nodeAttestor.k8sPsat.allowedNodeLabelKeys` | Node label keys considered for selectors | `[]` | +| `nodeAttestor.k8sPsat.allowedPodLabelKeys` | Pod label keys considered for selectors | `[]` | +| `nodeAttestor.externalK8sPsat.enabled` | Enable PSAT k8s nodeattestor for external Kubernetes clusters | `true` | +| `nodeAttestor.externalK8sPsat.defaults.serviceAccountAllowList` | Allowed service accounts for PSAT node attestor | `[]` | +| `nodeAttestor.externalK8sPsat.defaults.audience` | Audience for token validation. If it is set to an empty array ([]), Kubernetes API server audience is used | `[]` | +| `nodeAttestor.externalK8sPsat.defaults.allowedNodeLabelKeys` | Node label keys considered for selectors | `[]` | +| `nodeAttestor.externalK8sPsat.defaults.allowedPodLabelKeys` | Pod label keys considered for selectors | `[]` | +| `nodeAttestor.externalK8sPsat.clusters` | A dictionary of clusters to add with optional overrides. If empty, all clusters defined in kubeConfigs will be used. | `{}` | | `nodeAttestor.joinToken.enabled` | Enable the join_token nodeattestor | `false` | | `nodeAttestor.tpmDirect.enabled` | Enable the direct TPM node attestor, a 3rd party plugin by Boxboat. This plugin is experimental. | `false` | | `nodeAttestor.tpmDirect.image.registry` | The OCI registry to pull the image from | `docker.io` | @@ -358,3 +367,4 @@ In order to run Tornjak with simple HTTP Connection only, make sure you don't cr | `tests.bash.image.repository` | The repository within the registry | `chainguard/bash` | | `tests.bash.image.pullPolicy` | The image pull policy | `IfNotPresent` | | `tests.bash.image.tag` | Overrides the image tag whose default is the chart appVersion | `latest@sha256:e4e1f63802396154706a44017f23bd3bfba4f8684374c4c981ba7567636a948e` | +| `kubeConfigs` | Manage additional kubeconfig files to talk to external Kubernetes clusters | `{}` | diff --git a/charts/spire/charts/spire-server/templates/configmap.yaml b/charts/spire/charts/spire-server/templates/configmap.yaml index f211981..743657d 100644 --- a/charts/spire/charts/spire-server/templates/configmap.yaml +++ b/charts/spire/charts/spire-server/templates/configmap.yaml @@ -95,16 +95,46 @@ plugins: plugin_data: {{ include "spire-server.datastore-config" . | nindent 10 }} - {{- if or .Values.nodeAttestor.k8sPsat.enabled .Values.nodeAttestor.joinToken.enabled .Values.nodeAttestor.tpmDirect.enabled }} + {{- if or .Values.nodeAttestor.k8sPsat.enabled .Values.nodeAttestor.externalK8sPsat.enabled .Values.nodeAttestor.joinToken.enabled .Values.nodeAttestor.tpmDirect.enabled }} NodeAttestor: - {{- with .Values.nodeAttestor.k8sPsat }} - {{- if eq (.enabled | toString) "true" }} + {{- if or (eq (.Values.nodeAttestor.k8sPsat.enabled | toString) "true") (eq (.Values.nodeAttestor.externalK8sPsat.enabled | toString) "true") }} k8s_psat: plugin_data: clusters: - {{ include "spire-lib.cluster-name" $root }}: + {{- with .Values.nodeAttestor.k8sPsat }} + {{- if eq (.enabled | toString) "true" }} + - {{ include "spire-lib.cluster-name" $root }}: service_account_allow_list: {{ include "spire-server.serviceAccountAllowedList" $root | trim }} - {{- end }} + audience: {{ .audience }} + allowed_node_label_keys: + {{ toYaml .allowedNodeLabelKeys | nindent 14 }} + allowed_pod_label_keys: + {{ toYaml .allowedPodLabelKeys | nindent 14 }} + {{- end }} + {{- end }} + {{- if eq (.Values.nodeAttestor.externalK8sPsat.enabled | toString) "true" }} + {{- $clusters := default .Values.kubeConfigs .Values.nodeAttestor.externalK8sPsat.clusters }} + {{- $clusterDefaults := .Values.nodeAttestor.externalK8sPsat.defaults }} + {{- range $name, $_ := $clusters }} + {{- $clusterSettings := dict }} + {{- if hasKey $root.Values.nodeAttestor.externalK8sPsat.clusters $name }} + {{- $clusterSettings = index $root.Values.nodeAttestor.externalK8sPsat.clusters $name }} + {{- end }} + - {{ $name }}: + {{- if hasKey $clusterSettings "kubeConfigName" }} + kube_config_file: /kubeconfigs/{{ $clusterSettings.kubeConfigName }} + {{- else }} + kube_config_file: /kubeconfigs/{{ $name }} + {{- end }} + service_account_allow_list: + {{ if hasKey $clusterSettings "serviceAccountAllowList" }}{{ toYaml $clusterSettings.serviceAccountAllowList | nindent 14 }}{{ else }}{{ toYaml $clusterDefaults.serviceAccountAllowList | nindent 14 }}{{ end }} + audience: {{ if hasKey $clusterSettings "audience" }}{{ $clusterSettings.audience }}{{ else }}{{ $clusterDefaults.audience }}{{ end }} + allowed_node_label_keys: + {{ if hasKey $clusterSettings "allowedNodeLabelKeys" }}{{ toYaml $clusterSettings.allowedNodeLabelKeys | nindent 14 }}{{ else }}{{ toYaml $clusterDefaults.allowedNodeLabelKeys | nindent 14 }}{{ end }} + allowed_pod_label_keys: + {{ if hasKey $clusterSettings "allowedPodLabelKeys" }}{{ toYaml $clusterSettings.allowedPodLabelKeys | nindent 14 }}{{ else }}{{ toYaml $clusterDefaults.allowedPodLabelKeys | nindent 14 }}{{ end }} + {{- end }} + {{- end }} {{- end }} {{- with .Values.nodeAttestor.joinToken }} {{- if eq (.enabled | toString) "true" }} diff --git a/charts/spire/charts/spire-server/templates/kubeconfig-secret.yaml b/charts/spire/charts/spire-server/templates/kubeconfig-secret.yaml new file mode 100644 index 0000000..1a553d7 --- /dev/null +++ b/charts/spire/charts/spire-server/templates/kubeconfig-secret.yaml @@ -0,0 +1,19 @@ +{{- $root := . }} +{{- with .Values.kubeConfigs }} +apiVersion: v1 +kind: Secret +metadata: + name: {{ include "spire-server.fullname" $root }}-kubeconfigs + namespace: {{ include "spire-server.namespace" $root }} +data: + {{- range $name, $value := . }} + {{- if and (hasKey . "kubeConfig") (hasKey . "kubeConfigBase64") }} + {{- fail "You can not use both kubeConfig and kubeConfigBase64" }} + {{- end }} + {{- if (hasKey . "kubeConfig") }} + {{ $name }}: {{ .kubeConfig | b64enc }} + {{- else }} + {{ $name }}: {{ .kubeConfigBase64 | nospace }} + {{- end }} + {{- end }} +{{- end }} diff --git a/charts/spire/charts/spire-server/templates/statefulset.yaml b/charts/spire/charts/spire-server/templates/statefulset.yaml index 47f073a..2d3f48a 100644 --- a/charts/spire/charts/spire-server/templates/statefulset.yaml +++ b/charts/spire/charts/spire-server/templates/statefulset.yaml @@ -181,6 +181,11 @@ spec: - name: spire-data mountPath: /run/spire/data readOnly: false + {{- with .Values.kubeConfigs }} + - name: kubeconfigs + mountPath: /kubeconfigs + readOnly: true + {{- end }} {{- if .Values.nodeAttestor.tpmDirect.enabled }} - name: tpm-direct mountPath: /tpm @@ -362,6 +367,11 @@ spec: emptyDir: {} - name: spire-controller-manager-tmp emptyDir: {} + {{- if gt (len .Values.kubeConfigs) 0 }} + - name: kubeconfigs + secret: + secretName: {{ include "spire-server.fullname" . }}-kubeconfigs + {{- end }} {{- if .Values.nodeAttestor.tpmDirect.enabled }} - name: tpm-direct emptyDir: {} diff --git a/charts/spire/charts/spire-server/values.yaml b/charts/spire/charts/spire-server/values.yaml index 78ffd33..c3fffe8 100644 --- a/charts/spire/charts/spire-server/values.yaml +++ b/charts/spire/charts/spire-server/values.yaml @@ -700,6 +700,30 @@ nodeAttestor: enabled: true ## @param nodeAttestor.k8sPsat.serviceAccountAllowList [array] Allowed service accounts for Psat nodeattestor serviceAccountAllowList: [] + ## @param nodeAttestor.k8sPsat.audience [array] Audience for token validation. If set to [] (empty array), Kubernetes API server audience is used + audience: ["spire-server"] + ## @param nodeAttestor.k8sPsat.allowedNodeLabelKeys [array] Node label keys considered for selectors + allowedNodeLabelKeys: [] + ## @param nodeAttestor.k8sPsat.allowedPodLabelKeys [array] Pod label keys considered for selectors + allowedPodLabelKeys: [] + externalK8sPsat: + ## @param nodeAttestor.externalK8sPsat.enabled Enable PSAT k8s nodeattestor for external Kubernetes clusters + enabled: true + defaults: + ## @param nodeAttestor.externalK8sPsat.defaults.serviceAccountAllowList [array] Allowed service accounts for PSAT node attestor + serviceAccountAllowList: ["spire-system:spire-agent-upstream"] + ## @param nodeAttestor.externalK8sPsat.defaults.audience [array] Audience for token validation. If it is set to an empty array ([]), Kubernetes API server audience is used + audience: ["spire-server"] + ## @param nodeAttestor.externalK8sPsat.defaults.allowedNodeLabelKeys [array] Node label keys considered for selectors + allowedNodeLabelKeys: [] + ## @param nodeAttestor.externalK8sPsat.defaults.allowedPodLabelKeys [array] Pod label keys considered for selectors + allowedPodLabelKeys: [] + ## @param nodeAttestor.externalK8sPsat.clusters [object] A dictionary of clusters to add with optional overrides. If empty, all clusters defined in kubeConfigs will be used. + clusters: {} + # clustera: + # kubeConfigName: foo + # serviceAccountAllowList: ["other-ns:other-agent"] + # clusterb: {} joinToken: ## @param nodeAttestor.joinToken.enabled Enable the join_token nodeattestor enabled: false @@ -887,3 +911,12 @@ tests: repository: chainguard/bash pullPolicy: IfNotPresent tag: latest@sha256:e4e1f63802396154706a44017f23bd3bfba4f8684374c4c981ba7567636a948e + +## @param kubeConfigs [object] Manage additional kubeconfig files to talk to external Kubernetes clusters +kubeConfigs: {} +# clustera: +# kubeConfig: | +# xxxxx +# xxxxx +# clusterb: +# kubeConfigBase64: eXl5Cnl5eQo= diff --git a/tests/integration/psat/kind-config.yaml b/tests/integration/psat/kind-config.yaml new file mode 100644 index 0000000..d85992a --- /dev/null +++ b/tests/integration/psat/kind-config.yaml @@ -0,0 +1,7 @@ +kind: Cluster +apiVersion: kind.x-k8s.io/v1alpha4 +networking: + apiServerAddress: "172.17.0.1" + apiServerPort: 7443 + podSubnet: "10.245.0.0/16" + serviceSubnet: "10.97.0.0/12" diff --git a/tests/integration/psat/run-tests.sh b/tests/integration/psat/run-tests.sh new file mode 100755 index 0000000..bbcc22d --- /dev/null +++ b/tests/integration/psat/run-tests.sh @@ -0,0 +1,66 @@ +#!/usr/bin/env bash + +set -xe + +SCRIPT="$(readlink -f "$0")" +SCRIPTPATH="$(dirname "${SCRIPT}")" +TESTDIR="${SCRIPTPATH}/../../../.github/tests" +DEPS="${TESTDIR}/dependencies" + +# shellcheck source=/dev/null +source "${SCRIPTPATH}/../../../.github/scripts/parse-versions.sh" +# shellcheck source=/dev/null +source "${TESTDIR}/common.sh" + +CLEANUP=1 + +for i in "$@"; do + case $i in + -c) + CLEANUP=0 + shift # past argument=value + ;; + esac +done + +teardown() { + print_helm_releases + print_spire_workload_status spire-root-server + print_spire_workload_status spire-server spire-system + + if [[ "$1" -ne 0 ]]; then + get_namespace_details spire-root-server + get_namespace_details spire-server spire-system + fi + + if [ "${CLEANUP}" -eq 1 ]; then + helm uninstall --namespace spire-server spire 2>/dev/null || true + kubectl delete ns spire-server 2>/dev/null || true + kubectl delete ns spire-system 2>/dev/null || true + + helm uninstall --namespace mysql spire-root-server 2>/dev/null || true + kubectl delete ns spire-root-server 2>/dev/null || true + fi +} + +trap 'EC=$? && trap - SIGTERM && teardown $EC' SIGINT SIGTERM EXIT + +kubectl create namespace spire-system --dry-run=client -o yaml | kubectl apply -f - +kubectl label namespace spire-system pod-security.kubernetes.io/enforce=privileged || true +kubectl create namespace spire-server --dry-run=client -o yaml | kubectl apply -f - +kubectl label namespace spire-server pod-security.kubernetes.io/enforce=restricted || true + +helm upgrade --install --create-namespace spire charts/spire \ + --namespace spire-root-server \ + --values "${DEPS}/spire-root-server-values.yaml" \ + --wait + +kind create cluster --name other --kubeconfig "${SCRIPTPATH}/kubeconfig" --config "${SCRIPTPATH}/kind-config.yaml" +md5sum "${SCRIPTPATH}/kubeconfig" +wc -l "${SCRIPTPATH}/kubeconfig" +KCB64="$(base64 < "${SCRIPTPATH}/kubeconfig" | tr '\n' ' ' | sed 's/ //g')" + +helm upgrade --install --create-namespace --namespace spire-server --values "${SCRIPTPATH}/values.yaml" \ + --wait spire charts/spire --set "spire-server.kubeConfigs.other.kubeConfigBase64=$KCB64" +helm test --namespace spire-server spire + diff --git a/tests/integration/psat/values.yaml b/tests/integration/psat/values.yaml new file mode 100644 index 0000000..fbb7b55 --- /dev/null +++ b/tests/integration/psat/values.yaml @@ -0,0 +1,12 @@ +global: + spire: + recommendations: + enabled: true + clusterName: production + trustDomain: production.other + +spire-server: + ca_subject: + country: US + organization: Production + common_name: production.other