From 8fef1bd050bc99a9bb50f0fb0de3c6b86429093a Mon Sep 17 00:00:00 2001 From: kfox1111 Date: Thu, 2 May 2024 14:08:31 -0700 Subject: [PATCH] Add external spire-controller-managers (#284) * Complete Server K8S PSAT support Add all the SPIRE Server supported options for the K8S PSAT attestor. This retains the ease of use for configuring local cluster support while adding the ability to configure multiple/external clusters as well. Kubeconfig support is added in its own config block as it will be used/shared with spire-controller-manager support in the future. Signed-off-by: Kevin Fox * Fix merge conflict Signed-off-by: Kevin Fox * Add support for integration tests in the tests/integration dir Signed-off-by: Kevin Fox * Fix split issue and typo Signed-off-by: Kevin Fox * Add basic psat test Signed-off-by: Kevin Fox * Fix linter Signed-off-by: Kevin Fox * Fix up test Signed-off-by: Kevin Fox * Add missing file Signed-off-by: Kevin Fox * Better encode config Signed-off-by: Kevin Fox * Update charts/spire/charts/spire-server/values.yaml Co-authored-by: Faisal Memon Signed-off-by: kfox1111 * Update docs Signed-off-by: Kevin Fox * Apply suggestions from code review Co-authored-by: Faisal Memon Signed-off-by: kfox1111 * Fix docs Signed-off-by: Kevin Fox * Update charts/spire/charts/spire-server/values.yaml Co-authored-by: Faisal Memon Signed-off-by: kfox1111 * Fix docs Signed-off-by: Kevin Fox * Add external k8s bundler Adds support to sync the CA bundle to configmaps in external Kubernetes clusters Signed-off-by: Kevin Fox * Update default Signed-off-by: Kevin Fox * Fix config file layout. Incorperate feedback. Signed-off-by: Kevin Fox * Incorperate feedback Signed-off-by: Kevin Fox * Update based on parent pr feedback Signed-off-by: Kevin Fox * Reformat config file Signed-off-by: Kevin Fox * Fix some things Signed-off-by: Kevin Fox * Reconfigure kind Signed-off-by: Kevin Fox * More debugging Signed-off-by: Kevin Fox * Fix up kind Signed-off-by: Kevin Fox * Incorperate feedback Signed-off-by: Kevin Fox * Add external spire-controller-managers Only one external controller manager is supported at a time until https://github.com/spiffe/spire/issues/4898 is resolved. Signed-off-by: Kevin Fox * Fix tests Signed-off-by: Kevin Fox * Fix test Signed-off-by: Kevin Fox * Fix test Signed-off-by: Kevin Fox * Fix test Signed-off-by: Kevin Fox * Fix test Signed-off-by: Kevin Fox * Fix test Signed-off-by: Kevin Fox * Fix test Signed-off-by: Kevin Fox * Fix test Signed-off-by: Kevin Fox * Fix test Signed-off-by: Kevin Fox * Fix test Signed-off-by: Kevin Fox * Fix test Signed-off-by: Kevin Fox * Fix test Signed-off-by: Kevin Fox * Fix test Signed-off-by: Kevin Fox * Fix test Signed-off-by: Kevin Fox * Fix test Signed-off-by: Kevin Fox * Fix test Signed-off-by: Kevin Fox * Fix test Signed-off-by: Kevin Fox * Upgrade to spire-controller-manager 0.5.0 Signed-off-by: Kevin Fox * Update for released 0.5.0 Signed-off-by: Kevin Fox * Merge in some of the id prefix pr Signed-off-by: Kevin Fox * Entry ID Prefix (#287) * Add Entry ID Prefix support Signed-off-by: Kevin Fox * Mulitcluster test Signed-off-by: Kevin Fox * Fix test Signed-off-by: Kevin Fox * Fix test Signed-off-by: Kevin Fox * Fix test Signed-off-by: Kevin Fox * Fix test Signed-off-by: Kevin Fox * Fix test Signed-off-by: Kevin Fox * Fix test Signed-off-by: Kevin Fox * Implement cleanup setting too Signed-off-by: Kevin Fox * Fix test Signed-off-by: Kevin Fox * Fix docs Signed-off-by: Kevin Fox * Bump up test container Signed-off-by: Kevin Fox * Swith to testing with nightly Signed-off-by: Kevin Fox * Fix value name Signed-off-by: Kevin Fox * Fix docs Signed-off-by: Kevin Fox --------- Signed-off-by: Kevin Fox Signed-off-by: kfox1111 * Fix up doc formatting Signed-off-by: Kevin Fox * Fix merge conflict Signed-off-by: Kevin Fox * Update charts/spire/charts/spire-server/values.yaml Co-authored-by: Faisal Memon Signed-off-by: kfox1111 --------- Signed-off-by: Kevin Fox Signed-off-by: kfox1111 Co-authored-by: Faisal Memon --- .github/tests/common.sh | 2 +- .github/workflows/helm-chart-ci.yaml | 3 +- charts/spire/README.md | 13 +- charts/spire/charts/spire-server/README.md | 16 +++ .../_controller-manager-container.tpl | 125 ++++++++++++++++++ .../spire-server/templates/_helpers.tpl | 8 +- .../controller-manager-configmap.yaml | 118 +++++++++++------ .../templates/server-resource.yaml | 54 +------- charts/spire/charts/spire-server/values.yaml | 47 +++++++ charts/spire/values.yaml | 3 + ...ind-config.yaml => child-kind-config.yaml} | 2 +- tests/integration/psat/child-values.yaml | 20 +++ tests/integration/psat/other-kind-config.yaml | 7 + tests/integration/psat/run-tests.sh | 59 ++++++--- .../psat/sodp-clusterspiffeid.yaml | 23 ++++ tests/integration/psat/values.yaml | 26 +++- 16 files changed, 397 insertions(+), 129 deletions(-) create mode 100644 charts/spire/charts/spire-server/templates/_controller-manager-container.tpl rename tests/integration/psat/{kind-config.yaml => child-kind-config.yaml} (81%) create mode 100644 tests/integration/psat/child-values.yaml create mode 100644 tests/integration/psat/other-kind-config.yaml create mode 100644 tests/integration/psat/sodp-clusterspiffeid.yaml diff --git a/.github/tests/common.sh b/.github/tests/common.sh index 7d5323b..0566665 100755 --- a/.github/tests/common.sh +++ b/.github/tests/common.sh @@ -21,7 +21,7 @@ $(kubectl --request-timeout=30s describe pods --namespace "$1") #### Logs \`\`\`shell -$(kubectl get pods -o name -n "$1" | while read -r line; do echo logs for "${line}"; kubectl logs -n "$1" "${line}" --all-containers=true --ignore-errors=true; done) +$(kubectl get pods -o name -n "$1" | while read -r line; do echo logs for "${line}"; kubectl logs -n "$1" "${line}" --prefix --all-containers=true --ignore-errors=true; done) $( ([[ -n "$2" ]] && kubectl get pods -o name -n "$2") | while read -r line; do echo logs for "${line}"; kubectl logs -n "$2" "${line}" --all-containers=true --ignore-errors=true; done) \`\`\` diff --git a/.github/workflows/helm-chart-ci.yaml b/.github/workflows/helm-chart-ci.yaml index 08758b4..c664f9e 100644 --- a/.github/workflows/helm-chart-ci.yaml +++ b/.github/workflows/helm-chart-ci.yaml @@ -300,8 +300,7 @@ jobs: - name: Install and test integration run: | - kubectl create namespace spire-server - helm install -n spire-server spire-crds charts/spire-crds + helm install --create-namespace -n spire-mgmt spire-crds charts/spire-crds ${{ matrix.integrationtest }}/run-tests.sh upgrade-test: diff --git a/charts/spire/README.md b/charts/spire/README.md index d3267de..d9bc66a 100644 --- a/charts/spire/README.md +++ b/charts/spire/README.md @@ -255,12 +255,13 @@ Now you can interact with the Spire agent socket from your own application. The ### Spire server parameters -| Name | Description | Value | -| ---------------------------------------- | ------------------------------------------------------------------------- | ------------- | -| `spire-server.enabled` | Flag to enable Spire server | `true` | -| `spire-server.nameOverride` | Overrides the name of Spire server pods | `server` | -| `spire-server.kind` | Run spire server as deployment/statefulset. This feature is experimental. | `statefulset` | -| `spire-server.controllerManager.enabled` | Enable controller manager and provision CRD's | `true` | +| Name | Description | Value | +| ------------------------------------------------- | ------------------------------------------------------------------------- | ------------- | +| `spire-server.enabled` | Flag to enable Spire server | `true` | +| `spire-server.nameOverride` | Overrides the name of Spire server pods | `server` | +| `spire-server.kind` | Run spire server as deployment/statefulset. This feature is experimental. | `statefulset` | +| `spire-server.controllerManager.enabled` | Enable controller manager and provision CRD's | `true` | +| `spire-server.externalControllerManagers.enabled` | Enable external controller manager support | `true` | ### Spire agent parameters diff --git a/charts/spire/charts/spire-server/README.md b/charts/spire/charts/spire-server/README.md index 7d94afd..c0c9b49 100644 --- a/charts/spire/charts/spire-server/README.md +++ b/charts/spire/charts/spire-server/README.md @@ -286,6 +286,22 @@ In order to run Tornjak with simple HTTP Connection only, make sure you don't cr | `controllerManager.validatingWebhookConfiguration.enabled` | Disable only when you have another chart instance on the k8s cluster with webhooks enabled. | `true` | | `controllerManager.validatingWebhookConfiguration.failurePolicy` | Action when identity is not issued | `Fail` | | `controllerManager.cacheNamespaces` | If specified restricts the manager's cache to watch objects in the desired namespaces. Defaults to all namespaces. | `{}` | +| `externalControllerManagers.enabled` | Flag to enable external controller managers | `false` | +| `externalControllerManagers.defaults.reconcile.clusterSPIFFEIDs` | Enable reconciliation of clusterSPIFFEIDs from K8s to the SPIRE server | `true` | +| `externalControllerManagers.defaults.reconcile.clusterStaticEntries` | Enable reconciliation of clusterStaticEntries from K8s to the SPIRE server | `false` | +| `externalControllerManagers.defaults.reconcile.clusterFederatedTrustDomains` | Enable reconciliation of clusterFederatedTrustDomains from K8s to the SPIRE server | `false` | +| `externalControllerManagers.defaults.className` | specify to use an explicit class name. If empty, it will be automatically set to Release.Namespace-Release.Name to not conflict with other installs, enabling parallel installs. | `""` | +| `externalControllerManagers.defaults.watchClassless` | specify to process custom resources without class name specified. Useful to slowly migrate to class names from classless installs. Do not have two installs on the same k8s cluster both set to true. | `false` | +| `externalControllerManagers.defaults.entryIDPrefixCleanup` | consult the spiffe.io docs about this option before changing. Its unlikely you will need to ever change it. | `false` | +| `externalControllerManagers.defaults.parentIDTemplate` | The template that is used to register workloads. | `spiffe://{{ .TrustDomain }}/spire/agent/k8s_psat/{{ .ClusterName }}/{{ .NodeMeta.UID }}` | +| `externalControllerManagers.defaults.expandEnv` | Set to true to enable environment variable substitution of config file options | `false` | +| `externalControllerManagers.defaults.extraEnv` | Extra environment variables to add to the controller manager | `[]` | +| `externalControllerManagers.defaults.resources` | Resource requests and limits for controller manager | `{}` | +| `externalControllerManagers.defaults.securityContext` | Security context | `{}` | +| `externalControllerManagers.defaults.configMap.annotations` | Annotations to add to the Controller Manager ConfigMap | `{}` | +| `externalControllerManagers.defaults.ignoreNamespaces` | These namespaces are ignored by controller manager | `[]` | +| `externalControllerManagers.defaults.cacheNamespaces` | If specified restricts the manager's cache to watch objects in the desired namespaces. Defaults to all namespaces. | `{}` | +| `externalControllerManagers.clusters` | A dictionary of clusters to add with optional overrides. If empty, all clusters defined in kubeConfigs will be used. | `{}` | | `tools.kubectl.image.registry` | The OCI registry to pull the image from | `docker.io` | | `tools.kubectl.image.repository` | The repository within the registry | `rancher/kubectl` | | `tools.kubectl.image.pullPolicy` | The image pull policy | `IfNotPresent` | diff --git a/charts/spire/charts/spire-server/templates/_controller-manager-container.tpl b/charts/spire/charts/spire-server/templates/_controller-manager-container.tpl new file mode 100644 index 0000000..4c0b4e9 --- /dev/null +++ b/charts/spire/charts/spire-server/templates/_controller-manager-container.tpl @@ -0,0 +1,125 @@ +{{- define "spire-controller-manager.containers" }} +{{- $root := . }} +{{- $settings := dict }} +{{- $defaults := .Values.controllerManager }} +{{- $webhooksEnabled := .Values.controllerManager.validatingWebhookConfiguration.enabled }} +{{- $startPort := 8082 }} +{{- $reconcileFederation := 0 }} +{{- $reconcileEntries := 0 }} +{{- if eq (.Values.controllerManager.enabled | toString) "true" }} +{{- if .Values.controllerManager.reconcile.clusterFederatedTrustDomains }} +{{- $reconcileFederation = add $reconcileFederation 1 }} +{{- end }} +{{- if or .Values.controllerManager.reconcile.clusterSPIFFEIDs .Values.controllerManager.reconcile.clusterStaticEntries }} +{{- $reconcileEntries = add $reconcileEntries 1 }} +{{- end }} +{{- include "spire-controller-manager.container" (dict "Values" .Values "Chart" .Chart "startPort" $startPort "suffix" "" "settings" $settings "defaults" $defaults "webhooksEnabled" $webhooksEnabled) }} +{{- end }} +{{- if .Values.externalControllerManagers.enabled }} +{{- $clusters := default .Values.kubeConfigs .Values.externalControllerManagers.clusters }} +{{- $clusterDefaults := .Values.externalControllerManagers.defaults }} +{{- range $name, $_ := $clusters }} +{{- $clusterSettings := dict }} +{{- if hasKey $root.Values.externalControllerManagers.clusters $name }} +{{- $clusterSettings = index $root.Values.externalControllerManagers.clusters $name }} +{{- end }} +{{- $suffix := printf "-%s" $name }} +{{- $startPort = add $startPort 2 }} +{{- $kubeConfig := $name }} +{{- if hasKey $clusterSettings "kubeConfigName" }} +{{- $kubeConfig = $clusterSettings.kubeConfigName }} +{{- end }} +{{- $reconcile := dict }} +{{- if hasKey $clusterSettings "reconcile" }} +{{- $reconcile = $clusterSettings.reconcile }} +{{- end }} +{{- if and (hasKey $reconcile "clusterFederatedTrustDomains") $reconcile.clusterFederatedTrustDomains }} +{{- $reconcileFederation = add $reconcileFederation 1 }} +{{- else if $clusterDefaults.reconcile.clusterFederatedTrustDomains }} +{{- $reconcileFederation = add $reconcileFederation 1 }} +{{- end }} +{{- if gt $reconcileFederation 1 }} +{{- fail "You can only have one controller-manager with reconcile.clusterFederatedTrustDomains set to true" }} +{{- end }} +{{- include "spire-controller-manager.container" (dict "Values" $root.Values "Chart" $root.Chart "startPort" $startPort "suffix" $suffix "settings" $clusterSettings "defaults" $clusterDefaults "webhooksEnabled" false "kubeConfig" $kubeConfig ) }} +{{- end }} +{{- end }} +{{- end }} +{{- define "spire-controller-manager.container" }} +{{- $promPort := .startPort }} +{{- $healthPort := add .startPort 1 }} +{{- $extraEnv := .defaults.extraEnv }} +{{- if hasKey .settings "extraEnv" }} +{{- $extraEnv = .settings.extraEnv }} +{{- end }} +{{- $expandEnv := .defaults.expandEnv }} +{{- if hasKey .settings "expandEnv" }} +{{- $extraEnv = .settings.expandEnv }} +{{- end }} +{{- $securityContext := .defaults.securityContext }} +{{- if hasKey .settings "securityContext" }} +{{- $securityContext = mergeOverwrite .defaults.securityContext .settings.securityContext }} +{{- end }} +- name: spire-controller-manager{{ .suffix }} + securityContext: + {{- include "spire-lib.securitycontext-extended" (dict "root" . "securityContext" $securityContext) | nindent 4 }} + image: {{ template "spire-lib.image" (dict "appVersion" .Chart.AppVersion "image" .Values.controllerManager.image "global" .Values.global) }} + imagePullPolicy: {{ .Values.controllerManager.image.pullPolicy }} + args: + {{- if hasKey . "kubeConfig" }} + - --kubeconfig=/kubeconfigs/{{ .kubeConfig }} + {{- end }} + - --config=controller-manager-config{{ .suffix }}.yaml + {{- if $expandEnv }} + - --expand-env + {{- end }} + env: + - name: ENABLE_WEBHOOKS + value: {{ .webhooksEnabled | toString | quote }} + {{- if gt (len $extraEnv) 0 }} + {{- $extraEnv | toYaml | nindent 4 }} + {{- end }} + ports: + {{- if .webhooksEnabled }} + - name: https + containerPort: 9443 + protocol: TCP + {{- end }} + - containerPort: {{ $healthPort }} + name: healthz + {{- if or (dig "telemetry" "prometheus" "enabled" .Values.telemetry.prometheus.enabled .Values.global) (and (dig "spire" "recommendations" "enabled" false .Values.global) (dig "spire" "recommendations" "prometheus" true .Values.global)) }} + - containerPort: {{ $promPort }} + name: prom-cm{{ .suffix }} + {{- end }} + livenessProbe: + httpGet: + path: /healthz + port: healthz + readinessProbe: + httpGet: + path: /readyz + port: healthz + resources: + {{- toYaml .Values.controllerManager.resources | nindent 4 }} + volumeMounts: + - name: spire-server-socket + mountPath: /tmp/spire-server/private + readOnly: true + - name: controller-manager-config + mountPath: /controller-manager-config{{ .suffix }}.yaml + subPath: controller-manager-config{{ .suffix }}.yaml + readOnly: true + {{- with .kubeConfig }} + - name: kubeconfigs + mountPath: /kubeconfigs/{{ . }} + subPath: {{ . }} + readOnly: true + {{- end }} + - name: spire-controller-manager-tmp + mountPath: /tmp + subPath: {{ printf "spire-controller-manager%s" .suffix }} + readOnly: false + {{- if gt (len .Values.extraVolumeMounts) 0 }} + {{- toYaml .Values.extraVolumeMounts | nindent 4 }} + {{- end }} +{{- end }} diff --git a/charts/spire/charts/spire-server/templates/_helpers.tpl b/charts/spire/charts/spire-server/templates/_helpers.tpl index acb4697..0b3fb0f 100644 --- a/charts/spire/charts/spire-server/templates/_helpers.tpl +++ b/charts/spire/charts/spire-server/templates/_helpers.tpl @@ -280,8 +280,12 @@ The code below determines what connection type should be used. {{- end -}} {{- define "spire-server.controller-manager-class-name" -}} -{{- if .Values.controllerManager.className }} -{{- .Values.controllerManager.className }} +{{- if and (hasKey . "settings") (hasKey .settings "className") }} +{{- .settings.className }} +{{- else if and (hasKey . "defaults") .defaults.className }} +{{- .defaults.className }} +{{- else if .Values.controllerManager.className }} +{{- .Values.controllerManager.className }} {{- else }} {{- .Release.Namespace }}-{{ default .Release.Name .Values.crNameOverride }} {{- end -}} diff --git a/charts/spire/charts/spire-server/templates/controller-manager-configmap.yaml b/charts/spire/charts/spire-server/templates/controller-manager-configmap.yaml index 52e86ca..bcfa7ff 100644 --- a/charts/spire/charts/spire-server/templates/controller-manager-configmap.yaml +++ b/charts/spire/charts/spire-server/templates/controller-manager-configmap.yaml @@ -1,5 +1,9 @@ +{{- $root := . }} +{{- $startPort := 8082 }} +{{- $clusters := default .Values.kubeConfigs .Values.externalControllerManagers.clusters }} +{{- $clusterDefaults := .Values.externalControllerManagers.defaults }} {{- if not .Values.externalServer }} -{{- if eq (.Values.controllerManager.enabled | toString) "true" }} +{{- if or (eq (.Values.controllerManager.enabled | toString) "true") (and .Values.externalControllerManagers.enabled (gt (len $clusters) 0)) }} apiVersion: v1 kind: ConfigMap metadata: @@ -10,46 +14,78 @@ metadata: {{- toYaml . | nindent 4 }} {{- end }} data: +{{- if eq (.Values.controllerManager.enabled | toString) "true" }} +{{- $clusterName := include "spire-lib.cluster-name" . }} controller-manager-config.yaml: | - apiVersion: spire.spiffe.io/v1alpha1 - kind: ControllerManagerConfig - metadata: - name: {{ include "spire-controller-manager.fullname" . }} - namespace: {{ include "spire-server.namespace" . }} - labels: - {{- include "spire-server.labels" . | nindent 8 }} - metrics: - bindAddress: 0.0.0.0:8082 - health: - healthProbeBindAddress: 0.0.0.0:8083 - leaderElection: - leaderElect: true - resourceName: {{ printf "%s-%s" .Release.Namespace (default .Release.Name .Values.crNameOverride) | sha256sum | trunc 8 }}.spiffe.io - resourceNamespace: {{ include "spire-server.namespace" . }} - {{- with .Values.controllerManager.cacheNamespaces }} - cacheNamespaces: - {{- toYaml . | nindent 6 }} - {{- end }} - {{- if .Values.controllerManager.validatingWebhookConfiguration.enabled }} - validatingWebhookConfigurationName: {{ .Release.Namespace }}-{{ include "spire-controller-manager.fullname" . }}-webhook - {{- end }} - {{- if typeIs "string" .Values.controllerManager.entryIDPrefixCleanup }} - entryIDPrefixCleanup: {{ .Values.controllerManager.entryIDPrefixCleanup | quote }} - {{- end }} - entryIDPrefix: {{ include "spire-lib.cluster-name" . }} - clusterName: {{ include "spire-lib.cluster-name" . }} - trustDomain: {{ include "spire-lib.trust-domain" . }} - ignoreNamespaces: - {{- with .Values.controllerManager.ignoreNamespaces }} - {{- toYaml . | nindent 6 }} - {{- end }} - spireServerSocketPath: "/tmp/spire-server/private/api.sock" - className: {{ include "spire-server.controller-manager-class-name" . | quote}} - watchClassless: {{ .Values.controllerManager.watchClassless | toYaml }} - parentIDTemplate: {{ .Values.controllerManager.parentIDTemplate | quote }} - {{- with .Values.reconcile }} - reconcile: - {{- toYaml . | nindent 6 }} - {{- end }} + {{- include "spire-controller-manager.config" (dict "Values" .Values "Chart" .Chart "Release" .Release "startPort" $startPort "suffix" "" "settings" (dict) "defaults" .Values.controllerManager "webhookEnabled" .Values.controllerManager.validatingWebhookConfiguration.enabled "clusterName" $clusterName) | nindent 4 }} +{{- end }} +{{- if .Values.externalControllerManagers.enabled }} +{{- range $name, $_ := $clusters }} +{{- $clusterSettings := dict }} +{{- if hasKey $root.Values.externalControllerManagers.clusters $name }} +{{- $clusterSettings = index $root.Values.externalControllerManagers.clusters $name }} +{{- end }} +{{- $suffix := printf "-%s" $name }} +{{- $startPort = add $startPort 2 }} + controller-manager-config{{ $suffix }}.yaml: | + {{- include "spire-controller-manager.config" (dict "Values" $root.Values "Chart" $root.Chart "Release" $root.Release "startPort" $startPort "suffix" $suffix "settings" $clusterSettings "defaults" $clusterDefaults "webhookEnabled" false "clusterName" $name) | nindent 4 }} {{- end }} {{- end }} +{{- end }} +{{- end }} +{{- define "spire-controller-manager.config" }} +{{- $promPort := .startPort }} +{{- $healthPort := add $promPort 1 }} +apiVersion: spire.spiffe.io/v1alpha1 +kind: ControllerManagerConfig +metadata: + name: {{ include "spire-controller-manager.fullname" . }}{{ .suffix }} + namespace: {{ include "spire-server.namespace" . }} + labels: + {{- include "spire-server.labels" . | nindent 4 }} +metrics: + bindAddress: 0.0.0.0:{{ $promPort }} +health: + healthProbeBindAddress: 0.0.0.0:{{ $healthPort }} +leaderElection: + leaderElect: true + resourceName: {{ printf "%s-%s%s" .Release.Namespace (default .Release.Name .Values.crNameOverride) .suffix | sha256sum | trunc 8 }}.spiffe.io + resourceNamespace: {{ include "spire-server.namespace" . }} +{{- with .settings.cacheNamespaces }} +cacheNamespaces: + {{- toYaml . | nindent 2 }} +{{- end }} +{{- if .webhookEnabled }} +validatingWebhookConfigurationName: {{ .Release.Namespace }}-{{ include "spire-controller-manager.fullname" . }}-webhook +{{- end }} +{{- $entryIDPrefixCleanup := .defaults.entryIDPrefixCleanup }} +{{- if hasKey .settings "entryIDPrefixCleanup" }} +{{- $entryIDPrefixCleanup = .settings.entryIDPrefixCleanup }} +{{- end }} +{{- if typeIs "string" $entryIDPrefixCleanup }} +entryIDPrefixCleanup: {{ $entryIDPrefixCleanup | quote }} +{{- end }} +entryIDPrefix: {{ .clusterName }} +clusterName: {{ .clusterName }} +trustDomain: {{ include "spire-lib.trust-domain" . }} +{{- $ignoreNamespaces := .defaults.ignoreNamespaces }} +{{- if hasKey .settings "ignoreNamespaces" }} +{{- $ignoreNamespaces = .settings.ingoreNamespaces }} +{{- end }} +{{- with $ignoreNamespaces }} +ignoreNamespaces: + {{- toYaml . | nindent 2 }} +{{- end }} +spireServerSocketPath: "/tmp/spire-server/private/api.sock" +className: {{ include "spire-server.controller-manager-class-name" . | quote}} +watchClassless: {{ if hasKey .settings "watchClassless" }}{{ .settings.watchClassless | toYaml }}{{ else }}{{ .defaults.watchClassless | toYaml }}{{ end }} +parentIDTemplate: {{ if hasKey .settings "parentIDTemplate" }}{{ .settings.parentIDTemplate | quote }}{{ else }}{{ .defaults.parentIDTemplate | quote }}{{ end }} +{{- $reconcile := dict }} +{{- if hasKey .settings "reconcile" }} +{{- $reconcile = .settings.reconcile }} +{{- end }} +reconcile: + clusterSPIFFEIDs: {{ if hasKey $reconcile "clusterSPIFFEIDs" }}{{ toYaml $reconcile.clusterSPIFFEIDs }}{{ else }}{{ toYaml .defaults.reconcile.clusterSPIFFEIDs }}{{ end }} + clusterStaticEntries: {{ if hasKey $reconcile "clusterStaticEntries" }}{{ toYaml $reconcile.clusterStaticEntries }}{{ else }}{{ toYaml .defaults.reconcile.clusterStaticEntries }}{{ end }} + clusterFederatedTrustDomains: {{ if hasKey $reconcile "clusterFederatedTrustDomains" }}{{ toYaml $reconcile.clusterFederatedTrustDomains }}{{ else }}{{ toYaml .defaults.reconcile.clusterFederatedTrustDomains }}{{ end }} +{{- end }} diff --git a/charts/spire/charts/spire-server/templates/server-resource.yaml b/charts/spire/charts/spire-server/templates/server-resource.yaml index a90dba2..d309109 100644 --- a/charts/spire/charts/spire-server/templates/server-resource.yaml +++ b/charts/spire/charts/spire-server/templates/server-resource.yaml @@ -257,59 +257,7 @@ spec: - name: server-tmp mountPath: /tmp readOnly: false - {{- if eq (.Values.controllerManager.enabled | toString) "true" }} - - name: spire-controller-manager - securityContext: - {{- include "spire-lib.securitycontext-extended" (dict "root" . "securityContext" .Values.controllerManager.securityContext) | nindent 12 }} - image: {{ template "spire-lib.image" (dict "appVersion" $.Chart.AppVersion "image" .Values.controllerManager.image "global" .Values.global) }} - imagePullPolicy: {{ .Values.controllerManager.image.pullPolicy }} - args: - - --config=controller-manager-config.yaml - {{- if .Values.controllerManager.expandEnv }} - - --expand-env - {{- end }} - env: - - name: ENABLE_WEBHOOKS - value: {{ .Values.controllerManager.validatingWebhookConfiguration.enabled | toString | quote }} - {{- if gt (len .Values.controllerManager.extraEnv) 0 }} - {{- .Values.controllerManager.extraEnv | toYaml | nindent 12 }} - {{- end }} - ports: - - name: https - containerPort: 9443 - protocol: TCP - - containerPort: 8083 - name: healthz - {{- if or (dig "telemetry" "prometheus" "enabled" .Values.telemetry.prometheus.enabled .Values.global) (and (dig "spire" "recommendations" "enabled" false .Values.global) (dig "spire" "recommendations" "prometheus" true .Values.global)) }} - - containerPort: 8082 - name: prom2 - {{- end }} - livenessProbe: - httpGet: - path: /healthz - port: healthz - readinessProbe: - httpGet: - path: /readyz - port: healthz - resources: - {{- toYaml .Values.controllerManager.resources | nindent 12 }} - volumeMounts: - - name: spire-server-socket - mountPath: /tmp/spire-server/private - readOnly: true - - name: controller-manager-config - mountPath: /controller-manager-config.yaml - subPath: controller-manager-config.yaml - readOnly: true - - name: spire-controller-manager-tmp - mountPath: /tmp - readOnly: false - {{- if gt (len .Values.extraVolumeMounts) 0 }} - {{- toYaml .Values.extraVolumeMounts | nindent 12 }} - {{- end }} - {{- end }} - + {{- include "spire-controller-manager.containers" . | nindent 8 }} {{- if eq (.Values.tornjak.enabled | toString) "true" }} - name: tornjak securityContext: diff --git a/charts/spire/charts/spire-server/values.yaml b/charts/spire/charts/spire-server/values.yaml index 9149176..570454c 100644 --- a/charts/spire/charts/spire-server/values.yaml +++ b/charts/spire/charts/spire-server/values.yaml @@ -658,6 +658,53 @@ controllerManager: # fieldSelectors: # fName: f1 +externalControllerManagers: + ## @param externalControllerManagers.enabled Flag to enable external controller managers + enabled: false + defaults: + ## @param externalControllerManagers.defaults.reconcile.clusterSPIFFEIDs Enable reconciliation of clusterSPIFFEIDs from K8s to the SPIRE server + ## @param externalControllerManagers.defaults.reconcile.clusterStaticEntries Enable reconciliation of clusterStaticEntries from K8s to the SPIRE server + ## @param externalControllerManagers.defaults.reconcile.clusterFederatedTrustDomains Enable reconciliation of clusterFederatedTrustDomains from K8s to the SPIRE server + reconcile: + clusterSPIFFEIDs: true + clusterStaticEntries: false + clusterFederatedTrustDomains: false + ## @param externalControllerManagers.defaults.className specify to use an explicit class name. If empty, it will be automatically set to Release.Namespace-Release.Name to not conflict with other installs, enabling parallel installs. + className: "" + ## @param externalControllerManagers.defaults.watchClassless specify to process custom resources without class name specified. Useful to slowly migrate to class names from classless installs. Do not have two installs on the same k8s cluster both set to true. + watchClassless: false + ## @param externalControllerManagers.defaults.entryIDPrefixCleanup consult the spiffe.io docs about this option before changing. Its unlikely you will need to ever change it. + entryIDPrefixCleanup: false + ## @param externalControllerManagers.defaults.parentIDTemplate The template that is used to register workloads. + parentIDTemplate: "spiffe://{{ .TrustDomain }}/spire/agent/k8s_psat/{{ .ClusterName }}/{{ .NodeMeta.UID }}" + ## @param externalControllerManagers.defaults.expandEnv Set to true to enable environment variable substitution of config file options + expandEnv: false + ## @param externalControllerManagers.defaults.extraEnv [array] Extra environment variables to add to the controller manager + extraEnv: [] + ## @param externalControllerManagers.defaults.resources [object] Resource requests and limits for controller manager + resources: {} + ## @param externalControllerManagers.defaults.securityContext [object] Security context + securityContext: {} + configMap: + ## @param externalControllerManagers.defaults.configMap.annotations [object] Annotations to add to the Controller Manager ConfigMap + annotations: {} + ## @param externalControllerManagers.defaults.ignoreNamespaces [array] These namespaces are ignored by controller manager + ignoreNamespaces: + - kube-system + - kube-public + - local-path-storage + ## @param externalControllerManagers.defaults.cacheNamespaces [object] If specified restricts the manager's cache to watch objects in the desired namespaces. Defaults to all namespaces. + cacheNamespaces: {} + + ## @param externalControllerManagers.clusters [object] A dictionary of clusters to add with optional overrides. If empty, all clusters defined in kubeConfigs will be used. + clusters: {} + # clustera: + # Should match the name of the config in the kubeConfigs section + # kubeConfigName: foo + # reconcile: + # clusterStaticEntries: true + # other: {} + tools: kubectl: ## @param tools.kubectl.image.registry The OCI registry to pull the image from diff --git a/charts/spire/values.yaml b/charts/spire/values.yaml index c77eb69..ce3b539 100644 --- a/charts/spire/values.yaml +++ b/charts/spire/values.yaml @@ -117,6 +117,9 @@ spire-server: controllerManager: ## @param spire-server.controllerManager.enabled Enable controller manager and provision CRD's enabled: true + externalControllerManagers: + ## @param spire-server.externalControllerManagers.enabled Enable external controller manager support + enabled: true ## @section Spire agent parameters ## Parameter values for Spire agent diff --git a/tests/integration/psat/kind-config.yaml b/tests/integration/psat/child-kind-config.yaml similarity index 81% rename from tests/integration/psat/kind-config.yaml rename to tests/integration/psat/child-kind-config.yaml index d85992a..086e96b 100644 --- a/tests/integration/psat/kind-config.yaml +++ b/tests/integration/psat/child-kind-config.yaml @@ -4,4 +4,4 @@ networking: apiServerAddress: "172.17.0.1" apiServerPort: 7443 podSubnet: "10.245.0.0/16" - serviceSubnet: "10.97.0.0/12" + serviceSubnet: "10.97.0.0/16" diff --git a/tests/integration/psat/child-values.yaml b/tests/integration/psat/child-values.yaml new file mode 100644 index 0000000..63b655d --- /dev/null +++ b/tests/integration/psat/child-values.yaml @@ -0,0 +1,20 @@ +global: + spire: + recommendations: + enabled: true + namespaces: + create: true + clusterName: production + trustDomain: production.other + +spire-server: + enabled: false + +spire-agent: + enabled: false + +spiffe-csi-drvier: + enabled: false + +spiffe-oidc-discovery-provider: + enabled: true diff --git a/tests/integration/psat/other-kind-config.yaml b/tests/integration/psat/other-kind-config.yaml new file mode 100644 index 0000000..0204e75 --- /dev/null +++ b/tests/integration/psat/other-kind-config.yaml @@ -0,0 +1,7 @@ +kind: Cluster +apiVersion: kind.x-k8s.io/v1alpha4 +networking: + apiServerAddress: "172.17.0.1" + apiServerPort: 5443 + podSubnet: "10.246.0.0/16" + serviceSubnet: "10.98.0.0/16" diff --git a/tests/integration/psat/run-tests.sh b/tests/integration/psat/run-tests.sh index 6b7785a..fb302fe 100755 --- a/tests/integration/psat/run-tests.sh +++ b/tests/integration/psat/run-tests.sh @@ -5,7 +5,7 @@ set -xe SCRIPT="$(readlink -f "$0")" SCRIPTPATH="$(dirname "${SCRIPT}")" TESTDIR="${SCRIPTPATH}/../../../.github/tests" -DEPS="${TESTDIR}/dependencies" +#DEPS="${TESTDIR}/dependencies" # shellcheck source=/dev/null source "${SCRIPTPATH}/../../../.github/scripts/parse-versions.sh" @@ -45,24 +45,45 @@ teardown() { trap 'EC=$? && trap - SIGTERM && teardown $EC' SIGINT SIGTERM EXIT -kubectl create namespace spire-system --dry-run=client -o yaml | kubectl apply -f - -kubectl label namespace spire-system pod-security.kubernetes.io/enforce=privileged || true -kubectl create namespace spire-server --dry-run=client -o yaml | kubectl apply -f - -kubectl label namespace spire-server pod-security.kubernetes.io/enforce=restricted || true +#helm upgrade --install --create-namespace spire charts/spire \ +# --namespace spire-root-server \ +# --values "${DEPS}/spire-root-server-values.yaml" \ +# --wait -helm upgrade --install --create-namespace spire charts/spire \ - --namespace spire-root-server \ - --values "${DEPS}/spire-root-server-values.yaml" \ - --wait +kind create cluster --name child --kubeconfig "${SCRIPTPATH}/kubeconfig-child" --config "${SCRIPTPATH}/child-kind-config.yaml" +md5sum "${SCRIPTPATH}/kubeconfig-child" +wc -l "${SCRIPTPATH}/kubeconfig-child" +CHILD_KCB64="$(base64 < "${SCRIPTPATH}/kubeconfig-child" | tr '\n' ' ' | sed 's/ //g')" -kind create cluster --name other --kubeconfig "${SCRIPTPATH}/kubeconfig" --config "${SCRIPTPATH}/kind-config.yaml" -md5sum "${SCRIPTPATH}/kubeconfig" -wc -l "${SCRIPTPATH}/kubeconfig" -KCB64="$(base64 < "${SCRIPTPATH}/kubeconfig" | tr '\n' ' ' | sed 's/ //g')" -kubectl --kubeconfig "${SCRIPTPATH}/kubeconfig" create namespace spire-system -kubectl --kubeconfig "${SCRIPTPATH}/kubeconfig" create configmap -n spire-system spire-bundle-upstream +helm upgrade --kubeconfig "${SCRIPTPATH}/kubeconfig-child" --install --create-namespace --namespace spire-mgmt spire-crds charts/spire-crds +kubectl --kubeconfig "${SCRIPTPATH}/kubeconfig-child" apply -f "${SCRIPTPATH}/sodp-clusterspiffeid.yaml" +helm upgrade --kubeconfig "${SCRIPTPATH}/kubeconfig-child" --install --namespace spire-mgmt --values "${SCRIPTPATH}/child-values.yaml" \ + spire charts/spire +kubectl --kubeconfig "${SCRIPTPATH}/kubeconfig-child" create configmap -n spire-system spire-bundle-upstream + +kind create cluster --name other --kubeconfig "${SCRIPTPATH}/kubeconfig-other" --config "${SCRIPTPATH}/other-kind-config.yaml" +md5sum "${SCRIPTPATH}/kubeconfig-other" +wc -l "${SCRIPTPATH}/kubeconfig-other" +OTHER_KCB64="$(base64 < "${SCRIPTPATH}/kubeconfig-other" | tr '\n' ' ' | sed 's/ //g')" + +helm upgrade --kubeconfig "${SCRIPTPATH}/kubeconfig-other" --install --create-namespace --namespace spire-mgmt spire-crds charts/spire-crds +kubectl --kubeconfig "${SCRIPTPATH}/kubeconfig-other" apply -f "${SCRIPTPATH}/sodp-clusterspiffeid.yaml" +helm upgrade --kubeconfig "${SCRIPTPATH}/kubeconfig-other" --install --namespace spire-mgmt --values "${SCRIPTPATH}/child-values.yaml" \ + spire charts/spire +kubectl --kubeconfig "${SCRIPTPATH}/kubeconfig-other" create configmap -n spire-system spire-bundle-upstream + +helm upgrade --install --create-namespace --namespace spire-mgmt --values "${SCRIPTPATH}/values.yaml" \ + --wait spire charts/spire \ + --set "spire-server.kubeConfigs.child.kubeConfigBase64=${CHILD_KCB64}" \ + --set "spire-server.kubeConfigs.other.kubeConfigBase64=${OTHER_KCB64}" +helm test --namespace spire-mgmt spire +kubectl --kubeconfig "${SCRIPTPATH}/kubeconfig-child" get configmap -n spire-system spire-bundle-upstream +kubectl --kubeconfig "${SCRIPTPATH}/kubeconfig-other" get configmap -n spire-system spire-bundle-upstream + +ENTRIES="$(kubectl exec -i -n spire-server spire-server-0 -- spire-server entry show)" + +if [[ "${ENTRIES}" == "Found 0 entries" ]]; then + echo "${ENTRIES}" + exit 1 +fi -helm upgrade --install --create-namespace --namespace spire-server --values "${SCRIPTPATH}/values.yaml" \ - --wait spire charts/spire --set "spire-server.kubeConfigs.other.kubeConfigBase64=$KCB64" -helm test --namespace spire-server spire -kubectl --kubeconfig "${SCRIPTPATH}/kubeconfig" get configmap -n spire-system spire-bundle-upstream diff --git a/tests/integration/psat/sodp-clusterspiffeid.yaml b/tests/integration/psat/sodp-clusterspiffeid.yaml new file mode 100644 index 0000000..2e3de5f --- /dev/null +++ b/tests/integration/psat/sodp-clusterspiffeid.yaml @@ -0,0 +1,23 @@ +apiVersion: spire.spiffe.io/v1alpha1 +kind: ClusterSPIFFEID +metadata: + name: spire-mgmt-spire-oidc-discovery-provider +spec: + autoPopulateDNSNames: true + className: spire-mgmt-spire + dnsNameTemplates: + - oidc-discovery.{{ .TrustDomain }} + namespaceSelector: + matchExpressions: + - key: kubernetes.io/metadata.name + operator: In + values: + - spire-mgmt + - spire-server + - spire-system + podSelector: + matchLabels: + component: oidc-discovery-provider + release: spire + release-namespace: spire-mgmt + spiffeIDTemplate: spiffe://{{ .TrustDomain }}/ns/{{ .PodMeta.Namespace }}/sa/{{ .PodSpec.ServiceAccountName }} diff --git a/tests/integration/psat/values.yaml b/tests/integration/psat/values.yaml index fbb7b55..1144d71 100644 --- a/tests/integration/psat/values.yaml +++ b/tests/integration/psat/values.yaml @@ -2,11 +2,29 @@ global: spire: recommendations: enabled: true + namespaces: + create: true clusterName: production trustDomain: production.other + caSubject: + country: US + organization: Production + commonName: production.other spire-server: - ca_subject: - country: US - organization: Production - common_name: production.other + controllerManager: + reconcile: + clusterSPIFFEIDs: false + clusterStaticEntries: true + clusterFederatedTrustDomains: true + identities: + clusterSPIFFEIDs: + default: + enabled: false + oidc-discovery-provider: + enabled: false + test-keys: + enabled: false + +spiffe-oidc-discovery-provider: + enabled: false