From 72768ecdbc62df4b445b53d3e41597f0f780c5fa Mon Sep 17 00:00:00 2001 From: kfox1111 Date: Wed, 20 Mar 2024 13:58:43 -0700 Subject: [PATCH] Support hybrid nodes with different attestors or config (#218) * Add direct tpm support for spire-agent Signed-off-by: Kevin Fox * Add fingerprinting support Signed-off-by: Kevin Fox * Add example Signed-off-by: Kevin Fox * Update charts/spire/charts/spire-agent/templates/configmap.yaml Signed-off-by: kfox1111 * Support hybrid nodes with different attestors In some clusters, you may have a mix of nodes with differing attestor needs. For example, some nodes have hardware TPMs and some without. This patch enables configuring multiple daemonsets for the agent that you can target to pools of nodes. Signed-off-by: Kevin Fox * Remove extra slash Signed-off-by: Kevin Fox * Update docs Signed-off-by: Kevin Fox * Update charts/spire/charts/spire-agent/templates/configmap.yaml Signed-off-by: kfox1111 * Fix broken tests Signed-off-by: Kevin Fox * Add daemonset labels Signed-off-by: Kevin Fox * Add temporary upgrade hook Signed-off-by: Kevin Fox * Fix docs Signed-off-by: Kevin Fox * Fix include Signed-off-by: Kevin Fox * Add missing values Signed-off-by: Kevin Fox * Fix perms, add upgrade note Signed-off-by: Kevin Fox * Fix hardcoded nodeAttestor and keyManager in spire-agent Fixes: https://github.com/spiffe/helm-charts-hardened/issues/220 Signed-off-by: Kevin Fox * Fix merge issues Signed-off-by: Kevin Fox * Fix unit tests Signed-off-by: Kevin Fox * Pass the agent's securityContext on to Kubernetes Currently its ignored. Signed-off-by: Kevin Fox * Update example to be usable Signed-off-by: Kevin Fox * Apply suggestions from code review Signed-off-by: kfox1111 * Update example Signed-off-by: Kevin Fox * Incorperate feedback Signed-off-by: Kevin Fox * Incorperate feedback Signed-off-by: Kevin Fox * Fix merge conflict issue Signed-off-by: Kevin Fox * Update to the newest release Signed-off-by: Kevin Fox * Incorperate feedback Signed-off-by: Kevin Fox * Incorperate feedback Signed-off-by: Kevin Fox * Apply suggestions from code review Signed-off-by: kfox1111 * Apply suggestions from code review Co-authored-by: Faisal Memon Signed-off-by: kfox1111 * Fix version numbers in docs Signed-off-by: Kevin Fox * Remove merge conflicted extra code Signed-off-by: Kevin Fox --------- Signed-off-by: Kevin Fox Signed-off-by: kfox1111 Co-authored-by: Faisal Memon --- charts/spire/README.md | 6 +- charts/spire/charts/spire-agent/README.md | 6 ++ .../spire-agent/templates/configmap.yaml | 18 ++++- .../spire-agent/templates/daemonset.yaml | 20 ++++- .../templates/pre-upgrade-hook.yaml | 77 +++++++++++++++++++ charts/spire/charts/spire-agent/values.yaml | 31 ++++++++ examples/tpm-direct/different-nodes.yaml | 22 ++++++ examples/tpm-direct/same-nodes.yaml | 15 ++++ 8 files changed, 191 insertions(+), 4 deletions(-) create mode 100644 charts/spire/charts/spire-agent/templates/pre-upgrade-hook.yaml create mode 100644 examples/tpm-direct/different-nodes.yaml create mode 100644 examples/tpm-direct/same-nodes.yaml diff --git a/charts/spire/README.md b/charts/spire/README.md index 0135835..14a29d5 100644 --- a/charts/spire/README.md +++ b/charts/spire/README.md @@ -75,7 +75,11 @@ kubectl delete crds clusterfederatedtrustdomains.spire.spiffe.io clusterspiffeid We only support upgrading one major version at a time. Version skipping isn't supported. -### 0.18.x +### 0.19.X + +- The spire-agent daemonset gained a new label. For those disabling the upgrade hooks, you need to delete the spire-agent daemonset before issuing the helm upgrade. + +### 0.18.X - SPIRE no longer emits x509UniqueIdentifiers in x509-SVIDS by default. The old behavior can be reenabled with spire-server.credentialComposer.uniqueID.enabled=true. See https://github.com/spiffe/spire/pull/4862 for details. - SPIRE agents will now automatically reattest when they can. The old behavior can be reenabled with spire-agent.disableReattestToRenew=true. See https://github.com/spiffe/spire/pull/4791 for details. diff --git a/charts/spire/charts/spire-agent/README.md b/charts/spire/charts/spire-agent/README.md index 3fdc58f..f3af4ff 100644 --- a/charts/spire/charts/spire-agent/README.md +++ b/charts/spire/charts/spire-agent/README.md @@ -126,6 +126,12 @@ A Helm chart to install the SPIRE agent. | `experimental.enabled` | Allow configuration of experimental features | `false` | | `experimental.syncInterval` | Sync interval with SPIRE server with exponential backoff | `5s` | | `experimental.featureFlags` | List of developer feature flags | `[]` | +| `agents` | Configure multiple agent DaemonSets. Useful when you have different node types and nodeAttestors | `{}` | +| `installAndUpgradeHook.enabled` | Enable Helm hook to autofix common install/upgrade issues (should be disabled when using `helm template`) | `true` | +| `tools.kubectl.image.registry` | The OCI registry to pull the image from | `docker.io` | +| `tools.kubectl.image.repository` | The repository within the registry | `rancher/kubectl` | +| `tools.kubectl.image.pullPolicy` | The image pull policy | `IfNotPresent` | +| `tools.kubectl.image.tag` | Overrides the image tag whose default is the chart appVersion | `""` | | `sockets.hostBasePath` | Path on which the agent socket is made available when admin.mountOnHost is true | `/run/spire/agent/sockets` | | `sockets.admin.enabled` | Enable the admin socket. Useful for admin tasks or the Delegated Identity API. | `false` | | `sockets.admin.mountOnHost` | Enable the admin socket to be visible on the host. | `false` | diff --git a/charts/spire/charts/spire-agent/templates/configmap.yaml b/charts/spire/charts/spire-agent/templates/configmap.yaml index 1fb312b..4de965b 100644 --- a/charts/spire/charts/spire-agent/templates/configmap.yaml +++ b/charts/spire/charts/spire-agent/templates/configmap.yaml @@ -1,3 +1,4 @@ +{{- define "spire-agent.check-config-values" -}} {{- include "spire-lib.check-strict-mode" (list . "clusterName must be set" (eq (include "spire-lib.cluster-name" .) "example-cluster"))}} {{- include "spire-lib.check-strict-mode" (list . "trustDomain must be set" (eq (include "spire-lib.trust-domain" .) "example.org"))}} {{- range $type, $tvals := .Values.customPlugins }} @@ -21,6 +22,7 @@ {{- if hasPrefix (.Values.socketPath | dir | clean) (.Values.sockets.hostBasePath | clean) }} {{- fail "The sockets.hostBasePath can not be located under the socketPath direcotry" }} {{- end }} +{{- end }} {{- define "spire-agent.yaml-config" -}} agent: {{- if .Values.disableReattestToRenew }} @@ -132,10 +134,22 @@ telemetry: port: {{ .Values.telemetry.prometheus.port }} {{- end }} {{- end }} +{{- $root := . }} +{{- range $name := (concat (list "default") (keys .Values.agents)) | uniq }} +{{- with (dict "Release" $root.Release "Chart" $root.Chart "Values" (deepCopy $root.Values)) }} +{{- $nameSuffix := "" }} +{{- if ne $name "default" }} +{{- $nameSuffix = printf "-%s" $name }} +{{- end }} +{{- if hasKey $root.Values.agents $name }} +{{- $_ := set . "Values" (mergeOverwrite .Values (index $root.Values.agents $name)) }} +{{- end }} +{{- include "spire-agent.check-config-values" . }} +--- apiVersion: v1 kind: ConfigMap metadata: - name: {{ include "spire-agent.fullname" . }} + name: {{ include "spire-agent.fullname" . }}{{ $nameSuffix }} namespace: {{ include "spire-agent.namespace" . }} {{- with .Values.configMap.annotations }} annotations: @@ -144,3 +158,5 @@ metadata: data: agent.conf: | {{- include "spire-lib.reformat-and-yaml2json" (dict "config" (include "spire-agent.yaml-config" .) "root" .) | nindent 4 }} +{{- end }} +{{- end }} diff --git a/charts/spire/charts/spire-agent/templates/daemonset.yaml b/charts/spire/charts/spire-agent/templates/daemonset.yaml index d94cd96..db8fe9d 100644 --- a/charts/spire/charts/spire-agent/templates/daemonset.yaml +++ b/charts/spire/charts/spire-agent/templates/daemonset.yaml @@ -1,4 +1,14 @@ {{- $configSum := (include (print $.Template.BasePath "/configmap.yaml") . | sha256sum) }} +{{- $root := . }} +{{- range $name := (concat (list "default") (keys .Values.agents)) | uniq }} +{{- with (dict "Release" $root.Release "Chart" $root.Chart "Values" (deepCopy $root.Values)) }} +{{- $nameSuffix := "" }} +{{- if ne $name "default" }} +{{- $nameSuffix = printf "-%s" $name }} +{{- end }} +{{- if hasKey $root.Values.agents $name }} +{{- $_ := set . "Values" (mergeOverwrite .Values (index $root.Values.agents $name)) }} +{{- end }} {{- $podSecurityContext := fromYaml (include "spire-lib.podsecuritycontext" .) }} {{- $mainSecurityContext := deepCopy .Values.securityContext }} {{- if .Values.nodeAttestor.tpmDirect.enabled }} @@ -8,17 +18,20 @@ {{- $cbh := eq (include "spire-agent.connect-by-hostname" .) "true" }} {{- $socketAlternateNames := index (include "spire-agent.socket-alternate-names" . | fromYaml) "names" }} {{- $socketPath := include "spire-agent.socket-path" . }} +--- apiVersion: apps/v1 kind: DaemonSet metadata: - name: {{ include "spire-agent.fullname" . }} + name: {{ include "spire-agent.fullname" . }}{{ $nameSuffix }} namespace: {{ include "spire-agent.namespace" . }} labels: {{- include "spire-agent.labels" . | nindent 4 }} + app.kubernetes.io/component: {{ $name }} spec: selector: matchLabels: {{- include "spire-agent.selectorLabels" . | nindent 6 }} + app.kubernetes.io/component: {{ $name }} {{- with .Values.updateStrategy }} updateStrategy: {{- if not (has .type (list "RollingUpdate" "OnDelete")) }} @@ -40,6 +53,7 @@ spec: {{- end }} labels: {{- include "spire-agent.selectorLabels" . | nindent 8 }} + app.kubernetes.io/component: {{ $name }} {{- with .Values.podLabels }} {{- toYaml . | nindent 8 }} {{- end }} @@ -257,7 +271,7 @@ spec: {{- if eq (len .Values.trustBundleURL) 0 }} - name: spire-bundle configMap: - name: {{ include "spire-lib.bundle-configmap" . }} + name: {{ include "spire-lib.bundle-configmap" . }}{{ $nameSuffix }} {{- end }} {{- if .Values.nodeAttestor.tpmDirect.enabled }} - name: tpm-direct @@ -287,3 +301,5 @@ spec: {{- if gt (len .Values.extraVolumes) 0 }} {{- toYaml .Values.extraVolumes | nindent 8 }} {{- end }} +{{- end }} +{{- end }} diff --git a/charts/spire/charts/spire-agent/templates/pre-upgrade-hook.yaml b/charts/spire/charts/spire-agent/templates/pre-upgrade-hook.yaml new file mode 100644 index 0000000..e28c2bc --- /dev/null +++ b/charts/spire/charts/spire-agent/templates/pre-upgrade-hook.yaml @@ -0,0 +1,77 @@ +{{- if eq ((dig "installAndUpgradeHooks" "enabled" .Values.installAndUpgradeHook.enabled .Values.global) | toString) "true" }} +{{- $ds := lookup "apps/v1" "DaemonSet" (include "spire-agent.namespace" .) (include "spire-agent.fullname" .) }} +{{- if and $ds (not (hasKey $ds.metadata.labels "app.kubernetes.io/component")) }} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "spire-agent.serviceAccountName" . }}-pre-upgrade + namespace: {{ include "spire-agent.namespace" . }} + labels: + {{- include "spire-agent.labels" . | nindent 4 }} + annotations: + "helm.sh/hook": pre-upgrade + "helm.sh/hook-delete-policy": before-hook-creation, hook-succeeded, hook-failed +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ include "spire-agent.fullname" . }}-pre-upgrade + namespace: {{ include "spire-agent.namespace" . }} + annotations: + "helm.sh/hook": pre-upgrade + "helm.sh/hook-delete-policy": before-hook-creation, hook-succeeded, hook-failed +rules: + - apiGroups: ["apps"] + resources: ["daemonsets"] + resourceNames: [{{ include "spire-agent.fullname" . | quote }}] + verbs: ["get", "delete"] +--- +kind: RoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ include "spire-agent.fullname" . }}-pre-upgrade + namespace: {{ include "spire-agent.namespace" . }} + annotations: + "helm.sh/hook": pre-upgrade + "helm.sh/hook-delete-policy": before-hook-creation, hook-succeeded, hook-failed +subjects: + - kind: ServiceAccount + name: {{ include "spire-agent.serviceAccountName" . }}-pre-upgrade + namespace: {{ include "spire-agent.namespace" . }} +roleRef: + kind: Role + name: {{ include "spire-agent.fullname" . }}-pre-upgrade + apiGroup: rbac.authorization.k8s.io +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ include "spire-agent.fullname" . }}-pre-upgrade + namespace: {{ include "spire-agent.namespace" . }} + labels: + {{- include "spire-agent.labels" . | nindent 4 }} + annotations: + "helm.sh/hook": pre-upgrade + "helm.sh/hook-delete-policy": before-hook-creation, hook-succeeded, hook-failed +spec: + template: + metadata: + name: {{ include "spire-agent.fullname" . }}-pre-upgrade + spec: + restartPolicy: Never + serviceAccountName: {{ include "spire-agent.serviceAccountName" . }}-pre-upgrade + securityContext: + {{- include "spire-lib.podsecuritycontext" . | nindent 8 }} + containers: + - name: pre-upgrade + securityContext: + {{- include "spire-lib.securitycontext" . | nindent 10 }} + image: {{ template "spire-lib.kubectl-image" (dict "appVersion" $.Chart.AppVersion "image" .Values.tools.kubectl.image "global" .Values.global "KubeVersion" .Capabilities.KubeVersion.Version) }} + args: + - delete + - daemonset + - {{ include "spire-agent.fullname" . }} + - -n + - {{ include "spire-agent.namespace" . }} +{{- end }} +{{- end }} diff --git a/charts/spire/charts/spire-agent/values.yaml b/charts/spire/charts/spire-agent/values.yaml index 030292a..ea10d12 100644 --- a/charts/spire/charts/spire-agent/values.yaml +++ b/charts/spire/charts/spire-agent/values.yaml @@ -328,6 +328,37 @@ experimental: ## @param experimental.featureFlags [array] List of developer feature flags featureFlags: [] +## @param agents Configure multiple agent DaemonSets. Useful when you have different node types and nodeAttestors +agents: {} +# default: +# nodeSelector: +# tpm: without +# tpm: +# nodeSelector: +# tpm: with +# nodeAttestor: +# k8sPsat: +# enabled: false +# tpmDirect: +# enabled: true + +installAndUpgradeHook: + ## @param installAndUpgradeHook.enabled Enable Helm hook to autofix common install/upgrade issues (should be disabled when using `helm template`) + enabled: true + +tools: + kubectl: + ## @param tools.kubectl.image.registry The OCI registry to pull the image from + ## @param tools.kubectl.image.repository The repository within the registry + ## @param tools.kubectl.image.pullPolicy The image pull policy + ## @param tools.kubectl.image.tag Overrides the image tag whose default is the chart appVersion + ## + image: + registry: docker.io + repository: rancher/kubectl + pullPolicy: IfNotPresent + tag: "" + sockets: ## @param sockets.hostBasePath Path on which the agent socket is made available when admin.mountOnHost is true hostBasePath: /run/spire/agent/sockets diff --git a/examples/tpm-direct/different-nodes.yaml b/examples/tpm-direct/different-nodes.yaml new file mode 100644 index 0000000..86d913e --- /dev/null +++ b/examples/tpm-direct/different-nodes.yaml @@ -0,0 +1,22 @@ +spire-agent: + agents: + default: + nodeSelector: + tpm: without + tpm: + nodeSelector: + tpm: with + nodeAttestor: + k8sPsat: + enabled: false + tpmDirect: + enabled: true + +spire-server: + controllerManager: + # K8s labels have a 63 character limit. TPM hashes are 64 chars. So you need to label the node with two labels with half of the tpm's hash each. The 'node-restriction.kubernetes.io/' prefix is so that the + # nodes can't update the hash themselves, an important security constraint. + parentIDTemplate: 'spiffe://{{ .TrustDomain }}/spire/agent/{{if index .NodeMeta.Labels "node-restriction.kubernetes.io/tpm-pubhash"}}tpm/{{ index .NodeMeta.Labels "node-restriction.kubernetes.io/tpm-pubhash" }}{{ index .NodeMeta.Labels "node-restriction.kubernetes.io/tpm-pubhash2" }}{{ else }}spire/agent/k8s_psat/{{ .ClusterName }}/{{ .NodeMeta.UID }}{{ end }}' + nodeAttestor: + tpmDirect: + enabled: true diff --git a/examples/tpm-direct/same-nodes.yaml b/examples/tpm-direct/same-nodes.yaml new file mode 100644 index 0000000..ab53a82 --- /dev/null +++ b/examples/tpm-direct/same-nodes.yaml @@ -0,0 +1,15 @@ +spire-agent: + nodeAttestor: + k8sPsat: + enabled: false + tpmDirect: + enabled: true + +spire-server: + controllerManager: + # K8s labels have a 63 character limit. TPM hashes are 64 chars. So you need to label the node with two labels with half of the tpm's hash each. The 'node-restriction.kubernetes.io/' prefix is so that the + # nodes can't update the hash themselves, an important security constraint. + parentIDTemplate: 'spiffe://{{ .TrustDomain }}/spire/agent/tpm/{{ index .NodeMeta.Labels "node-restriction.kubernetes.io/tpm-pubhash" }}{{ index .NodeMeta.Labels "node-restriction.kubernetes.io/tpm-pubhash2" }}' + nodeAttestor: + tpmDirect: + enabled: true