Move the k8s-workload-registrar back into same pod as spire-server

The k8s-workload-registrar needs access to the spire-server socket and
therefore has to run in the same pod to be able to mount the socket
into the container.

Signed-off-by: Marco Franssen <[email protected]>
Signed-off-by: Marco Franssen <[email protected]>

charts/spire/Chart.yaml

@@ -52,7 +52,3 @@ dependencies:
     condition: spiffe-oidc-discovery-provider.enabled
     repository: file://./charts/spiffe-oidc-discovery-provider
     version: 0.1.0
-  - name: k8s-workload-registrar
-    condition: k8s-workload-registrar.enabled
-    repository: file://./charts/k8s-workload-registrar
-    version: 0.1.0

charts/spire/README.md

@@ -46,7 +46,6 @@ Kubernetes: `>=1.21.0-0`
 
 | Repository | Name | Version |
 |------------|------|---------|
-| file://./charts/k8s-workload-registrar | k8s-workload-registrar | 0.1.0 |
 | file://./charts/spiffe-csi-driver | spiffe-csi-driver | 0.1.0 |
 | file://./charts/spiffe-oidc-discovery-provider | spiffe-oidc-discovery-provider | 0.1.0 |
 | file://./charts/spire-agent | spire-agent | 0.1.0 |
@@ -57,10 +56,6 @@ Kubernetes: `>=1.21.0-0`
 | Key | Type | Default | Description |
 |-----|------|---------|-------------|
 | fullnameOverride | string | `""` |  |
-| k8s-workload-registrar.clusterName | string | `"example-cluster"` |  |
-| k8s-workload-registrar.enabled | bool | `true` |  |
-| k8s-workload-registrar.server.socketPath | string | `"/run/spire/server-sockets/spire-server.sock"` |  |
-| k8s-workload-registrar.trustDomain | string | `"example.org"` |  |
 | nameOverride | string | `""` |  |
 | spiffe-csi-driver.agentSocketPath | string | `"/run/spire/agent-sockets/spire-agent.sock"` |  |
 | spiffe-oidc-discovery-provider.enabled | bool | `false` |  |

charts/spire/charts/k8s-workload-registrar/.helmignore

@@ -1,23 +0,0 @@
-# Patterns to ignore when building packages.
-# This supports shell glob matching, relative path matching, and
-# negation (prefixed with !). Only one pattern per line.
-.DS_Store
-# Common VCS dirs
-.git/
-.gitignore
-.bzr/
-.bzrignore
-.hg/
-.hgignore
-.svn/
-# Common backup files
-*.swp
-*.bak
-*.tmp
-*.orig
-*~
-# Various IDEs
-.project
-.idea/
-*.tmproj
-.vscode/

charts/spire/charts/k8s-workload-registrar/Chart.yaml

@@ -1,6 +0,0 @@
-apiVersion: v2
-name: k8s-workload-registrar
-description: A Helm chart to install the SPIRE k8s workload registrator.
-type: application
-version: 0.1.0
-appVersion: "1.5.3"

charts/spire/charts/k8s-workload-registrar/templates/NOTES.txt

@@ -1 +0,0 @@
-Installed {{ include "k8s-workload-registrar.fullname" . }}…

charts/spire/charts/k8s-workload-registrar/templates/_helpers.tpl

@@ -1,74 +0,0 @@
-{{/*
-Expand the name of the chart.
-*/}}
-{{- define "k8s-workload-registrar.name" -}}
-{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }}
-{{- end }}
-
-{{/*
-Create a default fully qualified app name.
-We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
-If release name contains chart name it will be used as a full name.
-*/}}
-{{- define "k8s-workload-registrar.fullname" -}}
-{{- if .Values.fullnameOverride }}
-{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }}
-{{- else }}
-{{- $name := default .Chart.Name .Values.nameOverride }}
-{{- if contains $name .Release.Name }}
-{{- .Release.Name | trunc 63 | trimSuffix "-" }}
-{{- else }}
-{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }}
-{{- end }}
-{{- end }}
-{{- end }}
-
-{{/*
-Create chart name and version as used by the chart label.
-*/}}
-{{- define "k8s-workload-registrar.chart" -}}
-{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }}
-{{- end }}
-
-{{/*
-Common labels
-*/}}
-{{- define "k8s-workload-registrar.labels" -}}
-helm.sh/chart: {{ include "k8s-workload-registrar.chart" . }}
-{{ include "k8s-workload-registrar.selectorLabels" . }}
-{{- if .Chart.AppVersion }}
-app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
-{{- end }}
-app.kubernetes.io/managed-by: {{ .Release.Service }}
-{{- end }}
-
-{{/*
-Selector labels
-*/}}
-{{- define "k8s-workload-registrar.selectorLabels" -}}
-app.kubernetes.io/name: {{ include "k8s-workload-registrar.name" . }}
-app.kubernetes.io/instance: {{ .Release.Name }}
-{{- end }}
-
-{{/*
-Create the name of the service account to use
-*/}}
-{{- define "k8s-workload-registrar.serviceAccountName" -}}
-{{- if .Values.serviceAccount.create }}
-{{- default (include "k8s-workload-registrar.fullname" .) .Values.serviceAccount.name }}
-{{- else }}
-{{- default "default" .Values.serviceAccount.name }}
-{{- end }}
-{{- end }}
-
-{{- define "k8s-workload-registrar.image" -}}
-{{- if eq (substr 0 7 .image.version) "sha256:" -}}
-{{- printf "%s/%s@%s" .image.registry .image.repository .image.version -}}
-{{- else if .appVersion -}}
-{{- printf "%s/%s:%s" .image.registry .image.repository (default .appVersion .image.version) -}}
-{{- else if .image.version -}}
-{{- printf "%s/%s:%s" .image.registry .image.repository .image.version -}}
-{{- else -}}
-{{- printf "%s/%s" .image.registry .image.repository -}}
-{{- end -}}
-{{- end }}

charts/spire/charts/k8s-workload-registrar/templates/deployment.yaml

@@ -1,77 +0,0 @@
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  name: {{ include "k8s-workload-registrar.fullname" . }}
-  labels:
-    {{- include "k8s-workload-registrar.labels" . | nindent 4 }}
-spec:
-  {{- if not .Values.autoscaling.enabled }}
-  replicas: {{ .Values.replicaCount }}
-  {{- end }}
-  selector:
-    matchLabels:
-      {{- include "k8s-workload-registrar.selectorLabels" . | nindent 6 }}
-  template:
-    metadata:
-      {{- with .Values.podAnnotations }}
-      annotations:
-        {{- toYaml . | nindent 8 }}
-      {{- end }}
-      labels:
-        {{- include "k8s-workload-registrar.selectorLabels" . | nindent 8 }}
-    spec:
-      {{- with .Values.imagePullSecrets }}
-      imagePullSecrets:
-        {{- toYaml . | nindent 8 }}
-      {{- end }}
-      serviceAccountName: {{ include "k8s-workload-registrar.serviceAccountName" . }}
-      securityContext:
-        {{- toYaml .Values.podSecurityContext | nindent 8 }}
-      initContainers:
-        - name: init
-          # This is a small image with wait-for-it, choose whatever image
-          # you prefer that waits for a service to be up. This image is built
-          # from https://github.com/vishnubob/wait-for-it
-          image: {{ template "k8s-workload-registrar.image" .Values.waitForIt }}
-          imagePullPolicy: {{ .Values.waitForIt.image.pullPolicy }}
-          args: ["-t", "30", "-h", {{ .Values.server.host | quote }}, "-p", {{ .Values.server.port | quote }}]
-          resources:
-            {{- toYaml .Values.waitForIt.resources | nindent 12 }}
-      containers:
-        - name: {{ .Chart.Name }}
-          securityContext:
-            {{- toYaml .Values.securityContext | nindent 12 }}
-          image: {{ template "k8s-workload-registrar.image" (dict "appVersion" $.Chart.AppVersion "image" .Values.image) }}
-          imagePullPolicy: {{ .Values.image.pullPolicy }}
-          args:
-            - -config
-            - /run/spire/k8s-workload-registrar/config/workload-registrar.conf
-          resources:
-            {{- toYaml .Values.resources | nindent 12 }}
-          volumeMounts:
-            - name: spire-server-socket
-              mountPath: /run/spire/server-sockets
-              readOnly: true
-            - name: k8s-workload-registrar-config
-              mountPath: /run/spire/k8s-workload-registrar/config
-              readOnly: true
-      {{- with .Values.nodeSelector }}
-      nodeSelector:
-        {{- toYaml . | nindent 8 }}
-      {{- end }}
-      {{- with .Values.affinity }}
-      affinity:
-        {{- toYaml . | nindent 8 }}
-      {{- end }}
-      {{- with .Values.tolerations }}
-      tolerations:
-        {{- toYaml . | nindent 8 }}
-      {{- end }}
-      volumes:
-        - name: k8s-workload-registrar-config
-          configMap:
-            name: {{ include "k8s-workload-registrar.fullname" . }}
-        - name: spire-server-socket
-          hostPath:
-            path: /run/spire/server-sockets
-            type: DirectoryOrCreate

charts/spire/charts/k8s-workload-registrar/templates/hpa.yaml

@@ -1,28 +0,0 @@
-{{- if .Values.autoscaling.enabled }}
-apiVersion: autoscaling/v2beta1
-kind: HorizontalPodAutoscaler
-metadata:
-  name: {{ include "k8s-workload-registrar.fullname" . }}
-  labels:
-    {{- include "k8s-workload-registrar.labels" . | nindent 4 }}
-spec:
-  scaleTargetRef:
-    apiVersion: apps/v1
-    kind: Deployment
-    name: {{ include "k8s-workload-registrar.fullname" . }}
-  minReplicas: {{ .Values.autoscaling.minReplicas }}
-  maxReplicas: {{ .Values.autoscaling.maxReplicas }}
-  metrics:
-    {{- if .Values.autoscaling.targetCPUUtilizationPercentage }}
-    - type: Resource
-      resource:
-        name: cpu
-        targetAverageUtilization: {{ .Values.autoscaling.targetCPUUtilizationPercentage }}
-    {{- end }}
-    {{- if .Values.autoscaling.targetMemoryUtilizationPercentage }}
-    - type: Resource
-      resource:
-        name: memory
-        targetAverageUtilization: {{ .Values.autoscaling.targetMemoryUtilizationPercentage }}
-    {{- end }}
-{{- end }}

charts/spire/charts/k8s-workload-registrar/templates/roles.yaml

@@ -1,73 +0,0 @@
-kind: Role
-apiVersion: rbac.authorization.k8s.io/v1
-metadata:
-  name: {{ include "k8s-workload-registrar.fullname" . }}
-  namespace: {{ .Release.Namespace }}
-rules:
-  - apiGroups: [coordination.k8s.io]
-    resources: [leases]
-    verbs: [create]
-  - apiGroups: [coordination.k8s.io]
-    resources: [leases]
-    resourceNames: [spire-k8s-registrar-leader-election]
-    verbs:
-      - get
-      - update
-  # - apiGroups: [""]
-  #   resources: [configmaps]
-  #   verbs: [create]
-  # - apiGroups: [""]
-  #   resources: [configmaps]
-  #   resourceNames: [k8s-workload-registrar-leader-election]
-  #   verbs:
-  #     - get
-  #     - update
-  - apiGroups: [""]
-    resources: [events]
-    verbs: [create]
----
-kind: RoleBinding
-apiVersion: rbac.authorization.k8s.io/v1
-metadata:
-  name: {{ include "k8s-workload-registrar.fullname" . }}
-  namespace: {{ .Release.Namespace }}
-subjects:
-  - kind: ServiceAccount
-    name: {{ include "k8s-workload-registrar.serviceAccountName" . }}
-    namespace: {{ .Release.Namespace }}
-roleRef:
-  kind: Role
-  name: {{ include "k8s-workload-registrar.fullname" . }}
-  apiGroup: rbac.authorization.k8s.io
----
-kind: ClusterRole
-apiVersion: rbac.authorization.k8s.io/v1
-metadata:
-  name: {{ include "k8s-workload-registrar.fullname" . }}
-rules:
-  - apiGroups: [""]
-    resources: [pods, nodes, endpoints]
-    verbs:
-      - get
-      - list
-      - watch
-  - apiGroups: [coordination.k8s.io]
-    resources: [leases]
-    verbs:
-      - create
-      - get
-      - list
-      - update
----
-kind: ClusterRoleBinding
-apiVersion: rbac.authorization.k8s.io/v1
-metadata:
-  name: {{ include "k8s-workload-registrar.fullname" . }}
-subjects:
-  - kind: ServiceAccount
-    name: {{ include "k8s-workload-registrar.serviceAccountName" . }}
-    namespace: {{ .Release.Namespace }}
-roleRef:
-  kind: ClusterRole
-  name: {{ include "k8s-workload-registrar.fullname" . }}
-  apiGroup: rbac.authorization.k8s.io

charts/spire/charts/k8s-workload-registrar/templates/serviceaccount.yaml

@@ -1,12 +0,0 @@
-{{- if .Values.serviceAccount.create -}}
-apiVersion: v1
-kind: ServiceAccount
-metadata:
-  name: {{ include "k8s-workload-registrar.serviceAccountName" . }}
-  labels:
-    {{- include "k8s-workload-registrar.labels" . | nindent 4 }}
-  {{- with .Values.serviceAccount.annotations }}
-  annotations:
-    {{- toYaml . | nindent 4 }}
-  {{- end }}
-{{- end }}

charts/spire/charts/k8s-workload-registrar/values.yaml

@@ -1,80 +0,0 @@
-# Default values for k8s-workload-registrar.
-# This is a YAML-formatted file.
-# Declare variables to be passed into your templates.
-
-replicaCount: 1
-
-image:
-  registry: gcr.io
-  repository: spiffe-io/k8s-workload-registrar
-  pullPolicy: IfNotPresent
-  # Overrides the image tag whose default is the chart appVersion.
-  version: ""
-
-imagePullSecrets: []
-nameOverride: ""
-fullnameOverride: ""
-
-serviceAccount:
-  # Specifies whether a service account should be created
-  create: true
-  # Annotations to add to the service account
-  annotations: {}
-  # The name of the service account to use.
-  # If not set and create is true, a name is generated using the fullname template
-  name: ""
-
-podAnnotations: {}
-
-podSecurityContext: {}
-  # fsGroup: 2000
-
-securityContext: {}
-  # capabilities:
-  #   drop:
-  #   - ALL
-  # readOnlyRootFilesystem: true
-  # runAsNonRoot: true
-  # runAsUser: 1000
-
-resources: {}
-  # We usually recommend not to specify default resources and to leave this as a conscious
-  # choice for the user. This also increases chances charts run on environments with little
-  # resources, such as Minikube. If you do want to specify resources, uncomment the following
-  # lines, adjust them as necessary, and remove the curly braces after 'resources:'.
-  # limits:
-  #   cpu: 100m
-  #   memory: 128Mi
-  # requests:
-  #   cpu: 100m
-  #   memory: 128Mi
-
-autoscaling:
-  enabled: false
-  minReplicas: 1
-  maxReplicas: 5
-  targetCPUUtilizationPercentage: 80
-  targetMemoryUtilizationPercentage: 80
-
-nodeSelector:
-  kubernetes.io/arch: amd64
-
-tolerations: []
-
-affinity: {}
-
-server:
-  socketPath: /run/spire/server-sockets/spire-server.sock
-  host: spire-server
-  port: 8081
-clusterName: "example-cluster"
-trustDomain: "example.org"
-logLevel: info
-
-waitForIt:
-  image:
-    registry: cgr.dev
-    repository: chainguard/wait-for-it
-    pullPolicy: IfNotPresent
-    version: latest-20221223
-  resources: {}

charts/spire/charts/spire-server/README.md

@@ -29,6 +29,14 @@ A Helm chart to install the SPIRE server.
 | image.version | string | `""` |  |
 | imagePullSecrets | list | `[]` |  |
 | jwtIssuer | string | `"oidc-discovery.example.org"` |  |
+| k8sWorkloadRegistrar.enabled | bool | `true` |  |
+| k8sWorkloadRegistrar.image.pullPolicy | string | `"IfNotPresent"` |  |
+| k8sWorkloadRegistrar.image.registry | string | `"gcr.io"` |  |
+| k8sWorkloadRegistrar.image.repository | string | `"spiffe-io/k8s-workload-registrar"` |  |
+| k8sWorkloadRegistrar.image.version | string | `""` |  |
+| k8sWorkloadRegistrar.logLevel | string | `"info"` |  |
+| k8sWorkloadRegistrar.resources | object | `{}` |  |
+| k8sWorkloadRegistrar.securityContext | object | `{}` |  |
 | logLevel | string | `"info"` |  |
 | nameOverride | string | `""` |  |
 | nodeSelector."kubernetes.io/arch" | string | `"amd64"` |  |

charts/spire/charts/spire-server/templates/k8s-workload-registrar-configmap.yaml

@@ -1,14 +1,16 @@
+{{- if eq (.Values.k8sWorkloadRegistrar.enabled | toString) "true" }}
 apiVersion: v1
 kind: ConfigMap
 metadata:
-  name: {{ include "k8s-workload-registrar.fullname" . }}
+  name: {{ include "spire-server.fullname" . }}-k8s-workload-registrar
   namespace: {{ .Release.Namespace }}
 data:
   workload-registrar.conf: |
-    log_level       = "{{ .Values.logLevel }}"
+    log_level       = "{{ .Values.k8sWorkloadRegistrar.logLevel }}"
     mode            = "reconcile"
     trust_domain    = {{ .Values.trustDomain | quote }}
     cluster         = {{ .Values.clusterName | quote }}
-    server_address  = "unix://{{ .Values.server.socketPath }}"
+    server_address  = "unix://{{ .Values.socketPath }}"
     leader_election = true
     metrics_addr    = "0.0.0.0:18080"
+{{- end }}

charts/spire/charts/spire-server/templates/k8s-workload-registrar-roles.yaml

@@ -0,0 +1,56 @@
+{{- if eq (.Values.k8sWorkloadRegistrar.enabled | toString) "true" }}
+# This is copied directly from the spire/support/k8s/k8s-workload-registrar tree.
+# These roles are needed for the k8s registrar to work properly in reconciling mode.
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: {{ include "spire-server.fullname" . }}-k8s-workload-registrar
+rules:
+  - apiGroups: [""]
+    resources: ["pods", "nodes", "endpoints"]
+    verbs: ["get", "list", "watch"]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: {{ include "spire-server.fullname" . }}-k8s-workload-registrar
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: {{ include "spire-server.fullname" . }}-k8s-workload-registrar
+subjects:
+  - kind: ServiceAccount
+    name: {{ include "spire-server.serviceAccountName" . }}
+    namespace: {{ .Release.Namespace }}
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: {{ include "spire-server.fullname" . }}-k8s-workload-registrar
+  namespace: {{ .Release.Namespace }}
+rules:
+  - apiGroups: [""]
+    resources: ["configmaps"]
+    verbs: ["create"]
+  - apiGroups: [""]
+    resources: ["configmaps"]
+    resourceNames: ["controller-leader-election-helper"]
+    verbs: ["update", "get"]
+  - apiGroups: [""]
+    resources: ["events"]
+    verbs: ["create"]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: {{ include "spire-server.fullname" . }}-k8s-workload-registrar
+  namespace: {{ .Release.Namespace }}
+roleRef:
+  kind: Role
+  name: {{ include "spire-server.fullname" . }}-k8s-workload-registrar
+  apiGroup: rbac.authorization.k8s.io
+subjects:
+  - kind: ServiceAccount
+    name: {{ include "spire-server.serviceAccountName" . }}
+    namespace: {{ .Release.Namespace }}
+{{- end }}

charts/spire/charts/spire-server/templates/roles.yaml

@@ -46,11 +46,14 @@ rules:
     resources: [tokenreviews]
     verbs:
       - get
+      - watch
+      - list
       - create
   - apiGroups: [""]
-    resources: [nodes]
+    resources: [nodes, pods]
     verbs:
       - get
+      - list
 ---
 # Binds above cluster role to spire-server service account
 kind: ClusterRoleBinding

charts/spire/charts/spire-server/templates/statefulset.yaml

@@ -76,6 +76,25 @@ spec:
               mountPath: /run/spire/upstream_ca
               readOnly: false
             {{ end }}
+        {{- if eq (.Values.k8sWorkloadRegistrar.enabled | toString) "true" }}
+        - name: spire-k8s-workload-registrar
+          securityContext:
+            {{- toYaml .Values.k8sWorkloadRegistrar.securityContext | nindent 12 }}
+          image: {{ template "spire-server.image" (dict "appVersion" $.Chart.AppVersion "image" .Values.k8sWorkloadRegistrar.image) }}
+          imagePullPolicy: {{ .Values.image.pullPolicy }}
+          args:
+            - -config
+            - /run/spire/k8s-workload-registrar/config/workload-registrar.conf
+          resources:
+            {{- toYaml .Values.k8sWorkloadRegistrar.resources | nindent 12 }}
+          volumeMounts:
+            - name: spire-server-socket
+              mountPath: /run/spire/server-sockets
+              readOnly: true
+            - name: k8s-workload-registrar-config
+              mountPath: /run/spire/k8s-workload-registrar/config
+              readOnly: true
+        {{- end }}
       {{- with .Values.nodeSelector }}
       nodeSelector:
         {{- toYaml . | nindent 8 }}
@@ -105,6 +124,11 @@ spec:
           secret:
             secretName: {{ include "spire-server.upstream-ca-secret" . }}
         {{- end }}
+        {{- if eq (.Values.k8sWorkloadRegistrar.enabled | toString) "true" }}
+        - name: k8s-workload-registrar-config
+          configMap:
+            name: {{ include "spire-server.fullname" . }}-k8s-workload-registrar
+        {{- end }}
   volumeClaimTemplates:
     {{- if eq (.Values.dataStorage.enabled | toString) "true" }}
     - metadata:

charts/spire/charts/spire-server/values.yaml

@@ -105,3 +105,35 @@ upstreamAuthority:
         certificate: ""
         key: ""
         bundle: ""
+
+k8sWorkloadRegistrar:
+  enabled: true
+
+  image:
+    registry: gcr.io
+    repository: spiffe-io/k8s-workload-registrar
+    pullPolicy: IfNotPresent
+    # Overrides the image tag whose default is the chart appVersion.
+    version: ""
+
+  resources: {}
+    # We usually recommend not to specify default resources and to leave this as a conscious
+    # choice for the user. This also increases chances charts run on environments with little
+    # resources, such as Minikube. If you do want to specify resources, uncomment the following
+    # lines, adjust them as necessary, and remove the curly braces after 'resources:'.
+    # limits:
+    #   cpu: 100m
+    #   memory: 128Mi
+    # requests:
+    #   cpu: 100m
+    #   memory: 128Mi
+
+  securityContext: {}
+    # capabilities:
+    #   drop:
+    #   - ALL
+    # readOnlyRootFilesystem: true
+    # runAsNonRoot: true
+    # runAsUser: 1000
+
+  logLevel: info

charts/spire/values.yaml

@@ -26,12 +26,3 @@ spiffe-csi-driver:
 spiffe-oidc-discovery-provider:
   enabled: false
   trustDomain: *trustDomain
-
-k8s-workload-registrar:
-  enabled: true
-
-  clusterName: *clusterName
-  trustDomain: *trustDomain
-
-  server:
-    socketPath: *serverSocketPath