Move the k8s-workload-registrar back into same pod as spire-server

The k8s-workload-registrar needs access to the spire-server socket and
therefore has to run in the same pod to be able to mount the socket
into the container.

Signed-off-by: Marco Franssen <[email protected]>
Signed-off-by: Marco Franssen <[email protected]>
This commit is contained in:
Marco Franssen
2023-02-18 13:04:11 +01:00
committed by Marco Franssen
parent 738842de16
commit 0a0ddb4a9b
18 changed files with 129 additions and 396 deletions
-4
View File
@@ -52,7 +52,3 @@ dependencies:
condition: spiffe-oidc-discovery-provider.enabled
repository: file://./charts/spiffe-oidc-discovery-provider
version: 0.1.0
- name: k8s-workload-registrar
condition: k8s-workload-registrar.enabled
repository: file://./charts/k8s-workload-registrar
version: 0.1.0
-5
View File
@@ -46,7 +46,6 @@ Kubernetes: `>=1.21.0-0`
| Repository | Name | Version |
|------------|------|---------|
| file://./charts/k8s-workload-registrar | k8s-workload-registrar | 0.1.0 |
| file://./charts/spiffe-csi-driver | spiffe-csi-driver | 0.1.0 |
| file://./charts/spiffe-oidc-discovery-provider | spiffe-oidc-discovery-provider | 0.1.0 |
| file://./charts/spire-agent | spire-agent | 0.1.0 |
@@ -57,10 +56,6 @@ Kubernetes: `>=1.21.0-0`
| Key | Type | Default | Description |
|-----|------|---------|-------------|
| fullnameOverride | string | `""` | |
| k8s-workload-registrar.clusterName | string | `"example-cluster"` | |
| k8s-workload-registrar.enabled | bool | `true` | |
| k8s-workload-registrar.server.socketPath | string | `"/run/spire/server-sockets/spire-server.sock"` | |
| k8s-workload-registrar.trustDomain | string | `"example.org"` | |
| nameOverride | string | `""` | |
| spiffe-csi-driver.agentSocketPath | string | `"/run/spire/agent-sockets/spire-agent.sock"` | |
| spiffe-oidc-discovery-provider.enabled | bool | `false` | |
@@ -1,23 +0,0 @@
# Patterns to ignore when building packages.
# This supports shell glob matching, relative path matching, and
# negation (prefixed with !). Only one pattern per line.
.DS_Store
# Common VCS dirs
.git/
.gitignore
.bzr/
.bzrignore
.hg/
.hgignore
.svn/
# Common backup files
*.swp
*.bak
*.tmp
*.orig
*~
# Various IDEs
.project
.idea/
*.tmproj
.vscode/
@@ -1,6 +0,0 @@
apiVersion: v2
name: k8s-workload-registrar
description: A Helm chart to install the SPIRE k8s workload registrator.
type: application
version: 0.1.0
appVersion: "1.5.3"
@@ -1 +0,0 @@
Installed {{ include "k8s-workload-registrar.fullname" . }}…
@@ -1,74 +0,0 @@
{{/*
Expand the name of the chart.
*/}}
{{- define "k8s-workload-registrar.name" -}}
{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }}
{{- end }}
{{/*
Create a default fully qualified app name.
We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
If release name contains chart name it will be used as a full name.
*/}}
{{- define "k8s-workload-registrar.fullname" -}}
{{- if .Values.fullnameOverride }}
{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }}
{{- else }}
{{- $name := default .Chart.Name .Values.nameOverride }}
{{- if contains $name .Release.Name }}
{{- .Release.Name | trunc 63 | trimSuffix "-" }}
{{- else }}
{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }}
{{- end }}
{{- end }}
{{- end }}
{{/*
Create chart name and version as used by the chart label.
*/}}
{{- define "k8s-workload-registrar.chart" -}}
{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }}
{{- end }}
{{/*
Common labels
*/}}
{{- define "k8s-workload-registrar.labels" -}}
helm.sh/chart: {{ include "k8s-workload-registrar.chart" . }}
{{ include "k8s-workload-registrar.selectorLabels" . }}
{{- if .Chart.AppVersion }}
app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
{{- end }}
app.kubernetes.io/managed-by: {{ .Release.Service }}
{{- end }}
{{/*
Selector labels
*/}}
{{- define "k8s-workload-registrar.selectorLabels" -}}
app.kubernetes.io/name: {{ include "k8s-workload-registrar.name" . }}
app.kubernetes.io/instance: {{ .Release.Name }}
{{- end }}
{{/*
Create the name of the service account to use
*/}}
{{- define "k8s-workload-registrar.serviceAccountName" -}}
{{- if .Values.serviceAccount.create }}
{{- default (include "k8s-workload-registrar.fullname" .) .Values.serviceAccount.name }}
{{- else }}
{{- default "default" .Values.serviceAccount.name }}
{{- end }}
{{- end }}
{{- define "k8s-workload-registrar.image" -}}
{{- if eq (substr 0 7 .image.version) "sha256:" -}}
{{- printf "%s/%s@%s" .image.registry .image.repository .image.version -}}
{{- else if .appVersion -}}
{{- printf "%s/%s:%s" .image.registry .image.repository (default .appVersion .image.version) -}}
{{- else if .image.version -}}
{{- printf "%s/%s:%s" .image.registry .image.repository .image.version -}}
{{- else -}}
{{- printf "%s/%s" .image.registry .image.repository -}}
{{- end -}}
{{- end }}
@@ -1,77 +0,0 @@
apiVersion: apps/v1
kind: Deployment
metadata:
name: {{ include "k8s-workload-registrar.fullname" . }}
labels:
{{- include "k8s-workload-registrar.labels" . | nindent 4 }}
spec:
{{- if not .Values.autoscaling.enabled }}
replicas: {{ .Values.replicaCount }}
{{- end }}
selector:
matchLabels:
{{- include "k8s-workload-registrar.selectorLabels" . | nindent 6 }}
template:
metadata:
{{- with .Values.podAnnotations }}
annotations:
{{- toYaml . | nindent 8 }}
{{- end }}
labels:
{{- include "k8s-workload-registrar.selectorLabels" . | nindent 8 }}
spec:
{{- with .Values.imagePullSecrets }}
imagePullSecrets:
{{- toYaml . | nindent 8 }}
{{- end }}
serviceAccountName: {{ include "k8s-workload-registrar.serviceAccountName" . }}
securityContext:
{{- toYaml .Values.podSecurityContext | nindent 8 }}
initContainers:
- name: init
# This is a small image with wait-for-it, choose whatever image
# you prefer that waits for a service to be up. This image is built
# from https://github.com/vishnubob/wait-for-it
image: {{ template "k8s-workload-registrar.image" .Values.waitForIt }}
imagePullPolicy: {{ .Values.waitForIt.image.pullPolicy }}
args: ["-t", "30", "-h", {{ .Values.server.host | quote }}, "-p", {{ .Values.server.port | quote }}]
resources:
{{- toYaml .Values.waitForIt.resources | nindent 12 }}
containers:
- name: {{ .Chart.Name }}
securityContext:
{{- toYaml .Values.securityContext | nindent 12 }}
image: {{ template "k8s-workload-registrar.image" (dict "appVersion" $.Chart.AppVersion "image" .Values.image) }}
imagePullPolicy: {{ .Values.image.pullPolicy }}
args:
- -config
- /run/spire/k8s-workload-registrar/config/workload-registrar.conf
resources:
{{- toYaml .Values.resources | nindent 12 }}
volumeMounts:
- name: spire-server-socket
mountPath: /run/spire/server-sockets
readOnly: true
- name: k8s-workload-registrar-config
mountPath: /run/spire/k8s-workload-registrar/config
readOnly: true
{{- with .Values.nodeSelector }}
nodeSelector:
{{- toYaml . | nindent 8 }}
{{- end }}
{{- with .Values.affinity }}
affinity:
{{- toYaml . | nindent 8 }}
{{- end }}
{{- with .Values.tolerations }}
tolerations:
{{- toYaml . | nindent 8 }}
{{- end }}
volumes:
- name: k8s-workload-registrar-config
configMap:
name: {{ include "k8s-workload-registrar.fullname" . }}
- name: spire-server-socket
hostPath:
path: /run/spire/server-sockets
type: DirectoryOrCreate
@@ -1,28 +0,0 @@
{{- if .Values.autoscaling.enabled }}
apiVersion: autoscaling/v2beta1
kind: HorizontalPodAutoscaler
metadata:
name: {{ include "k8s-workload-registrar.fullname" . }}
labels:
{{- include "k8s-workload-registrar.labels" . | nindent 4 }}
spec:
scaleTargetRef:
apiVersion: apps/v1
kind: Deployment
name: {{ include "k8s-workload-registrar.fullname" . }}
minReplicas: {{ .Values.autoscaling.minReplicas }}
maxReplicas: {{ .Values.autoscaling.maxReplicas }}
metrics:
{{- if .Values.autoscaling.targetCPUUtilizationPercentage }}
- type: Resource
resource:
name: cpu
targetAverageUtilization: {{ .Values.autoscaling.targetCPUUtilizationPercentage }}
{{- end }}
{{- if .Values.autoscaling.targetMemoryUtilizationPercentage }}
- type: Resource
resource:
name: memory
targetAverageUtilization: {{ .Values.autoscaling.targetMemoryUtilizationPercentage }}
{{- end }}
{{- end }}
@@ -1,73 +0,0 @@
kind: Role
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: {{ include "k8s-workload-registrar.fullname" . }}
namespace: {{ .Release.Namespace }}
rules:
- apiGroups: [coordination.k8s.io]
resources: [leases]
verbs: [create]
- apiGroups: [coordination.k8s.io]
resources: [leases]
resourceNames: [spire-k8s-registrar-leader-election]
verbs:
- get
- update
# - apiGroups: [""]
# resources: [configmaps]
# verbs: [create]
# - apiGroups: [""]
# resources: [configmaps]
# resourceNames: [k8s-workload-registrar-leader-election]
# verbs:
# - get
# - update
- apiGroups: [""]
resources: [events]
verbs: [create]
---
kind: RoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: {{ include "k8s-workload-registrar.fullname" . }}
namespace: {{ .Release.Namespace }}
subjects:
- kind: ServiceAccount
name: {{ include "k8s-workload-registrar.serviceAccountName" . }}
namespace: {{ .Release.Namespace }}
roleRef:
kind: Role
name: {{ include "k8s-workload-registrar.fullname" . }}
apiGroup: rbac.authorization.k8s.io
---
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: {{ include "k8s-workload-registrar.fullname" . }}
rules:
- apiGroups: [""]
resources: [pods, nodes, endpoints]
verbs:
- get
- list
- watch
- apiGroups: [coordination.k8s.io]
resources: [leases]
verbs:
- create
- get
- list
- update
---
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: {{ include "k8s-workload-registrar.fullname" . }}
subjects:
- kind: ServiceAccount
name: {{ include "k8s-workload-registrar.serviceAccountName" . }}
namespace: {{ .Release.Namespace }}
roleRef:
kind: ClusterRole
name: {{ include "k8s-workload-registrar.fullname" . }}
apiGroup: rbac.authorization.k8s.io
@@ -1,12 +0,0 @@
{{- if .Values.serviceAccount.create -}}
apiVersion: v1
kind: ServiceAccount
metadata:
name: {{ include "k8s-workload-registrar.serviceAccountName" . }}
labels:
{{- include "k8s-workload-registrar.labels" . | nindent 4 }}
{{- with .Values.serviceAccount.annotations }}
annotations:
{{- toYaml . | nindent 4 }}
{{- end }}
{{- end }}
@@ -1,80 +0,0 @@
# Default values for k8s-workload-registrar.
# This is a YAML-formatted file.
# Declare variables to be passed into your templates.
replicaCount: 1
image:
registry: gcr.io
repository: spiffe-io/k8s-workload-registrar
pullPolicy: IfNotPresent
# Overrides the image tag whose default is the chart appVersion.
version: ""
imagePullSecrets: []
nameOverride: ""
fullnameOverride: ""
serviceAccount:
# Specifies whether a service account should be created
create: true
# Annotations to add to the service account
annotations: {}
# The name of the service account to use.
# If not set and create is true, a name is generated using the fullname template
name: ""
podAnnotations: {}
podSecurityContext: {}
# fsGroup: 2000
securityContext: {}
# capabilities:
# drop:
# - ALL
# readOnlyRootFilesystem: true
# runAsNonRoot: true
# runAsUser: 1000
resources: {}
# We usually recommend not to specify default resources and to leave this as a conscious
# choice for the user. This also increases chances charts run on environments with little
# resources, such as Minikube. If you do want to specify resources, uncomment the following
# lines, adjust them as necessary, and remove the curly braces after 'resources:'.
# limits:
# cpu: 100m
# memory: 128Mi
# requests:
# cpu: 100m
# memory: 128Mi
autoscaling:
enabled: false
minReplicas: 1
maxReplicas: 5
targetCPUUtilizationPercentage: 80
targetMemoryUtilizationPercentage: 80
nodeSelector:
kubernetes.io/arch: amd64
tolerations: []
affinity: {}
server:
socketPath: /run/spire/server-sockets/spire-server.sock
host: spire-server
port: 8081
clusterName: "example-cluster"
trustDomain: "example.org"
logLevel: info
waitForIt:
image:
registry: cgr.dev
repository: chainguard/wait-for-it
pullPolicy: IfNotPresent
version: latest-20221223
resources: {}
@@ -29,6 +29,14 @@ A Helm chart to install the SPIRE server.
| image.version | string | `""` | |
| imagePullSecrets | list | `[]` | |
| jwtIssuer | string | `"oidc-discovery.example.org"` | |
| k8sWorkloadRegistrar.enabled | bool | `true` | |
| k8sWorkloadRegistrar.image.pullPolicy | string | `"IfNotPresent"` | |
| k8sWorkloadRegistrar.image.registry | string | `"gcr.io"` | |
| k8sWorkloadRegistrar.image.repository | string | `"spiffe-io/k8s-workload-registrar"` | |
| k8sWorkloadRegistrar.image.version | string | `""` | |
| k8sWorkloadRegistrar.logLevel | string | `"info"` | |
| k8sWorkloadRegistrar.resources | object | `{}` | |
| k8sWorkloadRegistrar.securityContext | object | `{}` | |
| logLevel | string | `"info"` | |
| nameOverride | string | `""` | |
| nodeSelector."kubernetes.io/arch" | string | `"amd64"` | |
@@ -1,14 +1,16 @@
{{- if eq (.Values.k8sWorkloadRegistrar.enabled | toString) "true" }}
apiVersion: v1
kind: ConfigMap
metadata:
name: {{ include "k8s-workload-registrar.fullname" . }}
name: {{ include "spire-server.fullname" . }}-k8s-workload-registrar
namespace: {{ .Release.Namespace }}
data:
workload-registrar.conf: |
log_level = "{{ .Values.logLevel }}"
log_level = "{{ .Values.k8sWorkloadRegistrar.logLevel }}"
mode = "reconcile"
trust_domain = {{ .Values.trustDomain | quote }}
cluster = {{ .Values.clusterName | quote }}
server_address = "unix://{{ .Values.server.socketPath }}"
server_address = "unix://{{ .Values.socketPath }}"
leader_election = true
metrics_addr = "0.0.0.0:18080"
{{- end }}
@@ -0,0 +1,56 @@
{{- if eq (.Values.k8sWorkloadRegistrar.enabled | toString) "true" }}
# This is copied directly from the spire/support/k8s/k8s-workload-registrar tree.
# These roles are needed for the k8s registrar to work properly in reconciling mode.
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
name: {{ include "spire-server.fullname" . }}-k8s-workload-registrar
rules:
- apiGroups: [""]
resources: ["pods", "nodes", "endpoints"]
verbs: ["get", "list", "watch"]
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
name: {{ include "spire-server.fullname" . }}-k8s-workload-registrar
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: {{ include "spire-server.fullname" . }}-k8s-workload-registrar
subjects:
- kind: ServiceAccount
name: {{ include "spire-server.serviceAccountName" . }}
namespace: {{ .Release.Namespace }}
---
apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
name: {{ include "spire-server.fullname" . }}-k8s-workload-registrar
namespace: {{ .Release.Namespace }}
rules:
- apiGroups: [""]
resources: ["configmaps"]
verbs: ["create"]
- apiGroups: [""]
resources: ["configmaps"]
resourceNames: ["controller-leader-election-helper"]
verbs: ["update", "get"]
- apiGroups: [""]
resources: ["events"]
verbs: ["create"]
---
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
name: {{ include "spire-server.fullname" . }}-k8s-workload-registrar
namespace: {{ .Release.Namespace }}
roleRef:
kind: Role
name: {{ include "spire-server.fullname" . }}-k8s-workload-registrar
apiGroup: rbac.authorization.k8s.io
subjects:
- kind: ServiceAccount
name: {{ include "spire-server.serviceAccountName" . }}
namespace: {{ .Release.Namespace }}
{{- end }}
@@ -46,11 +46,14 @@ rules:
resources: [tokenreviews]
verbs:
- get
- watch
- list
- create
- apiGroups: [""]
resources: [nodes]
resources: [nodes, pods]
verbs:
- get
- list
---
# Binds above cluster role to spire-server service account
kind: ClusterRoleBinding
@@ -76,6 +76,25 @@ spec:
mountPath: /run/spire/upstream_ca
readOnly: false
{{ end }}
{{- if eq (.Values.k8sWorkloadRegistrar.enabled | toString) "true" }}
- name: spire-k8s-workload-registrar
securityContext:
{{- toYaml .Values.k8sWorkloadRegistrar.securityContext | nindent 12 }}
image: {{ template "spire-server.image" (dict "appVersion" $.Chart.AppVersion "image" .Values.k8sWorkloadRegistrar.image) }}
imagePullPolicy: {{ .Values.image.pullPolicy }}
args:
- -config
- /run/spire/k8s-workload-registrar/config/workload-registrar.conf
resources:
{{- toYaml .Values.k8sWorkloadRegistrar.resources | nindent 12 }}
volumeMounts:
- name: spire-server-socket
mountPath: /run/spire/server-sockets
readOnly: true
- name: k8s-workload-registrar-config
mountPath: /run/spire/k8s-workload-registrar/config
readOnly: true
{{- end }}
{{- with .Values.nodeSelector }}
nodeSelector:
{{- toYaml . | nindent 8 }}
@@ -105,6 +124,11 @@ spec:
secret:
secretName: {{ include "spire-server.upstream-ca-secret" . }}
{{- end }}
{{- if eq (.Values.k8sWorkloadRegistrar.enabled | toString) "true" }}
- name: k8s-workload-registrar-config
configMap:
name: {{ include "spire-server.fullname" . }}-k8s-workload-registrar
{{- end }}
volumeClaimTemplates:
{{- if eq (.Values.dataStorage.enabled | toString) "true" }}
- metadata:
@@ -105,3 +105,35 @@ upstreamAuthority:
certificate: ""
key: ""
bundle: ""
k8sWorkloadRegistrar:
enabled: true
image:
registry: gcr.io
repository: spiffe-io/k8s-workload-registrar
pullPolicy: IfNotPresent
# Overrides the image tag whose default is the chart appVersion.
version: ""
resources: {}
# We usually recommend not to specify default resources and to leave this as a conscious
# choice for the user. This also increases chances charts run on environments with little
# resources, such as Minikube. If you do want to specify resources, uncomment the following
# lines, adjust them as necessary, and remove the curly braces after 'resources:'.
# limits:
# cpu: 100m
# memory: 128Mi
# requests:
# cpu: 100m
# memory: 128Mi
securityContext: {}
# capabilities:
# drop:
# - ALL
# readOnlyRootFilesystem: true
# runAsNonRoot: true
# runAsUser: 1000
logLevel: info
-9
View File
@@ -26,12 +26,3 @@ spiffe-csi-driver:
spiffe-oidc-discovery-provider:
enabled: false
trustDomain: *trustDomain
k8s-workload-registrar:
enabled: true
clusterName: *clusterName
trustDomain: *trustDomain
server:
socketPath: *serverSocketPath