Basic Prometheus support (#28)

Co-authored-by: Marco Franssen <[email protected]>

.github/tests/prometheus/values.yaml

@@ -0,0 +1,4 @@
+global:
+  telemetry:
+    prometheus:
+      enabled: true

charts/spire/charts/spiffe-oidc-discovery-provider/README.md

@@ -48,6 +48,13 @@ A Helm chart to install the SPIFFE OIDC discovery provider.
 | serviceAccount.annotations | object | `{}` |  |
 | serviceAccount.create | bool | `true` |  |
 | serviceAccount.name | string | `""` |  |
+| telemetry.prometheus.enabled | bool | `false` |  |
+| telemetry.prometheus.nginxExporter.image.pullPolicy | string | `"IfNotPresent"` |  |
+| telemetry.prometheus.nginxExporter.image.registry | string | `"docker.io"` |  |
+| telemetry.prometheus.nginxExporter.image.repository | string | `"nginx/nginx-prometheus-exporter"` |  |
+| telemetry.prometheus.nginxExporter.image.version | string | `"0.11.0"` |  |
+| telemetry.prometheus.nginxExporter.resources | object | `{}` |  |
+| telemetry.prometheus.port | int | `9988` |  |
 | tolerations | list | `[]` |  |
 | trustDomain | string | `"example.org"` |  |
 

charts/spire/charts/spiffe-oidc-discovery-provider/templates/configmap.yaml

@@ -53,5 +53,11 @@ data:
         proxy_pass http://oidc;
         proxy_set_header Host $host;
       }
+
+      location /stub_status {
+        allow 127.0.0.1/32;
+        deny  all;
+        stub_status on;
+      }
     }
   {{- end }}

charts/spire/charts/spiffe-oidc-discovery-provider/templates/deployment.yaml

@@ -87,6 +87,20 @@ spec:
               readOnly: true
           resources:
             {{- toYaml .Values.insecureScheme.nginx.resources | nindent 12 }}
+        {{- if (dig "telemetry" "prometheus" "enabled" .Values.telemetry.prometheus.enabled .Values.global) }}
+        - name: nginx-exporter
+          securityContext:
+            {{- toYaml .Values.securityContext | nindent 12 }}
+          image: {{ template "spiffe-oidc-discovery-provider.image" .Values.telemetry.prometheus.nginxExporter }}
+          imagePullPolicy: {{ .Values.telemetry.prometheus.nginxExporter.image.pullPolicy }}
+          args:
+          - -nginx.scrape-uri=http://127.0.0.1/stub_status
+          resources:
+            {{- toYaml .Values.telemetry.prometheus.nginxExporter.resources | nindent 12 }}
+          ports:
+            - containerPort: 9113
+              name: prom
+        {{- end }}
         {{- end }}
       volumes:
         - name: spiffe-workload-api

charts/spire/charts/spiffe-oidc-discovery-provider/values.yaml

@@ -110,3 +110,27 @@ tolerations: []
 affinity: {}
 
 trustDomain: "example.org"
+
+telemetry:
+  prometheus:
+    enabled: false
+    port: 9988
+
+    nginxExporter:
+      image:
+        registry: docker.io
+        repository: nginx/nginx-prometheus-exporter
+        pullPolicy: IfNotPresent
+        version: "0.11.0"
+
+      resources: {}
+        # We usually recommend not to specify default resources and to leave this as a conscious
+        # choice for the user. This also increases chances charts run on environments with little
+        # resources, such as Minikube. If you do want to specify resources, uncomment the following
+        # lines, adjust them as necessary, and remove the curly braces after 'resources:'.
+        # requests:
+        #   cpu: 50m
+        #   memory: 32Mi
+        # limits:
+        #   cpu: 100m
+        #   memory: 64Mi

charts/spire/charts/spire-agent/README.md

@@ -30,6 +30,8 @@ A Helm chart to install the SPIRE agent.
 | serviceAccount.annotations | object | `{}` |  |
 | serviceAccount.create | bool | `true` |  |
 | serviceAccount.name | string | `""` |  |
+| telemetry.prometheus.enabled | bool | `false` |  |
+| telemetry.prometheus.port | int | `9988` |  |
 | trustDomain | string | `"example.org"` |  |
 | waitForIt.image.pullPolicy | string | `"IfNotPresent"` |  |
 | waitForIt.image.registry | string | `"cgr.dev"` |  |

charts/spire/charts/spire-agent/templates/configmap.yaml

@@ -51,3 +51,12 @@ data:
       live_path = "/live"
       ready_path = "/ready"
     }
+
+    {{- if (dig "telemetry" "prometheus" "enabled" .Values.telemetry.prometheus.enabled .Values.global) }}
+    telemetry {
+      Prometheus {
+        host = "0.0.0.0"
+        port = {{ .Values.telemetry.prometheus.port }}
+      }
+    }
+    {{- end }}

charts/spire/charts/spire-agent/templates/daemonset.yaml

@@ -1,3 +1,4 @@
+{{- $configSum := (include (print $.Template.BasePath "/configmap.yaml") . | sha256sum) }}
 apiVersion: apps/v1
 kind: DaemonSet
 metadata:
@@ -11,10 +12,11 @@ spec:
       {{- include "spire-agent.selectorLabels" . | nindent 6 }}
   template:
     metadata:
-      {{- with .Values.podAnnotations }}
       annotations:
-        {{- toYaml . | nindent 8 }}
+        checksum/config: {{ $configSum }}
-      {{- end }}
+        {{- with .Values.podAnnotations }}
+          {{- toYaml . | nindent 8 }}
+        {{- end }}
       labels:
         {{- include "spire-agent.selectorLabels" . | nindent 8 }}
     spec:
@@ -46,6 +48,10 @@ spec:
           ports:
             - containerPort: {{ .Values.healthChecks.port }}
               name: healthz
+            {{- if (dig "telemetry" "prometheus" "enabled" .Values.telemetry.prometheus.enabled .Values.global) }}
+            - containerPort: {{ .Values.telemetry.prometheus.port }}
+              name: prom
+            {{- end }}
           volumeMounts:
             - name: spire-config
               mountPath: /run/spire/config

charts/spire/charts/spire-agent/values.yaml

@@ -79,3 +79,8 @@ workloadAttestors:
   unix:
     # -- enables the Unix workload attestor
     enabled: false
+
+telemetry:
+  prometheus:
+    enabled: false
+    port: 9988

charts/spire/charts/spire-server/README.md

@@ -64,6 +64,7 @@ A Helm chart to install the SPIRE server.
 | serviceAccount.create | bool | `true` |  |
 | serviceAccount.name | string | `""` |  |
 | socketPath | string | `"/run/spire/server-sockets/spire-server.sock"` |  |
+| telemetry.prometheus.enabled | bool | `false` |  |
 | tolerations | list | `[]` |  |
 | topologySpreadConstraints | list | `[]` |  |
 | trustDomain | string | `"example.org"` |  |

charts/spire/charts/spire-server/templates/configmap.yaml

@@ -81,3 +81,12 @@ data:
       live_path = "/live"
       ready_path = "/ready"
     }
+
+    {{- if (dig "telemetry" "prometheus" "enabled" .Values.telemetry.prometheus.enabled .Values.global) }}
+    telemetry {
+      Prometheus {
+        host = "0.0.0.0"
+        port = 9988
+      }
+    }
+    {{- end }}

charts/spire/charts/spire-server/templates/controller-manager-configmap.yaml

@@ -14,7 +14,7 @@ data:
       labels:
         {{- include "spire-server.labels" . | nindent 8 }}
     metrics:
-      bindAddress: 127.0.0.1:8082
+      bindAddress: 0.0.0.0:8082
     healthProbe:
       bindAddress: 127.0.0.1:8083
     leaderElection:

charts/spire/charts/spire-server/templates/statefulset.yaml

@@ -1,3 +1,4 @@
+{{- $configSum := (include (print $.Template.BasePath "/configmap.yaml") . | sha256sum) }}
 {{- $fullname := include "spire-server.fullname" . }}
 apiVersion: apps/v1
 kind: StatefulSet
@@ -15,10 +16,11 @@ spec:
       {{- include "spire-server.selectorLabels" . | nindent 6 }}
   template:
     metadata:
-      {{- with .Values.podAnnotations }}
       annotations:
-        {{- toYaml . | nindent 8 }}
+        checksum/config: {{ $configSum }}
-      {{- end }}
+        {{- with .Values.podAnnotations }}
+          {{- toYaml . | nindent 8 }}
+        {{- end }}
       labels:
         {{- include "spire-server.selectorLabels" . | nindent 8 }}
     spec:
@@ -45,6 +47,10 @@ spec:
               protocol: TCP
             - containerPort: 8080
               name: healthz
+            {{- if (dig "telemetry" "prometheus" "enabled" .Values.telemetry.prometheus.enabled .Values.global) }}
+            - containerPort: 9988
+              name: prom
+            {{- end }}
           livenessProbe:
             httpGet:
               path: /live
@@ -92,6 +98,10 @@ spec:
               protocol: TCP
             - containerPort: 8008
               name: healthz
+            {{- if (dig "telemetry" "prometheus" "enabled" .Values.telemetry.prometheus.enabled .Values.global) }}
+            - containerPort: 8082
+              name: prom2
+            {{- end }}
           # TODO: implement probes
           # livenessProbe:
           #   httpGet:

charts/spire/charts/spire-server/values.yaml

@@ -159,3 +159,7 @@ controllerManager:
       #   spiffe.io/spiffe-id: "true"
     dnsNameTemplates: []
       # - '{{ index .PodMeta.Labels "app.kubernetes.io/name" }}.{{ .PodMeta.Namespace }}.svc.cluster.local'
+
+telemetry:
+  prometheus:
+    enabled: false

charts/spire/values.yaml

@@ -1,3 +1,9 @@
+# You can enable features that affect all services here.
+# global:
+#   telemetry:
+#     prometheus:
+#       enabled: true
+
 nameOverride: ""
 fullnameOverride: ""