ingress-nginx production tests and spiffe-oidc-discovery-provider example (#136)

Co-authored-by: Marco Franssen <[email protected]>

.github/tests/dependencies/testcert.yaml

@@ -0,0 +1,52 @@
+---
+apiVersion: cert-manager.io/v1
+kind: Issuer
+metadata:
+  name: selfsigned-issuer
+spec:
+  selfSigned: {}
+---
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+  name: demo-selfsigned-ca
+spec:
+  isCA: true
+  commonName: demo-selfsigned-ca
+  secretName: root-secret
+  privateKey:
+    algorithm: ECDSA
+    size: 256
+  issuerRef:
+    name: selfsigned-issuer
+    kind: Issuer
+    group: cert-manager.io
+  subject:
+    countries:
+    - US
+    organizations:
+    - test
+    organizationalUnits:
+    - test
+---
+apiVersion: cert-manager.io/v1
+kind: Issuer
+metadata:
+  name: demo-ca
+spec:
+  ca:
+    secretName: root-secret
+---
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+  name: oidc
+spec:
+  dnsNames:
+    - oidc-discovery.example.org
+    - spire-server-federation.example.org
+  secretName: tls-cert
+  issuerRef:
+    name: demo-ca
+    kind: Issuer
+    group: cert-manager.io

charts/spire/README.md

@@ -237,6 +237,7 @@ Now you can interact with the Spire agent socket from your own application. The
 | spire-agent.fsGroupFix.resources | object | `{}` | Specify resource needs as per https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ |
 | spire-agent.fullnameOverride | string | `""` |  |
 | spire-agent.healthChecks.port | int | `9980` | override the host port used for health checking |
+| spire-agent.hostAliases | list | `[]` | Customize /etc/hosts file as described here https://kubernetes.io/docs/tasks/network/customize-hosts-file-for-pods/ |
 | spire-agent.image.pullPolicy | string | `"IfNotPresent"` | The image pull policy |
 | spire-agent.image.registry | string | `"ghcr.io"` | The OCI registry to pull the image from |
 | spire-agent.image.repository | string | `"spiffe/spire-agent"` | The repository within the registry |
@@ -540,6 +541,7 @@ Now you can interact with the Spire agent socket from your own application. The
 | upstream-spire-agent.fsGroupFix.resources | object | `{}` | Specify resource needs as per https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ |
 | upstream-spire-agent.fullnameOverride | string | `""` |  |
 | upstream-spire-agent.healthChecks.port | int | `9980` | override the host port used for health checking |
+| upstream-spire-agent.hostAliases | list | `[]` | Customize /etc/hosts file as described here https://kubernetes.io/docs/tasks/network/customize-hosts-file-for-pods/ |
 | upstream-spire-agent.image.pullPolicy | string | `"IfNotPresent"` | The image pull policy |
 | upstream-spire-agent.image.registry | string | `"ghcr.io"` | The OCI registry to pull the image from |
 | upstream-spire-agent.image.repository | string | `"spiffe/spire-agent"` | The repository within the registry |

charts/spire/charts/spiffe-oidc-discovery-provider/templates/tests/test-connection.yaml

@@ -1,3 +1,4 @@
+{{ $values := merge .Values }}
 apiVersion: v1
 kind: Pod
 metadata:
@@ -33,8 +34,31 @@ spec:
     - name: curl-ingress
       image: {{ template "spire-lib.image" (dict "image" .Values.tests.bash.image "global" .Values.global) }}
       command: ['curl']
+      {{- if dig "tests" "tls" "enabled" false $values }}
+      {{- if ne (len (dig "tests" "tls" "customCA" "" $values)) 0 }}
+      args: ['-s', '-f', '--cacert', '/ca/ca.crt', 'https://{{ (index .Values.ingress.hosts 0).host }}/.well-known/openid-configuration']
+      {{- else }}
+      args: ['-s', '-f', 'https://{{ (index .Values.ingress.hosts 0).host }}/.well-known/openid-configuration']
+      {{- end }}
+      {{- if ne (len (dig "tests" "tls" "customCA" "" $values)) 0 }}
+      volumeMounts:
+        - name: custom-ca
+          mountPath: /ca
+      {{- end }}
+      {{- else }}
       args: ['-s', '-f', 'http://{{ (index .Values.ingress.hosts 0).host }}/.well-known/openid-configuration']
+      {{- end }}
       securityContext:
         {{- toYaml .Values.securityContext | nindent 8 }}
     {{- end }}
+  {{- if ne (len (dig "tests" "hostAliases" "" $values)) 0 }}
+  hostAliases:
+    {{- toYaml .Values.tests.hostAliases | nindent 4 }}
+  {{- end }}
+  {{- if ne (len (dig "tests" "tls" "customCA" "" $values)) 0 }}
+  volumes:
+    - name: custom-ca
+      secret:
+        secretName: {{ .Values.tests.tls.customCA }}
+  {{- end }}
   restartPolicy: Never

charts/spire/charts/spiffe-oidc-discovery-provider/values.yaml

@@ -212,6 +212,10 @@ ingress:
 
 # @ignored
 tests:
+  hostAliases: []
+  tls:
+    enabled: false
+    customCA: ""
   bash:
     image:
       # -- The OCI registry to pull the tests image from

charts/spire/charts/spire-agent/README.md

@@ -43,6 +43,7 @@ A Helm chart to install the SPIRE agent.
 | fsGroupFix.resources | object | `{}` | Specify resource needs as per https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ |
 | fullnameOverride | string | `""` |  |
 | healthChecks.port | int | `9980` | override the host port used for health checking |
+| hostAliases | list | `[]` | Customize /etc/hosts file as described here https://kubernetes.io/docs/tasks/network/customize-hosts-file-for-pods/ |
 | image.pullPolicy | string | `"IfNotPresent"` | The image pull policy |
 | image.registry | string | `"ghcr.io"` | The OCI registry to pull the image from |
 | image.repository | string | `"spiffe/spire-agent"` | The repository within the registry |

charts/spire/charts/spire-agent/templates/daemonset.yaml

@@ -33,6 +33,10 @@ spec:
       {{- if .Values.priorityClassName }}
       priorityClassName: {{ .Values.priorityClassName }}
       {{- end }}
+      {{- if ne (len .Values.hostAliases) 0 }}
+      hostAliases:
+        {{- toYaml .Values.hostAliases | nindent 8 }}
+      {{- end }}
       initContainers:
         - name: init
           # This is a small image with wait-for-it, choose whatever image

charts/spire/charts/spire-agent/values.yaml

@@ -164,3 +164,5 @@ extraVolumeMounts: []
 extraContainers: []
 
 initContainers: []
+# -- Customize /etc/hosts file as described here https://kubernetes.io/docs/tasks/network/customize-hosts-file-for-pods/
+hostAliases: []

charts/spire/charts/spire-server/templates/_helpers.tpl

@@ -203,6 +203,21 @@ The code below determines what connection type should be used.
 {{- include "spire-tornjak.backend" . -}}
 {{- end -}}
 
+{{- define "spire-server.test.federation-ingress-args" }}
+{{-   $args := list }}
+{{-   $host := index (index (index .Values.federation.ingress.tls 0) "hosts") 0 }}
+{{-   if dig "tests" "tls" "enabled" false .Values }}
+{{-     if ne (len (dig "tests" "tls" "customCA" "" .Values)) 0 }}
+{{-       $args = append $args "--cacert" }}
+{{-       $args = append $args "/ca/ca.crt" }}
+{{-     end }}
+{{-     $args = append $args (printf "https://%s/" $host) }}
+{{-   else }}
+{{-     $args = append $args (printf "http://%s/" $host) }}
+{{-   end }}
+{{ $args | toYaml }}
+{{- end -}}
+
 {{/*
 Take a copy of the config and merge in .Values.customPlugins and .Values.unsupportedBuiltInPlugins passed through as root.
 */}}

charts/spire/charts/spire-server/templates/tests/test-connection.yaml

@@ -1,3 +1,4 @@
+{{ $values := merge .Values }}
 apiVersion: v1
 kind: Pod
 metadata:
@@ -38,4 +39,28 @@ spec:
       securityContext:
         {{- toYaml .Values.securityContext | nindent 8 }}
     {{- end }}
+    {{- if .Values.federation.ingress.enabled }}
+    - name: federation-ingress
+      image: {{ template "spire-lib.image" (dict "image" .Values.tests.bash.image "global" .Values.global) }}
+      command: ['curl']
+      args:
+        {{ include "spire-server.test.federation-ingress-args" (dict "Values" $values) | nindent 8 }}
+      {{- if ne (len (dig "tests" "tls" "customCA" "" $values)) 0 }}
+      volumeMounts:
+        - name: custom-ca
+          mountPath: /ca
+      {{- end }}
+      securityContext:
+        {{- toYaml .Values.securityContext | nindent 8 }}
+    {{- end }}
+  {{- if ne (len (dig "tests" "hostAliases" "" $values)) 0 }}
+  hostAliases:
+    {{- toYaml .Values.tests.hostAliases | nindent 4 }}
+  {{- end }}
+  {{- if ne (len (dig "tests" "tls" "customCA" "" $values)) 0 }}
+  volumes:
+    - name: custom-ca
+      secret:
+        secretName: {{ .Values.tests.tls.customCA }}
+  {{- end }}
   restartPolicy: Never

charts/spire/charts/spire-server/values.yaml

@@ -486,6 +486,10 @@ customPlugins:
 
 # @ignored
 tests:
+  hostAliases: []
+  tls:
+    enabled: false
+    customCA: ""
   bash:
     image:
       # -- The OCI registry to pull the image from

examples/production/README.md

@@ -8,16 +8,26 @@ kubectl label namespace "spire-system" pod-security.kubernetes.io/enforce=privil
 kubectl create namespace "spire-server"
 kubectl label namespace "spire-server" pod-security.kubernetes.io/enforce=restricted
 
+```shell
 helm upgrade --install --namespace spire-server spire charts/spire -f values.yaml
 ```
 
-See [values.yaml](./values.yaml) for more details on the chart configurations to achieve this setup.
+If your using ingress-nginx and want to expose the spiffe oidc discovery provider outside the
+cluster, add the following to the end of the helm upgrade example:
+
+```shell
+-f values-export-spiffe-oidc-discovery-provider-ingress-nginx.yaml
+```
 
 If you want to expose your spire-server outside of Kubernetes and are using ingress-nginx, add following values file when running `helm template/install/upgrade`.
 
 ```shell
 -f values-expose-spire-server-ingress-nginx.yaml
 ```
+For example:
+```shell
+helm upgrade --install --namespace spire-server spire charts/spire -f values.yaml -f values-expose-spire-server-ingress-nginx.yaml
+```
 
 If you want to expose your federation endpoint outside of Kubernetes and are using ingress-nginx
 you have two options as described here:
@@ -28,9 +38,20 @@ If you chose profile https_web, use:
 ```shell
 -f values-expose-federation-https-web-ingress-nginx.yaml
 ```
+For example:
+```shell
+helm upgrade --install --namespace spire-server spire charts/spire -f values.yaml -f values-expose-federation-https-web-ingress-nginx.yaml
+```
 
 If you chose profile https_spiffe, use:
 
 ```shell
 -f values-expose-federation-https-spiffe-ingress-nginx.yaml
 ```
+For example:
+```shell
+helm upgrade --install --namespace spire-server spire charts/spire -f values.yaml -f values-expose-federation-https-spiffe-ingress-nginx.yaml
+```
+
+See [values.yaml](./values.yaml) for more details on the chart configurations to achieve this setup.
+

examples/production/run-tests.sh

@@ -5,7 +5,10 @@ set -xe
 SCRIPT="$(readlink -f "$0")"
 SCRIPTPATH="$(dirname "${SCRIPT}")"
 TESTDIR="${SCRIPTPATH}/../../.github/tests"
+DEPS="${TESTDIR}/dependencies"
 
+# shellcheck source=/dev/null
+source "${SCRIPTPATH}/../../.github/scripts/parse-versions.sh"
 # shellcheck source=/dev/null
 source "${TESTDIR}/common.sh"
 
@@ -16,6 +19,10 @@ teardown() {
   helm uninstall --namespace "${ns}" spire 2>/dev/null || true
   kubectl delete ns "${ns}" 2>/dev/null || true
   kubectl delete ns spire-system 2>/dev/null || true
+  helm uninstall --namespace cert-manager cert-manager 2>/dev/null || true
+  kubectl delete ns cert-manager 2>/dev/null || true
+  helm uninstall --namespace ingress-nginx 2>/dev/null || true
+  kubectl delete ns ingress-nginx 2>/dev/null || true
 }
 
 trap 'trap - SIGTERM && teardown' SIGINT SIGTERM EXIT
@@ -25,7 +32,61 @@ kubectl label namespace spire-system pod-security.kubernetes.io/enforce=privileg
 kubectl create namespace "${ns}" 2>/dev/null || true
 kubectl label namespace "${ns}" pod-security.kubernetes.io/enforce=restricted || true
 
-"${helm_install[@]}" --namespace "${ns}" --values "${SCRIPTPATH}/values.yaml" --wait spire charts/spire
+"${helm_install[@]}" cert-manager cert-manager --version "$VERSION_CERT_MANAGER" --repo "$HELM_REPO_CERT_MANAGER" \
+  --namespace cert-manager \
+  --create-namespace \
+  --set installCRDs=true \
+  --wait
+
+kubectl apply -f "${DEPS}/testcert.yaml" -n spire-server
+
+"${helm_install[@]}" ingress-nginx ingress-nginx --version "$VERSION_INGRESS_NGINX" --repo "$HELM_REPO_INGRESS_NGINX" \
+  --namespace ingress-nginx \
+  --create-namespace \
+  --set controller.extraArgs.enable-ssl-passthrough=,controller.admissionWebhooks.enabled=false,controller.service.type=ClusterIP \
+  --set controller.ingressClassResource.default=true \
+ --wait
+
+ip=$(kubectl get svc -n ingress-nginx ingress-nginx-controller -o go-template='{{ .spec.clusterIP }}')
+echo "$ip" oidc-discovery.example.org
+
+cat > /tmp/dummydns <<EOF
+spiffe-oidc-discovery-provider:
+  tests:
+    hostAliases:
+      - ip: "$ip"
+        hostnames:
+          - "oidc-discovery.example.org"
+spire-agent:
+  hostAliases:
+    - ip: "$ip"
+      hostnames:
+        - "spire-server.example.org"
+spire-server:
+  tests:
+    hostAliases:
+      - ip: "$ip"
+        hostnames:
+          - "spire-server-federation.example.org"
+  federation:
+    ingress:
+      tls:
+        - hosts:
+            - spire-server-federation.example.org
+          secretName: tls-cert
+EOF
+
+"${helm_install[@]}" spire charts/spire \
+  --namespace "${ns}" \
+  --values "${SCRIPTPATH}/values.yaml" \
+  --values "${SCRIPTPATH}/values-export-spiffe-oidc-discovery-provider-ingress-nginx.yaml" \
+  --values "${SCRIPTPATH}/values-export-spire-server-ingress-nginx.yaml" \
+  --values "${SCRIPTPATH}/values-export-federation-https-web-ingress-nginx.yaml" \
+  --values /tmp/dummydns \
+  --set spiffe-oidc-discovery-provider.tests.tls.customCA=tls-cert,spire-server.tests.tls.customCA=tls-cert \
+  --set spire-agent.server.address=spire-server.example.org,spire-agent.server.port=443 \
+  --wait
+
 helm test --namespace "${ns}" spire
 
 print_helm_releases

examples/production/values-export-federation-https-web-ingress-nginx.yaml

@@ -1,4 +1,7 @@
 spire-server:
+  tests:
+    tls:
+      enabled: true
   federation:
     enabled: true
     ingress:

examples/production/values-export-spiffe-oidc-discovery-provider-ingress-nginx.yaml

@@ -0,0 +1,27 @@
+spiffe-oidc-discovery-provider:
+  tests:
+    tls:
+      enabled: true
+  ingress:
+    enabled: true
+    annotations:
+      nginx.ingress.kubernetes.io/ssl-redirect: "true"
+      nginx.ingress.kubernetes.io/force-ssl-redirect: "true"
+
+      # Optional settings you may put in your own values.
+      # kubernetes.io/ingress.class: nginx
+      # kubernetes.io/tls-acme: "true"
+      
+    # className: nginx
+
+    # You must override these in your own values file with the appropriate hostname
+    # and secret or it wont start.
+    hosts:
+      - host: oidc-discovery.example.org
+        paths:
+          - path: /
+            pathType: Prefix
+    tls:
+      - secretName: tls-cert
+        hosts:
+          - oidc-discovery.example.org