helm-charts-hardened/examples/bottom-turtle-ha/run-tests.sh

#!/usr/bin/env bash
# shellcheck disable=SC2317

set -xe

SCRIPT="$(readlink -f "$0")"
SCRIPTPATH="$(dirname "${SCRIPT}")"
TESTDIR="${SCRIPTPATH}/../../.github/tests"
#DEPS="${TESTDIR}/dependencies"

# shellcheck source=/dev/null
source "${SCRIPTPATH}/../../.github/scripts/parse-versions.sh"
# shellcheck source=/dev/null
source "${TESTDIR}/common.sh"

CLEANUP=1
BROKER=0

for i in "$@"; do
  case $i in
    -c)
      CLEANUP=0
      shift # past argument=value
      ;;
    -b)
      BROKER=1
      shift # past argument=value
      ;;
  esac
done

# With -b, test the spire-ha-agent broker api instead of the delegated api.
# Broker mode also supports federated trust bundles, so federate the ha-agent's own entry and a
# dedicated federation-test workload entry with the other.org trust domain on both sides. Delegated
# mode only tolerates the local and spire-ha bundles, so none of this may apply without -b.
BROKER_MODE_ARGS=()
BROKER_SOCKET_ARGS_A=()
BROKER_SOCKET_ARGS_B=()
if [ "${BROKER}" -eq 1 ]; then
  BROKER_MODE_ARGS=(--set "spire-ha-agent.mode=broker")
  BROKER_SOCKET_ARGS_A=(
    --set downstream-spire-agent-bottom-turtle-ha-a.sockets.broker.enabled=true
    --set downstream-spire-agent-bottom-turtle-ha-a.sockets.broker.mountOnHost=true
    --set 'internal-spire-server-bottom-turtle-ha-a.controllerManager.identities.clusterSPIFFEIDs.spire-ha-agent.federatesWith={spire-ha,other.org}'
    --set 'internal-spire-server-bottom-turtle-ha-a.controllerManager.identities.clusterSPIFFEIDs.federation-test.federatesWith={other.org}'
    --set 'internal-spire-server-bottom-turtle-ha-a.controllerManager.identities.clusterSPIFFEIDs.federation-test.podSelector.matchLabels.app=federation-test'
  )
  BROKER_SOCKET_ARGS_B=(
    --set downstream-spire-agent-bottom-turtle-ha-b.sockets.broker.enabled=true
    --set downstream-spire-agent-bottom-turtle-ha-b.sockets.broker.mountOnHost=true
    --set 'internal-spire-server-bottom-turtle-ha-b.controllerManager.identities.clusterSPIFFEIDs.spire-ha-agent.federatesWith={spire-ha,other.org}'
    --set 'internal-spire-server-bottom-turtle-ha-b.controllerManager.identities.clusterSPIFFEIDs.federation-test.federatesWith={other.org}'
    --set 'internal-spire-server-bottom-turtle-ha-b.controllerManager.identities.clusterSPIFFEIDs.federation-test.podSelector.matchLabels.app=federation-test'
  )
fi

if [ "x${GITHUB_JOB}" != "x" ]; then
  echo "Running in GitHub"
else
  echo "Do not run this script on your own box. For testing, it deploys a testing local spire ha setup using sudo. This is likely not what you want. Only use this script as a reference."
  exit 1
fi

teardown() {
  echo ---------------------------
  docker exec -i chart-testing-worker /bin/bash -c "more /var/lib/kubelet/pods/*/volumes/kubernetes.io~empty-dir/disk-keymanager/keys.json /var/lib/kubelet/pods/*/volumes/kubernetes.io~empty-dir/spire-agent-persistence/agent-data.json | cat"
  sudo systemctl status spire-server@a || true
  sudo systemctl status spire-server@b || true
  sudo systemctl status spire-server@other || true
  kubectl describe job federation-test || true
  kubectl logs job/federation-test || true
  sudo spire-server entry show -instance a || true
  sudo spire-server entry show -instance b || true
  sudo systemctl status spire-controller-manager@a || true
  sudo systemctl status spire-controller-manager@b || true
  sudo systemctl status spire-agent@a || true
  sudo systemctl status spire-agent@b || true
  sudo systemctl status spire-trust-sync@a || true
  sudo systemctl status spire-trust-sync@b || true
  sudo systemctl status spiffe-socat-unix@k8s-spire-server-a || true
  sudo systemctl status spiffe-socat-unix@k8s-spire-server-b || true
  sudo systemctl status spiffe-socat-unix@k8s-spire-agent-2-a || true
  sudo systemctl status spiffe-socat-unix@k8s-spire-agent-2-b || true
  sudo systemctl status spiffe-socat-unix@k8s-spire-agent-3-a || true
  sudo systemctl status spiffe-socat-unix@k8s-spire-agent-3-b || true
  sudo systemctl status spiffe-socat-unix@k8s-spire-agent-4-a || true
  sudo systemctl status spiffe-socat-unix@k8s-spire-agent-4-b || true
  sudo spire-server bundle list -socketPath /var/run/spire/server/sockets/a/private/api.sock || true
  sudo spire-server bundle list -socketPath /var/run/spire/server/sockets/b/private/api.sock || true
  kubectl exec -i -n spire-server spire-a-internal-server-0 -- spire-server entry show || true
  kubectl exec -i -n spire-server spire-b-internal-server-0 -- spire-server entry show || true
  kubectl exec -i -n spire-server spire-a-internal-server-0 -- spire-server agent list -output json | yq e . - -P || true
  kubectl exec -i -n spire-server spire-b-internal-server-0 -- spire-server agent list -output json | yq e . - -P || true
  kubectl get pods -A -o wide || true
  kubectl describe daemonset pods -n spire-system || true
  kubectl get configmap -n spire-system || true
  kubectl get configmap -n spire-system spire-a-agent-downstream -o yaml || true

  print_helm_releases

  if [[ "$1" -ne 0 ]]; then
    get_namespace_details spire-server spire-system
    kubectl describe pod -n spire-system
  fi

  if [ "${CLEANUP}" -eq 1 ]; then
    kubectl delete job federation-test 2>/dev/null || true
    helm uninstall --namespace spire-mgmt spire-b 2>/dev/null || true
    helm uninstall --namespace spire-mgmt spire-a 2>/dev/null || true
    helm uninstall --namespace spire-mgmt spire 2>/dev/null || true
    kubectl delete ns spire-server 2>/dev/null || true
    kubectl delete ns spire-system 2>/dev/null || true
    kubectl delete ns spire-mgmt 2>/dev/null || true
  fi
}

trap 'EC=$? && trap - SIGTERM && teardown $EC' SIGINT SIGTERM EXIT

wait_for_healthcheck() {
  local app="$1"
  local socket="$2"
  local timeout=30
  local count=0
  while [ "$count" -lt "$timeout" ]; do
    rc=0
    sudo "$app" healthcheck -socketPath "$socket" || rc=$?
    if [ "$rc" -eq 0 ]; then
      return 0
    fi
    sleep 1
    ((count++)) || true
  done
  return 1
}

wait_for_trust_sync() {
  local socket="$1"
  local timeout=30
  local count=0
  while [ "$count" -lt "$timeout" ]; do
    entries=$(sudo spire-server bundle list -socketPath "$socket" | wc -l)
    if [ "$entries" -ne 0 ]; then
      return 0
    fi
    sleep 1
    ((count++)) || true
  done
  return 1
}

wait_for_jwt() {
  local socket="$1"
  local timeout=30
  local count=0
  while [ "$count" -lt "$timeout" ]; do
      rc=0
      sudo spire-agent api fetch jwt -audience test -socketPath "$socket" || rc=$?
      if [ "$rc" -eq 0 ]; then
        return 0
      fi
      sleep 1
      ((count++)) || true
  done
  return 1
}

wait_for_entry_federation() {
  local pod="$1"
  local trustdomain="$2"
  local timeout=60
  local count=0
  while [ "$count" -lt "$timeout" ]; do
    if kubectl exec -i -n spire-server "$pod" -- spire-server entry show -spiffeID spiffe://production.other/spire-ha-agent | grep -q "$trustdomain"; then
      return 0
    fi
    sleep 2
    ((count++)) || true
  done
  return 1
}

run_federation_test_job() {
  kubectl delete job federation-test 2>/dev/null || true
  # Inject the images from the charts into the job so they always sync up
  yq e "(.spec.template.spec.initContainers[] | select(.name == \"static-busybox\") | .image) = \"${BUSYBOX_IMAGE}\" | (.spec.template.spec.containers[] | select(.name == \"main\") | .image) = \"${AGENT_IMAGE}\"" \
    "${SCRIPTPATH}/federation-test-job.yaml" | kubectl apply -f -
  kubectl wait --for=condition=complete --timeout=240s job/federation-test
  kubectl logs job/federation-test | grep FEDERATION-OK
}

"${SCRIPTPATH}/../../.github/scripts/prepare-local-chart-deps.sh"

# Get the package repo and install the packages
sudo curl -s -o /etc/apt/sources.list.d/spire-examples.list https://raw.githubusercontent.com/spiffe/spire-examples/refs/heads/main/examples/debs/amd64/spire-examples.list
sudo apt-get update
sudo apt-get install -y spire-common spire-agent spire-server spire-controller-manager spiffe-socat-unix socat spire-trust-sync spiffe-helper

# Set our testing trust domain
sudo sed -i 's/example.org/production.other/' /etc/spiffe/default-trust-domain.env

if [ "${BROKER}" -eq 1 ]; then
  # Pull the federation test job images out of the charts so they always sync up.
  AGENT_IMAGE=$(helm template t charts/spire -s charts/spire-agent/templates/daemonset.yaml --values "${COMMON_TEST_YOUR_VALUES}" --set spire-agent.enabled=true | yq e 'select(.kind=="DaemonSet") | .spec.template.spec.containers[] | select(.name=="spire-agent") | .image' -)
  BUSYBOX_IMAGE=$(helm template t charts/spire -s charts/spiffe-oidc-discovery-provider/templates/tests/test-keys.yaml --values "${COMMON_TEST_YOUR_VALUES}" --set spiffe-oidc-discovery-provider.enabled=true | yq e 'select(.kind=="Pod") | .spec.initContainers[] | select(.name=="static-busybox") | .image' -)
  echo "federation test job images: ${AGENT_IMAGE} ${BUSYBOX_IMAGE}"

  # Mint a trust bundle for a foreign trust domain (other.org) to test federated trust bundle
  # support. A throwaway third spire-server instance produces a genuine spiffe format bundle
  # carrying both x509 and jwt authorities. The instance env file overrides the global trust
  # domain since systemd applies later EnvironmentFiles last.
  sudo /bin/bash -c '(echo SPIFFE_TRUST_DOMAIN=other.org; echo SPIRE_BIND_PORT=8083) > /etc/spire/server/other.env'
  sudo systemctl start spire-server@other
  wait_for_healthcheck spire-server /run/spire/server/sockets/other/private/api.sock
  sudo spire-server bundle show -format spiffe -socketPath /run/spire/server/sockets/other/private/api.sock | sudo tee /tmp/other-org-bundle.json > /dev/null
  sudo systemctl stop spire-server@other
  grep -q '"x509-svid"' /tmp/other-org-bundle.json
  grep -q '"jwt-svid"' /tmp/other-org-bundle.json
fi

# register some workloads with the spire server using manifests
sudo mkdir -p /etc/spire/server/a/manifests/ /etc/spire/server/b/manifests/
sudo cp "${SCRIPTPATH}/example-manifests"/* /etc/spire/server/a/manifests/
sudo cp "${SCRIPTPATH}/example-manifests"/* /etc/spire/server/b/manifests/

# For testing, help speed up the sync
sudo rm -f /etc/spire/server/a/manifests/node1-k8s-spire-server.yaml
sudo rm -f /etc/spire/server/b/manifests/node1-k8s-spire-server.yaml

# Since we are running the two root spire servers on the same machine, we need to ensure ports do not conflict for server b
sudo /bin/bash -c 'echo SPIRE_BIND_PORT=8082 > /etc/spire/server/b.env'
sudo /bin/bash -c '(echo METRICS_BIND_ADDRESS="0.0.0.0:9125"; echo HEALTH_PROBE_BIND_ADDRESS="0.0.0.0:9126") > /etc/spire/controller-manager/b.env'

# Startup servers and make sure they are ready
sudo systemctl start spire-server@a spire-server@b spire-controller-manager@a spire-controller-manager@b
wait_for_healthcheck spire-server /run/spire/server/sockets/a/private/api.sock
wait_for_healthcheck spire-server /run/spire/server/sockets/b/private/api.sock

# Configure our agents. For the test, create join tokens for both agents. You should really use a node attestor other then join tokens such as tpm-direct, http_challenge, or a cloud provider one
JOIN_TOKEN_A=$(sudo spire-server token generate -spiffeID spiffe://production.other/agent/node1 -socketPath /run/spire/server/sockets/a/private/api.sock | awk '{print "\""$2"\""}')
JOIN_TOKEN_B=$(sudo spire-server token generate -spiffeID spiffe://production.other/agent/node1 -socketPath /run/spire/server/sockets/b/private/api.sock | awk '{print "\""$2"\""}')
export JOIN_TOKEN_A
export JOIN_TOKEN_B
sudo /bin/bash -c "echo JOIN_TOKEN=${JOIN_TOKEN_A} > /etc/spire/agent/a.env"
sudo /bin/bash -c "echo JOIN_TOKEN=${JOIN_TOKEN_B} > /etc/spire/agent/b.env"
sudo /bin/bash -c "echo SPIRE_SERVER_PORT=8082 >> /etc/spire/agent/b.env"

# Since we are running the two root spire servers on the same machine, we need to configure the trust sync instances to point to the opposite server
sudo /bin/bash -c 'echo "SPIRE_SERVER_SOCKET=/var/run/spire/server/sockets/b/private/api.sock" > /etc/spire/trust-sync/a.conf'
sudo /bin/bash -c 'echo "SPIRE_SERVER_SOCKET=/var/run/spire/server/sockets/a/private/api.sock" > /etc/spire/trust-sync/b.conf'

# Startup the agent
sudo systemctl start spire-agent@a spire-agent@b
sudo systemctl start spire-trust-sync@a spire-trust-sync@b
wait_for_healthcheck spire-agent /var/run/spire/agent/sockets/a/public/api.sock
wait_for_healthcheck spire-agent /var/run/spire/agent/sockets/b/public/api.sock
wait_for_trust_sync /var/run/spire/server/sockets/a/private/api.sock
wait_for_trust_sync /var/run/spire/server/sockets/b/private/api.sock

sudo cp "${SCRIPTPATH}/example-manifests"/node1-k8s-spire-server.yaml /etc/spire/server/a/manifests/
sudo cp "${SCRIPTPATH}/example-manifests"/node1-k8s-spire-server.yaml /etc/spire/server/b/manifests/

# Startup the socat bridge to allow the k8s spire servers to get an identity/trust bundles from the host
sudo systemctl start spiffe-socat-unix@k8s-spire-server-a spiffe-socat-unix@k8s-spire-server-b
wait_for_healthcheck spire-agent /var/run/spiffe/socat/unix/k8s-spire-server-a/public/spire-agent.sock
wait_for_healthcheck spire-agent /var/run/spiffe/socat/unix/k8s-spire-server-b/public/spire-agent.sock
wait_for_jwt /var/run/spiffe/socat/unix/k8s-spire-server-a/public/spire-agent.sock
wait_for_jwt /var/run/spiffe/socat/unix/k8s-spire-server-b/public/spire-agent.sock

# Configure and start up the socat bridges to allow the k8s spire-agents to get an identity/trust bundles from the host.
# We only have one vm mapped to multiple k8s virtual nodes in kind, so we run a pair per k8s virtual node. Normally you would only run one pair per host/vm.
sudo /bin/bash -c "echo SPIFFE_INSTANCE=a > /etc/spiffe/socat/unix/k8s-spire-agent-2-a.conf"
sudo /bin/bash -c "echo SPIFFE_INSTANCE=a > /etc/spiffe/socat/unix/k8s-spire-agent-3-a.conf"
sudo /bin/bash -c "echo SPIFFE_INSTANCE=a > /etc/spiffe/socat/unix/k8s-spire-agent-4-a.conf"
sudo /bin/bash -c "echo SPIFFE_INSTANCE=b > /etc/spiffe/socat/unix/k8s-spire-agent-2-b.conf"
sudo /bin/bash -c "echo SPIFFE_INSTANCE=b > /etc/spiffe/socat/unix/k8s-spire-agent-3-b.conf"
sudo /bin/bash -c "echo SPIFFE_INSTANCE=b > /etc/spiffe/socat/unix/k8s-spire-agent-4-b.conf"
sudo systemctl start spiffe-socat-unix@k8s-spire-agent-2-a spiffe-socat-unix@k8s-spire-agent-2-b
sudo systemctl start spiffe-socat-unix@k8s-spire-agent-3-a spiffe-socat-unix@k8s-spire-agent-3-b
sudo systemctl start spiffe-socat-unix@k8s-spire-agent-4-a spiffe-socat-unix@k8s-spire-agent-4-b
wait_for_healthcheck spire-agent /var/run/spiffe/socat/unix/k8s-spire-agent-2-a/public/api.sock
wait_for_healthcheck spire-agent /var/run/spiffe/socat/unix/k8s-spire-agent-2-b/public/api.sock
wait_for_healthcheck spire-agent /var/run/spiffe/socat/unix/k8s-spire-agent-3-a/public/api.sock
wait_for_healthcheck spire-agent /var/run/spiffe/socat/unix/k8s-spire-agent-3-b/public/api.sock
wait_for_healthcheck spire-agent /var/run/spiffe/socat/unix/k8s-spire-agent-4-a/public/api.sock
wait_for_healthcheck spire-agent /var/run/spiffe/socat/unix/k8s-spire-agent-4-b/public/api.sock
wait_for_jwt /var/run/spiffe/socat/unix/k8s-spire-agent-2-a/public/api.sock
wait_for_jwt /var/run/spiffe/socat/unix/k8s-spire-agent-2-b/public/api.sock
wait_for_jwt /var/run/spiffe/socat/unix/k8s-spire-agent-3-a/public/api.sock
wait_for_jwt /var/run/spiffe/socat/unix/k8s-spire-agent-3-b/public/api.sock
wait_for_jwt /var/run/spiffe/socat/unix/k8s-spire-agent-4-a/public/api.sock
wait_for_jwt /var/run/spiffe/socat/unix/k8s-spire-agent-4-b/public/api.sock

# Deploy an ingress controller
IP=$(kubectl get nodes chart-testing-control-plane -o go-template='{{ range .status.addresses }}{{ if eq .type "InternalIP" }}{{ .address }}{{ end }}{{ end }}')
helm upgrade --install ingress-nginx ingress-nginx --version "$VERSION_INGRESS_NGINX" --repo "$HELM_REPO_INGRESS_NGINX" \
  --namespace ingress-nginx \
  --create-namespace \
  --set "controller.extraArgs.enable-ssl-passthrough=,controller.admissionWebhooks.enabled=false,controller.service.type=ClusterIP,controller.service.externalIPs[0]=$IP" \
  --set controller.ingressClassResource.default=true \
  --wait

# Test the ingress controller. Should 404 as there is no services yet.
common_test_url "$IP"

# Get the host IP And add spire-server-[ab].${trust_domain} records to it so the spire-servers can talk back to root servers running on the host
HOSTIP=$(ip addr show docker0 | grep 'inet ' | awk '{print $2}' | cut -d/ -f1)
kubectl get configmap -n kube-system coredns -o yaml | grep hosts || kubectl get configmap -n kube-system coredns -o yaml | sed "/ready/a\        hosts {\n           fallthrough\n        }" | kubectl apply -f -
kubectl get configmap -n kube-system coredns -o yaml | grep production.other || kubectl get configmap -n kube-system coredns -o yaml | sed "/hosts/a\           $HOSTIP spire-server-a.production.other\n           $HOSTIP oidc-discovery.production.other\n           $HOSTIP spire-server-b.production.other\n" | kubectl apply -f -
kubectl rollout restart -n kube-system deployment/coredns
kubectl rollout status -n kube-system -w --timeout=1m deploy/coredns

# Install the common components
helm upgrade --install --create-namespace --namespace spire-mgmt --values "${COMMON_TEST_YOUR_VALUES},${SCRIPTPATH}/spire-values.yaml" \
  spire charts/spire-nested \
  --set tags.haAgentCommon=true \
  --set "global.spire.namespaces.create=true" \
  --set "global.spire.ingressControllerType=ingress-nginx" \
  --set "spiffe-oidc-discovery-provider.ingress.enabled=true" \
  "${BROKER_MODE_ARGS[@]}"

# Create spire-identity-exchange cert for testing.
mkdir -p certs
openssl req -x509 -newkey rsa:2048 \
    -keyout certs/server.key \
    -out certs/server.pem -sha256 -days 365 -nodes \
    -subj "/CN=localhost" \
    -addext "basicConstraints=critical,CA:TRUE" \
    -addext "subjectAltName=DNS:spire-identity-exchange.production.other,DNS:spire-identity-exchange-a.production.other,DNS:spire-identity-exchange-b.production.other"
kubectl create secret tls -n spire-server spire-identity-exchange --key=certs/server.key --cert=certs/server.pem

# Install server side a
helm upgrade --install --namespace spire-mgmt --values "${COMMON_TEST_YOUR_VALUES},${SCRIPTPATH}/spire-values.yaml" \
  --wait spire-a charts/spire-nested \
  --set tags.bottomTurtleHAA=true \
  --values "${SCRIPTPATH}/spire-identity-exchange-values.yaml" \
  --set "spire-identity-exchange-bottom-turtle-ha-a.enabled=true" \
  --set "global.spire.ingressControllerType=ingress-nginx" \
  "${BROKER_SOCKET_ARGS_A[@]}"

if [ "${BROKER}" -eq 1 ]; then
  # Install the other.org bundle so the controller manager can create the entries that federate
  # with it. It retries any entries that failed with "unable to find federated bundle".
  kubectl exec -i -n spire-server spire-a-internal-server-0 -- spire-server bundle set -format spiffe -id spiffe://other.org < /tmp/other-org-bundle.json
fi

docker exec -i chart-testing-worker /bin/bash -c "more /var/lib/kubelet/pods/*/volumes/kubernetes.io~empty-dir/disk-keymanager/keys.json /var/lib/kubelet/pods/*/volumes/kubernetes.io~empty-dir/spire-agent-persistence/agent-data.json | cat"

# Rollout just to sped up the tests
kubectl patch deployment spiffe-oidc-discovery-provider -n spire-server --type='strategic' -p '{"spec": {"strategy": {"type": "Recreate", "rollingUpdate": null}}}'
kubectl rollout restart daemonset -n spire-system spire-ha-agent
kubectl rollout status daemonset -n spire-system spire-ha-agent --timeout=1m
kubectl rollout restart deployment -n spire-server spiffe-oidc-discovery-provider
kubectl rollout status deployment -n spire-server spiffe-oidc-discovery-provider --timeout=1m
kubectl wait -n spire-server --for=condition=ready pod -l "app.kubernetes.io/name=spiffe-oidc-discovery-provider" --field-selector=status.phase=Running --timeout=90s
curl -k --resolve "oidc-discovery.production.other:443:$IP" "https://oidc-discovery.production.other/.well-known/openid-configuration" -s --fail

# Install server side b
helm upgrade --install --namespace spire-mgmt --values "${COMMON_TEST_YOUR_VALUES},${SCRIPTPATH}/spire-values.yaml" \
  --wait spire-b charts/spire-nested \
  --set tags.bottomTurtleHAB=true \
  --set internal-spire-server-bottom-turtle-ha-b.upstreamAuthority.spire.server.port=8082 \
  --values "${SCRIPTPATH}/spire-identity-exchange-values.yaml" \
  --set "spire-identity-exchange-bottom-turtle-ha-b.enabled=true" \
  --set "global.spire.ingressControllerType=ingress-nginx" \
  "${BROKER_SOCKET_ARGS_B[@]}"

if [ "${BROKER}" -eq 1 ]; then
  kubectl exec -i -n spire-server spire-b-internal-server-0 -- spire-server bundle set -format spiffe -id spiffe://other.org < /tmp/other-org-bundle.json
  # Both sides' spire-ha-agent entries must federate with other.org before the workload test.
  wait_for_entry_federation spire-a-internal-server-0 other.org
  wait_for_entry_federation spire-b-internal-server-0 other.org
fi

docker ps
docker exec -i chart-testing-worker /bin/bash -c "more /var/lib/kubelet/pods/*/volumes/kubernetes.io~empty-dir/disk-keymanager/keys.json /var/lib/kubelet/pods/*/volumes/kubernetes.io~empty-dir/spire-agent-persistence/agent-data.json | cat"

# From here on out, we sanity check that everything is working properly with both servers running.

ENTRIES="$(kubectl exec -i -n spire-server spire-b-internal-server-0 -- spire-server entry show)"
if [[ "${ENTRIES}" == "Found 0 entries" ]]; then
  echo "${ENTRIES}"
  exit 1
fi

ENTRIES="$(kubectl exec -i -n spire-server spire-a-internal-server-0 -- spire-server entry show)"
if [[ "${ENTRIES}" == "Found 0 entries" ]]; then
  echo "${ENTRIES}"
  exit 1
fi

kubectl get pods -A -o wide
kubectl get ingress -A

helm test --namespace spire-mgmt spire-a
helm test --namespace spire-mgmt spire-b
curl -k --resolve "oidc-discovery.production.other:443:$IP" "https://oidc-discovery.production.other/.well-known/openid-configuration" -s --fail

kubectl apply -f "${SCRIPTPATH}/test-job.yaml"
kubectl wait --for=condition=complete --timeout=60s job/test && \
TOKEN=$(kubectl logs job/test)
curl -f -H "Authorization: Bearer ${TOKEN}" -X POST --resolve "spire-identity-exchange-a-rest.production.other:443:$IP" "https://spire-identity-exchange-a-rest.production.other/api/v1/svid/k8s_psat/x509" -k -sS -q
curl -f -H "Authorization: Bearer ${TOKEN}" -X POST --resolve "spire-identity-exchange-b-rest.production.other:443:$IP" "https://spire-identity-exchange-b-rest.production.other/api/v1/svid/k8s_psat/x509" -k -sS -q

if [ "${BROKER}" -eq 1 ]; then
  # Verify a workload on the ha-agent socket receives the other.org federated trust bundles,
  # x509 and jwt, merged from both sides.
  run_federation_test_job
fi

#Test out running only on side b since we know already only both servers work together, and that only side a works if we made it this far.
helm delete -n spire-mgmt spire-a
kubectl rollout restart daemonset -n spire-system spire-ha-agent
kubectl rollout status daemonset -n spire-system spire-ha-agent
kubectl rollout restart deployment -n spire-server spiffe-oidc-discovery-provider
kubectl rollout status deployment -n spire-server spiffe-oidc-discovery-provider --timeout=5m
curl -k --resolve "oidc-discovery.production.other:443:$IP" "https://oidc-discovery.production.other/.well-known/openid-configuration" -s --fail

if [ "${BROKER}" -eq 1 ]; then
  # Verify the other.org federated trust bundles still serve with only side b running.
  run_federation_test_job
fi