diff --git a/clusters/homelab/apps/dynamic-runner.yaml b/clusters/homelab/apps/dynamic-runner.yaml index d67375f..06e8ac7 100644 --- a/clusters/homelab/apps/dynamic-runner.yaml +++ b/clusters/homelab/apps/dynamic-runner.yaml @@ -6,11 +6,12 @@ metadata: spec: dependsOn: - name: external-secrets - - name: nats - name: spire interval: 10m path: ./platform/dynamic-runner - prune: false + # The runner backends are replaceable. Prune is required when a retired + # worker is removed from the component; otherwise it keeps consuming work. + prune: true sourceRef: kind: GitRepository name: flux-system diff --git a/infrastructure/proxmox/ansible/roles/vyos_router/defaults/main.yml b/infrastructure/proxmox/ansible/roles/vyos_router/defaults/main.yml index f9f35e0..b00577d 100644 --- a/infrastructure/proxmox/ansible/roles/vyos_router/defaults/main.yml +++ b/infrastructure/proxmox/ansible/roles/vyos_router/defaults/main.yml @@ -85,6 +85,9 @@ vyos_sandbox_k3s_api_interface: eth1 vyos_sandbox_k3s_api_port: 6443 vyos_opensandbox_api_port: 8080 vyos_opensandbox_node_port: 30080 +# Lifecycle create is synchronous and a cold Kata image pull can exceed the +# HAProxy 50-second default. Keep this below OpenSandbox's overall timeout. +vyos_opensandbox_api_timeout: 600 vyos_sandbox_k3s_servers: - name: sandbox1 address: "10.60.0.11" diff --git a/infrastructure/proxmox/ansible/roles/vyos_router/templates/vyos.conf.j2 b/infrastructure/proxmox/ansible/roles/vyos_router/templates/vyos.conf.j2 index 1621fd4..71a433d 100644 --- a/infrastructure/proxmox/ansible/roles/vyos_router/templates/vyos.conf.j2 +++ b/infrastructure/proxmox/ansible/roles/vyos_router/templates/vyos.conf.j2 @@ -20,7 +20,9 @@ set load-balancing haproxy service opensandbox-api listen-address {{ vyos_sandbo set load-balancing haproxy service opensandbox-api port {{ vyos_opensandbox_api_port }} set load-balancing haproxy service opensandbox-api mode 'tcp' set load-balancing haproxy service opensandbox-api backend 'opensandbox-api' +set load-balancing haproxy service opensandbox-api timeout client {{ vyos_opensandbox_api_timeout }} set load-balancing haproxy backend opensandbox-api mode 'tcp' +set load-balancing haproxy backend opensandbox-api timeout server {{ vyos_opensandbox_api_timeout }} {% for server in vyos_sandbox_k3s_servers %} set load-balancing haproxy backend opensandbox-api server {{ server.name }} address {{ server.address }} set load-balancing haproxy backend opensandbox-api server {{ server.name }} port {{ vyos_opensandbox_node_port }} diff --git a/platform/dynamic-runner/deployment.yaml b/platform/dynamic-runner/deployment.yaml index 1989060..14d711d 100644 --- a/platform/dynamic-runner/deployment.yaml +++ b/platform/dynamic-runner/deployment.yaml @@ -15,11 +15,44 @@ spec: spec: serviceAccountName: dynamic-runner-controller automountServiceAccountToken: false + initContainers: + - name: fetch-internal-ca + image: curlimages/curl:8.16.0@sha256:463eaf6072688fe96ac64fa623fe73e1dbe25d8ad6c34404a669ad3ce1f104b6 + args: + - --fail + - --silent + - --show-error + - --output + - /trust/ca.pem + - https://bao.ad.ddupan.top:8200/v1/pki/ca/pem + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: [ALL] + readOnlyRootFilesystem: true + runAsNonRoot: true + runAsUser: 101 + runAsGroup: 102 + seccompProfile: + type: RuntimeDefault + volumeMounts: + - name: trust + mountPath: /trust containers: - name: controller - image: zot.ad.ddupan.top/panxiao81/gitea-dynamic-runner-controller@sha256:1e481726cbcf6da50f1ef61aa5a1b070d2859209329842e954463293c27c0e66 + image: zot.ad.ddupan.top/panxiao81/gitea-dynamic-runner-controller@sha256:83dd5c6f96b446e3aa2324e88e347b61f375fa95e99707b9b0ed505849a989da imagePullPolicy: IfNotPresent env: + - name: NATS_URL + value: tls://nats.ad.ddupan.top:4222 + - name: NATS_CA_FILE + value: /run/trust/ca.pem + - name: NATS_PRODUCER_PASSWORD_FILE + value: /run/dynamic-runner-secrets/nats-password + - name: NATS_WORKER_PASSWORD_FILE + value: /run/dynamic-runner-secrets/nats-worker-password + - name: POD_CONSUMER_ENABLED + value: "true" - name: WEBHOOK_SECRET_FILE value: /run/dynamic-runner-secrets/webhook-secret - name: REGISTRATION_TOKEN_FILE @@ -65,6 +98,9 @@ spec: - name: secret mountPath: /run/dynamic-runner-secrets readOnly: true + - name: trust + mountPath: /run/trust + readOnly: true securityContext: fsGroup: 65532 fsGroupChangePolicy: OnRootMismatch @@ -75,3 +111,6 @@ spec: secret: secretName: dynamic-runner defaultMode: 0400 + - name: trust + emptyDir: + sizeLimit: 1Mi diff --git a/platform/dynamic-runner/external-secret.yaml b/platform/dynamic-runner/external-secret.yaml index 60afce9..6d602ee 100644 --- a/platform/dynamic-runner/external-secret.yaml +++ b/platform/dynamic-runner/external-secret.yaml @@ -12,6 +12,14 @@ spec: creationPolicy: Owner name: dynamic-runner data: + - secretKey: nats-password + remoteRef: + key: k8s/nats + property: ci_producer_password + - secretKey: nats-worker-password + remoteRef: + key: k8s/nats + property: ci_worker_password - secretKey: opensandbox-api-key remoteRef: key: k8s/opensandbox-api diff --git a/platform/sandbox-ci-runners/README.md b/platform/sandbox-ci-runners/README.md index 6974ff7..93830c0 100644 --- a/platform/sandbox-ci-runners/README.md +++ b/platform/sandbox-ci-runners/README.md @@ -9,6 +9,12 @@ Pod-bound PSAT 向中央 SPIRE 注册;identity controller 从 BatchSandbox all 取得真实 Pod UID,再创建精确的 `ClusterStaticEntry`。runner 只有拿到请求中的完整 repository/task SVID 后才领取一次性 Gitea registration token。 +这些 `ClusterStaticEntry` 位于 sandbox 集群,由 central SPIRE Server 内的 +`spire-controller-manager-sandbox` 通过受限 external kubeconfig reconcile。必须在 +`platform/spire/values.yaml` 显式启用 external controller-manager 的 +`reconcile.clusterStaticEntries`(chart 默认关闭);仅看到 CR 存在但没有 status,不算 +身份链路就绪。 + ## 部署依赖 - OpenSandbox chart 和 CRD 已 Ready; diff --git a/platform/sandbox-ci-runners/identity-controller.yaml b/platform/sandbox-ci-runners/identity-controller.yaml index 90aae47..61a8c76 100644 --- a/platform/sandbox-ci-runners/identity-controller.yaml +++ b/platform/sandbox-ci-runners/identity-controller.yaml @@ -16,7 +16,7 @@ spec: serviceAccountName: opensandbox-identity containers: - name: controller - image: zot.ad.ddupan.top/panxiao81/gitea-dynamic-runner-controller@sha256:1e481726cbcf6da50f1ef61aa5a1b070d2859209329842e954463293c27c0e66 + image: zot.ad.ddupan.top/panxiao81/gitea-dynamic-runner-controller@sha256:dfbfaf2a7aa5951d1dc4e7f941a8cd05ceb4a860ac1ab23eb8be234861edcb55 imagePullPolicy: IfNotPresent command: [/venv/bin/gitea-dynamic-runner-opensandbox-identity] resources: diff --git a/platform/sandbox-ci-runners/pools.yaml b/platform/sandbox-ci-runners/pools.yaml index a0daee7..40b6868 100644 --- a/platform/sandbox-ci-runners/pools.yaml +++ b/platform/sandbox-ci-runners/pools.yaml @@ -51,7 +51,10 @@ spec: image: zot.ad.ddupan.top/panxiao81/gitea-dynamic-runner-runner@sha256:a45875fd2d0e67429b0b7bc3914735581669c705bb4e1a05d712134d2bceb86f imagePullPolicy: IfNotPresent command: [/opt/opensandbox/task-executor] - args: [-listen-addr=0.0.0.0:5758, -log-dir=/tmp] + args: + - -listen-addr=0.0.0.0:5758 + - -log-dir=/tmp + - -data-dir=/tmp/tasks env: - name: SANDBOX_MAIN_CONTAINER value: sandbox @@ -198,7 +201,10 @@ spec: image: zot.ad.ddupan.top/panxiao81/gitea-dynamic-runner-runner@sha256:a45875fd2d0e67429b0b7bc3914735581669c705bb4e1a05d712134d2bceb86f imagePullPolicy: IfNotPresent command: [/opt/opensandbox/task-executor] - args: [-listen-addr=0.0.0.0:5758, -log-dir=/tmp] + args: + - -listen-addr=0.0.0.0:5758 + - -log-dir=/tmp + - -data-dir=/tmp/tasks env: - name: SANDBOX_MAIN_CONTAINER value: sandbox diff --git a/platform/spire/values.yaml b/platform/spire/values.yaml index d628ae2..54d9bfb 100644 --- a/platform/spire/values.yaml +++ b/platform/spire/values.yaml @@ -66,6 +66,12 @@ spire-server: clusters: sandbox: kubeConfigName: sandbox-controller + # Dynamic OpenSandbox runner identities are exact Pod-UID-bound + # ClusterStaticEntries created in the sandbox cluster. The chart + # defaults this reconciler to false, so enable the central registration + # path explicitly. + reconcile: + clusterStaticEntries: true bundlePublisher: externalK8sConfigMap: enabled: true