合并 OpenSandbox Runner 部署修复

clusters/homelab/apps/dynamic-runner.yaml

@@ -6,11 +6,12 @@ metadata:
 spec:
   dependsOn:
     - name: external-secrets
-    - name: nats
     - name: spire
   interval: 10m
   path: ./platform/dynamic-runner
-  prune: false
+  # The runner backends are replaceable. Prune is required when a retired
+  # worker is removed from the component; otherwise it keeps consuming work.
+  prune: true
   sourceRef:
     kind: GitRepository
     name: flux-system

infrastructure/proxmox/ansible/roles/vyos_router/defaults/main.yml

@@ -85,6 +85,9 @@ vyos_sandbox_k3s_api_interface: eth1
 vyos_sandbox_k3s_api_port: 6443
 vyos_opensandbox_api_port: 8080
 vyos_opensandbox_node_port: 30080
+# Lifecycle create is synchronous and a cold Kata image pull can exceed the
+# HAProxy 50-second default. Keep this below OpenSandbox's overall timeout.
+vyos_opensandbox_api_timeout: 600
 vyos_sandbox_k3s_servers:
   - name: sandbox1
     address: "10.60.0.11"

infrastructure/proxmox/ansible/roles/vyos_router/templates/vyos.conf.j2

@@ -20,7 +20,9 @@ set load-balancing haproxy service opensandbox-api listen-address {{ vyos_sandbo
 set load-balancing haproxy service opensandbox-api port {{ vyos_opensandbox_api_port }}
 set load-balancing haproxy service opensandbox-api mode 'tcp'
 set load-balancing haproxy service opensandbox-api backend 'opensandbox-api'
+set load-balancing haproxy service opensandbox-api timeout client {{ vyos_opensandbox_api_timeout }}
 set load-balancing haproxy backend opensandbox-api mode 'tcp'
+set load-balancing haproxy backend opensandbox-api timeout server {{ vyos_opensandbox_api_timeout }}
 {% for server in vyos_sandbox_k3s_servers %}
 set load-balancing haproxy backend opensandbox-api server {{ server.name }} address {{ server.address }}
 set load-balancing haproxy backend opensandbox-api server {{ server.name }} port {{ vyos_opensandbox_node_port }}

platform/dynamic-runner/deployment.yaml

@@ -15,11 +15,44 @@ spec:
     spec:
       serviceAccountName: dynamic-runner-controller
       automountServiceAccountToken: false
+      initContainers:
+        - name: fetch-internal-ca
+          image: curlimages/curl:8.16.0@sha256:463eaf6072688fe96ac64fa623fe73e1dbe25d8ad6c34404a669ad3ce1f104b6
+          args:
+            - --fail
+            - --silent
+            - --show-error
+            - --output
+            - /trust/ca.pem
+            - https://bao.ad.ddupan.top:8200/v1/pki/ca/pem
+          securityContext:
+            allowPrivilegeEscalation: false
+            capabilities:
+              drop: [ALL]
+            readOnlyRootFilesystem: true
+            runAsNonRoot: true
+            runAsUser: 101
+            runAsGroup: 102
+            seccompProfile:
+              type: RuntimeDefault
+          volumeMounts:
+            - name: trust
+              mountPath: /trust
       containers:
         - name: controller
-          image: zot.ad.ddupan.top/panxiao81/gitea-dynamic-runner-controller@sha256:1e481726cbcf6da50f1ef61aa5a1b070d2859209329842e954463293c27c0e66
+          image: zot.ad.ddupan.top/panxiao81/gitea-dynamic-runner-controller@sha256:83dd5c6f96b446e3aa2324e88e347b61f375fa95e99707b9b0ed505849a989da
           imagePullPolicy: IfNotPresent
           env:
+            - name: NATS_URL
+              value: tls://nats.ad.ddupan.top:4222
+            - name: NATS_CA_FILE
+              value: /run/trust/ca.pem
+            - name: NATS_PRODUCER_PASSWORD_FILE
+              value: /run/dynamic-runner-secrets/nats-password
+            - name: NATS_WORKER_PASSWORD_FILE
+              value: /run/dynamic-runner-secrets/nats-worker-password
+            - name: POD_CONSUMER_ENABLED
+              value: "true"
             - name: WEBHOOK_SECRET_FILE
               value: /run/dynamic-runner-secrets/webhook-secret
             - name: REGISTRATION_TOKEN_FILE
@@ -65,6 +98,9 @@ spec:
             - name: secret
               mountPath: /run/dynamic-runner-secrets
               readOnly: true
+            - name: trust
+              mountPath: /run/trust
+              readOnly: true
       securityContext:
         fsGroup: 65532
         fsGroupChangePolicy: OnRootMismatch
@@ -75,3 +111,6 @@ spec:
           secret:
             secretName: dynamic-runner
             defaultMode: 0400
+        - name: trust
+          emptyDir:
+            sizeLimit: 1Mi

platform/dynamic-runner/external-secret.yaml

@@ -12,6 +12,14 @@ spec:
     creationPolicy: Owner
     name: dynamic-runner
   data:
+    - secretKey: nats-password
+      remoteRef:
+        key: k8s/nats
+        property: ci_producer_password
+    - secretKey: nats-worker-password
+      remoteRef:
+        key: k8s/nats
+        property: ci_worker_password
     - secretKey: opensandbox-api-key
       remoteRef:
         key: k8s/opensandbox-api

platform/sandbox-ci-runners/README.md

@@ -9,6 +9,12 @@ Pod-bound PSAT 向中央 SPIRE 注册；identity controller 从 BatchSandbox all
 取得真实 Pod UID，再创建精确的 `ClusterStaticEntry`。runner 只有拿到请求中的完整
 repository/task SVID 后才领取一次性 Gitea registration token。
 
+这些 `ClusterStaticEntry` 位于 sandbox 集群，由 central SPIRE Server 内的
+`spire-controller-manager-sandbox` 通过受限 external kubeconfig reconcile。必须在
+`platform/spire/values.yaml` 显式启用 external controller-manager 的
+`reconcile.clusterStaticEntries`（chart 默认关闭）；仅看到 CR 存在但没有 status，不算
+身份链路就绪。
+
 ## 部署依赖
 
 - OpenSandbox chart 和 CRD 已 Ready；

platform/sandbox-ci-runners/identity-controller.yaml

@@ -16,7 +16,7 @@ spec:
       serviceAccountName: opensandbox-identity
       containers:
         - name: controller
-          image: zot.ad.ddupan.top/panxiao81/gitea-dynamic-runner-controller@sha256:1e481726cbcf6da50f1ef61aa5a1b070d2859209329842e954463293c27c0e66
+          image: zot.ad.ddupan.top/panxiao81/gitea-dynamic-runner-controller@sha256:dfbfaf2a7aa5951d1dc4e7f941a8cd05ceb4a860ac1ab23eb8be234861edcb55
           imagePullPolicy: IfNotPresent
           command: [/venv/bin/gitea-dynamic-runner-opensandbox-identity]
           resources:

platform/sandbox-ci-runners/pools.yaml

@@ -51,7 +51,10 @@ spec:
           image: zot.ad.ddupan.top/panxiao81/gitea-dynamic-runner-runner@sha256:a45875fd2d0e67429b0b7bc3914735581669c705bb4e1a05d712134d2bceb86f
           imagePullPolicy: IfNotPresent
           command: [/opt/opensandbox/task-executor]
-          args: [-listen-addr=0.0.0.0:5758, -log-dir=/tmp]
+          args:
+            - -listen-addr=0.0.0.0:5758
+            - -log-dir=/tmp
+            - -data-dir=/tmp/tasks
           env:
             - name: SANDBOX_MAIN_CONTAINER
               value: sandbox
@@ -198,7 +201,10 @@ spec:
           image: zot.ad.ddupan.top/panxiao81/gitea-dynamic-runner-runner@sha256:a45875fd2d0e67429b0b7bc3914735581669c705bb4e1a05d712134d2bceb86f
           imagePullPolicy: IfNotPresent
           command: [/opt/opensandbox/task-executor]
-          args: [-listen-addr=0.0.0.0:5758, -log-dir=/tmp]
+          args:
+            - -listen-addr=0.0.0.0:5758
+            - -log-dir=/tmp
+            - -data-dir=/tmp/tasks
           env:
             - name: SANDBOX_MAIN_CONTAINER
               value: sandbox

platform/spire/values.yaml

@@ -66,6 +66,12 @@ spire-server:
     clusters:
       sandbox:
         kubeConfigName: sandbox-controller
+        # Dynamic OpenSandbox runner identities are exact Pod-UID-bound
+        # ClusterStaticEntries created in the sandbox cluster. The chart
+        # defaults this reconciler to false, so enable the central registration
+        # path explicitly.
+        reconcile:
+          clusterStaticEntries: true
   bundlePublisher:
     externalK8sConfigMap:
       enabled: true