在 homelab 启用原生 Pod Worker

platform/dynamic-runner/README.md

@@ -1,14 +1,18 @@
 # Gitea dynamic runner controller
 
-controller 接收 Gitea `workflow_job` webhook，对带 `[self-hosted, pod]` 或
+controller 接收 Gitea `workflow_job` webhook，并将带 `[self-hosted, pod]` 或
-`[self-hosted, vm]` 的 queued job 直接调用 OpenSandbox Lifecycle API。链路不使用
+`[self-hosted, vm]` 的 queued job 持久化到 NATS JetStream。Pod workload 在 homelab
-NATS、消息队列或平台侧 webhook worker。
+集群由原生 Kubernetes worker 创建；只有 VM workload 通过 OpenSandbox Lifecycle API。
 
 ```text
-Gitea -> dynamic-runner-controller -> http://10.60.0.13:8080/v1/sandboxes
+Gitea -> controller -> NATS -> pod worker -> homelab Kubernetes Pod
-                                      -> ci-pod / ci-vm Pool
+                           `-> VM worker  -> OpenSandbox -> Kata/microVM
 ```
 
+`dynamic-runner-pod-worker` 使用 durable `pod`，最多并发创建 4 个一次性 privileged
+Pod。任务 Pod、ServiceAccount、SPIFFE CSI socket 和生命周期都位于 homelab 的
+`dynamic-runner` namespace，不依赖 sandbox 集群或 OpenSandbox API。
+
 `10.60.0.13:8080` 是 VyOS 上的内网 HAProxy frontend，后端是 sandbox 节点的
 `30080` NodePort。它只依赖现有跨网段路由，不暴露公网，也不经过 Cloudflare
 Tunnel。controller 自身的 `192.168.10.127:8787` 仅供 Gitea webhook 和 sandbox

platform/dynamic-runner/deployment.yaml

@@ -15,11 +15,46 @@ spec:
     spec:
       serviceAccountName: dynamic-runner-controller
       automountServiceAccountToken: false
+      initContainers:
+        - name: fetch-internal-ca
+          image: curlimages/curl:8.16.0@sha256:463eaf6072688fe96ac64fa623fe73e1dbe25d8ad6c34404a669ad3ce1f104b6
+          args:
+            - --fail
+            - --silent
+            - --show-error
+            - --output
+            - /trust/ca.pem
+            - https://bao.ad.ddupan.top:8200/v1/pki/ca/pem
+          securityContext:
+            allowPrivilegeEscalation: false
+            capabilities:
+              drop: [ALL]
+            readOnlyRootFilesystem: true
+            runAsNonRoot: true
+            runAsUser: 101
+            runAsGroup: 102
+            seccompProfile:
+              type: RuntimeDefault
+          volumeMounts:
+            - name: trust
+              mountPath: /trust
       containers:
         - name: controller
-          image: zot.ad.ddupan.top/panxiao81/gitea-dynamic-runner-controller@sha256:1e481726cbcf6da50f1ef61aa5a1b070d2859209329842e954463293c27c0e66
+          image: zot.ad.ddupan.top/panxiao81/gitea-dynamic-runner-controller@sha256:9f23e84934643bbe859dc0f8da418421dc8ab7b0958fdf1fdc91d562c30e246f
           imagePullPolicy: IfNotPresent
           env:
+            - name: NATS_URL
+              value: tls://nats.ad.ddupan.top:4222
+            - name: NATS_CA_FILE
+              value: /run/trust/ca.pem
+            - name: NATS_PRODUCER_PASSWORD_FILE
+              value: /run/dynamic-runner-secrets/nats-password
+            - name: NATS_WORKER_PASSWORD_FILE
+              value: /run/dynamic-runner-secrets/nats-worker-password
+            - name: POD_CONSUMER_ENABLED
+              value: "false"
+            - name: VM_CONSUMER_ENABLED
+              value: "false"
             - name: WEBHOOK_SECRET_FILE
               value: /run/dynamic-runner-secrets/webhook-secret
             - name: REGISTRATION_TOKEN_FILE
@@ -65,6 +100,9 @@ spec:
             - name: secret
               mountPath: /run/dynamic-runner-secrets
               readOnly: true
+            - name: trust
+              mountPath: /run/trust
+              readOnly: true
       securityContext:
         fsGroup: 65532
         fsGroupChangePolicy: OnRootMismatch
@@ -75,3 +113,6 @@ spec:
           secret:
             secretName: dynamic-runner
             defaultMode: 0400
+        - name: trust
+          emptyDir:
+            sizeLimit: 1Mi

platform/dynamic-runner/external-secret.yaml

@@ -12,6 +12,14 @@ spec:
     creationPolicy: Owner
     name: dynamic-runner
   data:
+    - secretKey: nats-password
+      remoteRef:
+        key: k8s/nats
+        property: ci_producer_password
+    - secretKey: nats-worker-password
+      remoteRef:
+        key: k8s/nats
+        property: ci_worker_password
     - secretKey: opensandbox-api-key
       remoteRef:
         key: k8s/opensandbox-api

platform/dynamic-runner/kustomization.yaml

@@ -6,4 +6,5 @@ resources:
   - rbac.yaml
   - clusterspiffeid.yaml
   - deployment.yaml
+  - pod-worker-deployment.yaml
   - service.yaml

platform/dynamic-runner/pod-worker-deployment.yaml

@@ -39,8 +39,8 @@ spec:
               mountPath: /trust
       containers:
         - name: pod-worker
-          image: gitea-dynamic-runner-controller:0.3.0-bootstrap
+          image: zot.ad.ddupan.top/panxiao81/gitea-dynamic-runner-controller@sha256:9f23e84934643bbe859dc0f8da418421dc8ab7b0958fdf1fdc91d562c30e246f
-          imagePullPolicy: Never
+          imagePullPolicy: IfNotPresent
           command: [/venv/bin/gitea-dynamic-runner-pod-worker]
           env:
             - name: NATS_URL

platform/sandbox-ci-runners/README.md

@@ -9,6 +9,12 @@ Pod-bound PSAT 向中央 SPIRE 注册；identity controller 从 BatchSandbox all
 取得真实 Pod UID，再创建精确的 `ClusterStaticEntry`。runner 只有拿到请求中的完整
 repository/task SVID 后才领取一次性 Gitea registration token。
 
+这些 `ClusterStaticEntry` 位于 sandbox 集群，由 central SPIRE Server 内的
+`spire-controller-manager-sandbox` 通过受限 external kubeconfig reconcile。必须在
+`platform/spire/values.yaml` 显式启用 external controller-manager 的
+`reconcile.clusterStaticEntries`（chart 默认关闭）；仅看到 CR 存在但没有 status，不算
+身份链路就绪。
+
 ## 部署依赖
 
 - OpenSandbox chart 和 CRD 已 Ready；

platform/sandbox-ci-runners/identity-controller.yaml

@@ -16,7 +16,7 @@ spec:
       serviceAccountName: opensandbox-identity
       containers:
         - name: controller
-          image: zot.ad.ddupan.top/panxiao81/gitea-dynamic-runner-controller@sha256:1e481726cbcf6da50f1ef61aa5a1b070d2859209329842e954463293c27c0e66
+          image: zot.ad.ddupan.top/panxiao81/gitea-dynamic-runner-controller@sha256:dfbfaf2a7aa5951d1dc4e7f941a8cd05ceb4a860ac1ab23eb8be234861edcb55
           imagePullPolicy: IfNotPresent
           command: [/venv/bin/gitea-dynamic-runner-opensandbox-identity]
           resources:

platform/sandbox-ci-runners/pools.yaml

@@ -51,7 +51,10 @@ spec:
           image: zot.ad.ddupan.top/panxiao81/gitea-dynamic-runner-runner@sha256:a45875fd2d0e67429b0b7bc3914735581669c705bb4e1a05d712134d2bceb86f
           imagePullPolicy: IfNotPresent
           command: [/opt/opensandbox/task-executor]
-          args: [-listen-addr=0.0.0.0:5758, -log-dir=/tmp]
+          args:
+            - -listen-addr=0.0.0.0:5758
+            - -log-dir=/tmp
+            - -data-dir=/tmp/tasks
           env:
             - name: SANDBOX_MAIN_CONTAINER
               value: sandbox
@@ -198,7 +201,10 @@ spec:
           image: zot.ad.ddupan.top/panxiao81/gitea-dynamic-runner-runner@sha256:a45875fd2d0e67429b0b7bc3914735581669c705bb4e1a05d712134d2bceb86f
           imagePullPolicy: IfNotPresent
           command: [/opt/opensandbox/task-executor]
-          args: [-listen-addr=0.0.0.0:5758, -log-dir=/tmp]
+          args:
+            - -listen-addr=0.0.0.0:5758
+            - -log-dir=/tmp
+            - -data-dir=/tmp/tasks
           env:
             - name: SANDBOX_MAIN_CONTAINER
               value: sandbox

platform/spire/values.yaml

@@ -66,6 +66,12 @@ spire-server:
     clusters:
       sandbox:
         kubeConfigName: sandbox-controller
+        # Dynamic OpenSandbox runner identities are exact Pod-UID-bound
+        # ClusterStaticEntries created in the sandbox cluster. The chart
+        # defaults this reconciler to false, so enable the central registration
+        # path explicitly.
+        reconcile:
+          clusterStaticEntries: true
   bundlePublisher:
     externalK8sConfigMap:
       enabled: true