合并 OpenSandbox Runner 部署修复

2026-09-18 19:34:25 +00:00
parent ec3ce94e12 28c20fd3e5
commit 9fe910a4c9
9 changed files with 77 additions and 6 deletions
@@ -6,11 +6,12 @@ metadata:
 spec:
  dependsOn:
    - name: external-secrets
-    - name: nats
    - name: spire
  interval: 10m
  path: ./platform/dynamic-runner
-  prune: false
+  # The runner backends are replaceable. Prune is required when a retired
+  # worker is removed from the component; otherwise it keeps consuming work.
+  prune: true
  sourceRef:
    kind: GitRepository
    name: flux-system
@@ -85,6 +85,9 @@ vyos_sandbox_k3s_api_interface: eth1
 vyos_sandbox_k3s_api_port: 6443
 vyos_opensandbox_api_port: 8080
 vyos_opensandbox_node_port: 30080
+# Lifecycle create is synchronous and a cold Kata image pull can exceed the
+# HAProxy 50-second default. Keep this below OpenSandbox's overall timeout.
+vyos_opensandbox_api_timeout: 600
 vyos_sandbox_k3s_servers:
  - name: sandbox1
    address: "10.60.0.11"
@@ -20,7 +20,9 @@ set load-balancing haproxy service opensandbox-api listen-address {{ vyos_sandbo
 set load-balancing haproxy service opensandbox-api port {{ vyos_opensandbox_api_port }}
 set load-balancing haproxy service opensandbox-api mode 'tcp'
 set load-balancing haproxy service opensandbox-api backend 'opensandbox-api'
+set load-balancing haproxy service opensandbox-api timeout client {{ vyos_opensandbox_api_timeout }}
 set load-balancing haproxy backend opensandbox-api mode 'tcp'
+set load-balancing haproxy backend opensandbox-api timeout server {{ vyos_opensandbox_api_timeout }}
 {% for server in vyos_sandbox_k3s_servers %}
 set load-balancing haproxy backend opensandbox-api server {{ server.name }} address {{ server.address }}
 set load-balancing haproxy backend opensandbox-api server {{ server.name }} port {{ vyos_opensandbox_node_port }}
@@ -15,11 +15,44 @@ spec:
    spec:
      serviceAccountName: dynamic-runner-controller
      automountServiceAccountToken: false
+      initContainers:
+        - name: fetch-internal-ca
+          image: curlimages/curl:8.16.0@sha256:463eaf6072688fe96ac64fa623fe73e1dbe25d8ad6c34404a669ad3ce1f104b6
+          args:
+            - --fail
+            - --silent
+            - --show-error
+            - --output
+            - /trust/ca.pem
+            - https://bao.ad.ddupan.top:8200/v1/pki/ca/pem
+          securityContext:
+            allowPrivilegeEscalation: false
+            capabilities:
+              drop: [ALL]
+            readOnlyRootFilesystem: true
+            runAsNonRoot: true
+            runAsUser: 101
+            runAsGroup: 102
+            seccompProfile:
+              type: RuntimeDefault
+          volumeMounts:
+            - name: trust
+              mountPath: /trust
      containers:
        - name: controller
-          image: zot.ad.ddupan.top/panxiao81/gitea-dynamic-runner-controller@sha256:1e481726cbcf6da50f1ef61aa5a1b070d2859209329842e954463293c27c0e66
+          image: zot.ad.ddupan.top/panxiao81/gitea-dynamic-runner-controller@sha256:83dd5c6f96b446e3aa2324e88e347b61f375fa95e99707b9b0ed505849a989da
          imagePullPolicy: IfNotPresent
          env:
+            - name: NATS_URL
+              value: tls://nats.ad.ddupan.top:4222
+            - name: NATS_CA_FILE
+              value: /run/trust/ca.pem
+            - name: NATS_PRODUCER_PASSWORD_FILE
+              value: /run/dynamic-runner-secrets/nats-password
+            - name: NATS_WORKER_PASSWORD_FILE
+              value: /run/dynamic-runner-secrets/nats-worker-password
+            - name: POD_CONSUMER_ENABLED
+              value: "true"
            - name: WEBHOOK_SECRET_FILE
              value: /run/dynamic-runner-secrets/webhook-secret
            - name: REGISTRATION_TOKEN_FILE
@@ -65,6 +98,9 @@ spec:
            - name: secret
              mountPath: /run/dynamic-runner-secrets
              readOnly: true
+            - name: trust
+              mountPath: /run/trust
+              readOnly: true
      securityContext:
        fsGroup: 65532
        fsGroupChangePolicy: OnRootMismatch
@@ -75,3 +111,6 @@ spec:
          secret:
            secretName: dynamic-runner
            defaultMode: 0400
+        - name: trust
+          emptyDir:
+            sizeLimit: 1Mi
@@ -12,6 +12,14 @@ spec:
    creationPolicy: Owner
    name: dynamic-runner
  data:
+    - secretKey: nats-password
+      remoteRef:
+        key: k8s/nats
+        property: ci_producer_password
+    - secretKey: nats-worker-password
+      remoteRef:
+        key: k8s/nats
+        property: ci_worker_password
    - secretKey: opensandbox-api-key
      remoteRef:
        key: k8s/opensandbox-api
@@ -9,6 +9,12 @@ Pod-bound PSAT 向中央 SPIRE 注册；identity controller 从 BatchSandbox all
 取得真实 Pod UID，再创建精确的 `ClusterStaticEntry`。runner 只有拿到请求中的完整
 repository/task SVID 后才领取一次性 Gitea registration token。

+这些 `ClusterStaticEntry` 位于 sandbox 集群，由 central SPIRE Server 内的
+`spire-controller-manager-sandbox` 通过受限 external kubeconfig reconcile。必须在
+`platform/spire/values.yaml` 显式启用 external controller-manager 的
+`reconcile.clusterStaticEntries`（chart 默认关闭）；仅看到 CR 存在但没有 status，不算
+身份链路就绪。
+
 ## 部署依赖

 - OpenSandbox chart 和 CRD 已 Ready；
@@ -16,7 +16,7 @@ spec:
      serviceAccountName: opensandbox-identity
      containers:
        - name: controller
-          image: zot.ad.ddupan.top/panxiao81/gitea-dynamic-runner-controller@sha256:1e481726cbcf6da50f1ef61aa5a1b070d2859209329842e954463293c27c0e66
+          image: zot.ad.ddupan.top/panxiao81/gitea-dynamic-runner-controller@sha256:dfbfaf2a7aa5951d1dc4e7f941a8cd05ceb4a860ac1ab23eb8be234861edcb55
          imagePullPolicy: IfNotPresent
          command: [/venv/bin/gitea-dynamic-runner-opensandbox-identity]
          resources:
@@ -51,7 +51,10 @@ spec:
          image: zot.ad.ddupan.top/panxiao81/gitea-dynamic-runner-runner@sha256:a45875fd2d0e67429b0b7bc3914735581669c705bb4e1a05d712134d2bceb86f
          imagePullPolicy: IfNotPresent
          command: [/opt/opensandbox/task-executor]
-          args: [-listen-addr=0.0.0.0:5758, -log-dir=/tmp]
+          args:
+            - -listen-addr=0.0.0.0:5758
+            - -log-dir=/tmp
+            - -data-dir=/tmp/tasks
          env:
            - name: SANDBOX_MAIN_CONTAINER
              value: sandbox
@@ -198,7 +201,10 @@ spec:
          image: zot.ad.ddupan.top/panxiao81/gitea-dynamic-runner-runner@sha256:a45875fd2d0e67429b0b7bc3914735581669c705bb4e1a05d712134d2bceb86f
          imagePullPolicy: IfNotPresent
          command: [/opt/opensandbox/task-executor]
-          args: [-listen-addr=0.0.0.0:5758, -log-dir=/tmp]
+          args:
+            - -listen-addr=0.0.0.0:5758
+            - -log-dir=/tmp
+            - -data-dir=/tmp/tasks
          env:
            - name: SANDBOX_MAIN_CONTAINER
              value: sandbox
@@ -66,6 +66,12 @@ spire-server:
    clusters:
      sandbox:
        kubeConfigName: sandbox-controller
+        # Dynamic OpenSandbox runner identities are exact Pod-UID-bound
+        # ClusterStaticEntries created in the sandbox cluster. The chart
+        # defaults this reconciler to false, so enable the central registration
+        # path explicitly.
+        reconcile:
+          clusterStaticEntries: true
  bundlePublisher:
    externalK8sConfigMap:
      enabled: true