From 758299065540c8686acaee136b801138fd210dae Mon Sep 17 00:00:00 2001 From: panxiao81 Date: Fri, 18 Sep 2026 18:53:26 +0000 Subject: [PATCH 1/5] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=20OpenSandbox=20?= =?UTF-8?q?=E5=86=B7=E5=90=AF=E5=8A=A8=E8=B6=85=E6=97=B6=E4=B8=8E=E6=97=A7?= =?UTF-8?q?=20worker=20=E6=B8=85=E7=90=86?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- clusters/homelab/apps/dynamic-runner.yaml | 5 +++-- .../proxmox/ansible/roles/vyos_router/defaults/main.yml | 3 +++ .../proxmox/ansible/roles/vyos_router/templates/vyos.conf.j2 | 2 ++ 3 files changed, 8 insertions(+), 2 deletions(-) diff --git a/clusters/homelab/apps/dynamic-runner.yaml b/clusters/homelab/apps/dynamic-runner.yaml index d67375f..06e8ac7 100644 --- a/clusters/homelab/apps/dynamic-runner.yaml +++ b/clusters/homelab/apps/dynamic-runner.yaml @@ -6,11 +6,12 @@ metadata: spec: dependsOn: - name: external-secrets - - name: nats - name: spire interval: 10m path: ./platform/dynamic-runner - prune: false + # The runner backends are replaceable. Prune is required when a retired + # worker is removed from the component; otherwise it keeps consuming work. + prune: true sourceRef: kind: GitRepository name: flux-system diff --git a/infrastructure/proxmox/ansible/roles/vyos_router/defaults/main.yml b/infrastructure/proxmox/ansible/roles/vyos_router/defaults/main.yml index f9f35e0..b00577d 100644 --- a/infrastructure/proxmox/ansible/roles/vyos_router/defaults/main.yml +++ b/infrastructure/proxmox/ansible/roles/vyos_router/defaults/main.yml @@ -85,6 +85,9 @@ vyos_sandbox_k3s_api_interface: eth1 vyos_sandbox_k3s_api_port: 6443 vyos_opensandbox_api_port: 8080 vyos_opensandbox_node_port: 30080 +# Lifecycle create is synchronous and a cold Kata image pull can exceed the +# HAProxy 50-second default. Keep this below OpenSandbox's overall timeout. +vyos_opensandbox_api_timeout: 600 vyos_sandbox_k3s_servers: - name: sandbox1 address: "10.60.0.11" diff --git a/infrastructure/proxmox/ansible/roles/vyos_router/templates/vyos.conf.j2 b/infrastructure/proxmox/ansible/roles/vyos_router/templates/vyos.conf.j2 index 1621fd4..71a433d 100644 --- a/infrastructure/proxmox/ansible/roles/vyos_router/templates/vyos.conf.j2 +++ b/infrastructure/proxmox/ansible/roles/vyos_router/templates/vyos.conf.j2 @@ -20,7 +20,9 @@ set load-balancing haproxy service opensandbox-api listen-address {{ vyos_sandbo set load-balancing haproxy service opensandbox-api port {{ vyos_opensandbox_api_port }} set load-balancing haproxy service opensandbox-api mode 'tcp' set load-balancing haproxy service opensandbox-api backend 'opensandbox-api' +set load-balancing haproxy service opensandbox-api timeout client {{ vyos_opensandbox_api_timeout }} set load-balancing haproxy backend opensandbox-api mode 'tcp' +set load-balancing haproxy backend opensandbox-api timeout server {{ vyos_opensandbox_api_timeout }} {% for server in vyos_sandbox_k3s_servers %} set load-balancing haproxy backend opensandbox-api server {{ server.name }} address {{ server.address }} set load-balancing haproxy backend opensandbox-api server {{ server.name }} port {{ vyos_opensandbox_node_port }} From 611a5a3d3a917fb5e7aa580daba44fc0b54b9f23 Mon Sep 17 00:00:00 2001 From: panxiao81 Date: Fri, 18 Sep 2026 18:59:02 +0000 Subject: [PATCH 2/5] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=20Pool=20task-executor?= =?UTF-8?q?=20=E6=95=B0=E6=8D=AE=E7=9B=AE=E5=BD=95=E4=B8=8E=20guest=20?= =?UTF-8?q?=E8=BA=AB=E4=BB=BD=E9=95=9C=E5=83=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- platform/dynamic-runner/deployment.yaml | 2 +- platform/sandbox-ci-runners/identity-controller.yaml | 2 +- platform/sandbox-ci-runners/pools.yaml | 10 ++++++++-- 3 files changed, 10 insertions(+), 4 deletions(-) diff --git a/platform/dynamic-runner/deployment.yaml b/platform/dynamic-runner/deployment.yaml index 1989060..716895a 100644 --- a/platform/dynamic-runner/deployment.yaml +++ b/platform/dynamic-runner/deployment.yaml @@ -17,7 +17,7 @@ spec: automountServiceAccountToken: false containers: - name: controller - image: zot.ad.ddupan.top/panxiao81/gitea-dynamic-runner-controller@sha256:1e481726cbcf6da50f1ef61aa5a1b070d2859209329842e954463293c27c0e66 + image: zot.ad.ddupan.top/panxiao81/gitea-dynamic-runner-controller@sha256:dfbfaf2a7aa5951d1dc4e7f941a8cd05ceb4a860ac1ab23eb8be234861edcb55 imagePullPolicy: IfNotPresent env: - name: WEBHOOK_SECRET_FILE diff --git a/platform/sandbox-ci-runners/identity-controller.yaml b/platform/sandbox-ci-runners/identity-controller.yaml index 90aae47..61a8c76 100644 --- a/platform/sandbox-ci-runners/identity-controller.yaml +++ b/platform/sandbox-ci-runners/identity-controller.yaml @@ -16,7 +16,7 @@ spec: serviceAccountName: opensandbox-identity containers: - name: controller - image: zot.ad.ddupan.top/panxiao81/gitea-dynamic-runner-controller@sha256:1e481726cbcf6da50f1ef61aa5a1b070d2859209329842e954463293c27c0e66 + image: zot.ad.ddupan.top/panxiao81/gitea-dynamic-runner-controller@sha256:dfbfaf2a7aa5951d1dc4e7f941a8cd05ceb4a860ac1ab23eb8be234861edcb55 imagePullPolicy: IfNotPresent command: [/venv/bin/gitea-dynamic-runner-opensandbox-identity] resources: diff --git a/platform/sandbox-ci-runners/pools.yaml b/platform/sandbox-ci-runners/pools.yaml index a0daee7..40b6868 100644 --- a/platform/sandbox-ci-runners/pools.yaml +++ b/platform/sandbox-ci-runners/pools.yaml @@ -51,7 +51,10 @@ spec: image: zot.ad.ddupan.top/panxiao81/gitea-dynamic-runner-runner@sha256:a45875fd2d0e67429b0b7bc3914735581669c705bb4e1a05d712134d2bceb86f imagePullPolicy: IfNotPresent command: [/opt/opensandbox/task-executor] - args: [-listen-addr=0.0.0.0:5758, -log-dir=/tmp] + args: + - -listen-addr=0.0.0.0:5758 + - -log-dir=/tmp + - -data-dir=/tmp/tasks env: - name: SANDBOX_MAIN_CONTAINER value: sandbox @@ -198,7 +201,10 @@ spec: image: zot.ad.ddupan.top/panxiao81/gitea-dynamic-runner-runner@sha256:a45875fd2d0e67429b0b7bc3914735581669c705bb4e1a05d712134d2bceb86f imagePullPolicy: IfNotPresent command: [/opt/opensandbox/task-executor] - args: [-listen-addr=0.0.0.0:5758, -log-dir=/tmp] + args: + - -listen-addr=0.0.0.0:5758 + - -log-dir=/tmp + - -data-dir=/tmp/tasks env: - name: SANDBOX_MAIN_CONTAINER value: sandbox From ad4ed1f0d6cedf5354d5e9755a7f68dabe8ae9b4 Mon Sep 17 00:00:00 2001 From: panxiao81 Date: Fri, 18 Sep 2026 19:02:44 +0000 Subject: [PATCH 3/5] =?UTF-8?q?=E9=87=8D=E6=96=B0=E8=A7=A6=E5=8F=91=20Open?= =?UTF-8?q?Sandbox=20=E9=9B=86=E6=88=90=E9=AA=8C=E8=AF=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit From 28f1de35cbdd3447d62505120cc542e0d0e9f76a Mon Sep 17 00:00:00 2001 From: panxiao81 Date: Fri, 18 Sep 2026 19:06:13 +0000 Subject: [PATCH 4/5] =?UTF-8?q?=E5=90=AF=E7=94=A8=20sandbox=20=E9=9D=99?= =?UTF-8?q?=E6=80=81=E8=BA=AB=E4=BB=BD=E7=99=BB=E8=AE=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- platform/sandbox-ci-runners/README.md | 6 ++++++ platform/spire/values.yaml | 6 ++++++ 2 files changed, 12 insertions(+) diff --git a/platform/sandbox-ci-runners/README.md b/platform/sandbox-ci-runners/README.md index 6974ff7..93830c0 100644 --- a/platform/sandbox-ci-runners/README.md +++ b/platform/sandbox-ci-runners/README.md @@ -9,6 +9,12 @@ Pod-bound PSAT 向中央 SPIRE 注册；identity controller 从 BatchSandbox all 取得真实 Pod UID，再创建精确的 `ClusterStaticEntry`。runner 只有拿到请求中的完整 repository/task SVID 后才领取一次性 Gitea registration token。 +这些 `ClusterStaticEntry` 位于 sandbox 集群，由 central SPIRE Server 内的 +`spire-controller-manager-sandbox` 通过受限 external kubeconfig reconcile。必须在 +`platform/spire/values.yaml` 显式启用 external controller-manager 的 +`reconcile.clusterStaticEntries`（chart 默认关闭）；仅看到 CR 存在但没有 status，不算 +身份链路就绪。 + ## 部署依赖 - OpenSandbox chart 和 CRD 已 Ready； diff --git a/platform/spire/values.yaml b/platform/spire/values.yaml index d628ae2..54d9bfb 100644 --- a/platform/spire/values.yaml +++ b/platform/spire/values.yaml @@ -66,6 +66,12 @@ spire-server: clusters: sandbox: kubeConfigName: sandbox-controller + # Dynamic OpenSandbox runner identities are exact Pod-UID-bound + # ClusterStaticEntries created in the sandbox cluster. The chart + # defaults this reconciler to false, so enable the central registration + # path explicitly. + reconcile: + clusterStaticEntries: true bundlePublisher: externalK8sConfigMap: enabled: true From 28c20fd3e55c5d7e059562454e6afa94c5d7714c Mon Sep 17 00:00:00 2001 From: panxiao81 Date: Fri, 18 Sep 2026 19:27:25 +0000 Subject: [PATCH 5/5] =?UTF-8?q?=E6=81=A2=E5=A4=8D=E6=8C=81=E4=B9=85?= =?UTF-8?q?=E4=BA=8B=E4=BB=B6=E9=A9=B1=E5=8A=A8=20Runner=20=E8=B0=83?= =?UTF-8?q?=E5=BA=A6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- platform/dynamic-runner/deployment.yaml | 41 +++++++++++++++++++- platform/dynamic-runner/external-secret.yaml | 8 ++++ 2 files changed, 48 insertions(+), 1 deletion(-) diff --git a/platform/dynamic-runner/deployment.yaml b/platform/dynamic-runner/deployment.yaml index 716895a..14d711d 100644 --- a/platform/dynamic-runner/deployment.yaml +++ b/platform/dynamic-runner/deployment.yaml @@ -15,11 +15,44 @@ spec: spec: serviceAccountName: dynamic-runner-controller automountServiceAccountToken: false + initContainers: + - name: fetch-internal-ca + image: curlimages/curl:8.16.0@sha256:463eaf6072688fe96ac64fa623fe73e1dbe25d8ad6c34404a669ad3ce1f104b6 + args: + - --fail + - --silent + - --show-error + - --output + - /trust/ca.pem + - https://bao.ad.ddupan.top:8200/v1/pki/ca/pem + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: [ALL] + readOnlyRootFilesystem: true + runAsNonRoot: true + runAsUser: 101 + runAsGroup: 102 + seccompProfile: + type: RuntimeDefault + volumeMounts: + - name: trust + mountPath: /trust containers: - name: controller - image: zot.ad.ddupan.top/panxiao81/gitea-dynamic-runner-controller@sha256:dfbfaf2a7aa5951d1dc4e7f941a8cd05ceb4a860ac1ab23eb8be234861edcb55 + image: zot.ad.ddupan.top/panxiao81/gitea-dynamic-runner-controller@sha256:83dd5c6f96b446e3aa2324e88e347b61f375fa95e99707b9b0ed505849a989da imagePullPolicy: IfNotPresent env: + - name: NATS_URL + value: tls://nats.ad.ddupan.top:4222 + - name: NATS_CA_FILE + value: /run/trust/ca.pem + - name: NATS_PRODUCER_PASSWORD_FILE + value: /run/dynamic-runner-secrets/nats-password + - name: NATS_WORKER_PASSWORD_FILE + value: /run/dynamic-runner-secrets/nats-worker-password + - name: POD_CONSUMER_ENABLED + value: "true" - name: WEBHOOK_SECRET_FILE value: /run/dynamic-runner-secrets/webhook-secret - name: REGISTRATION_TOKEN_FILE @@ -65,6 +98,9 @@ spec: - name: secret mountPath: /run/dynamic-runner-secrets readOnly: true + - name: trust + mountPath: /run/trust + readOnly: true securityContext: fsGroup: 65532 fsGroupChangePolicy: OnRootMismatch @@ -75,3 +111,6 @@ spec: secret: secretName: dynamic-runner defaultMode: 0400 + - name: trust + emptyDir: + sizeLimit: 1Mi diff --git a/platform/dynamic-runner/external-secret.yaml b/platform/dynamic-runner/external-secret.yaml index 60afce9..6d602ee 100644 --- a/platform/dynamic-runner/external-secret.yaml +++ b/platform/dynamic-runner/external-secret.yaml @@ -12,6 +12,14 @@ spec: creationPolicy: Owner name: dynamic-runner data: + - secretKey: nats-password + remoteRef: + key: k8s/nats + property: ci_producer_password + - secretKey: nats-worker-password + remoteRef: + key: k8s/nats + property: ci_worker_password - secretKey: opensandbox-api-key remoteRef: key: k8s/opensandbox-api