feat: 部署 OpenSandbox API 与双运行时 Pool

2026-09-18 01:10:07 +00:00
32 changed files with 174 additions and 449 deletions
@@ -15,8 +15,7 @@ Root bootstrap 已完成。后续按依赖顺序分别引入：
 1. 监控 CRD、kube-state-metrics 以及 kubelet/cAdvisor 抓取配置；
 2. SPIRE Agent、SPIFFE CSI Driver 与 workload registration；
 3. Kata Containers、`block-plain` RuntimeClass；
-4. 独立 External Secrets Operator 与 sandbox 专用 OpenBao auth backend；
-5. OpenSandbox controller/server；CI Pool 与 runner 调度器随后独立接入。
+4. OpenSandbox operator/server 及 `ci-pod`、`ci-vm` Pools。

 每一阶段单独合并并等待对应 Flux Kustomization Ready，不在 bootstrap 时一次性部署。
 第一阶段监控拆为 `monitoring-operator` 与依赖它的 `monitoring`，防止 VM CR 在
@@ -43,25 +42,15 @@ attestation。Server 使用 external bundle publisher 持续维护 sandbox
 显式关闭 Server 与 OIDC Provider，只部署 Agent DaemonSet 和 SPIFFE CSI Driver；因此
 不会产生第二个 trust root。

-`spire-smoke` namespace、ServiceAccount 和 `sandbox-spire-smoke` ClusterSPIFFEID 只用于
-普通 Pod 的 CSI 回归夹具，稳定身份为 `spiffe://ddupan.top/sandbox/smoke`。Kata guest
-不能复用 node Agent 暴露的 Unix socket；virtio-fs 只能呈现 socket 路径，不能把连接
-跨过 VM 边界。Kata workload 必须使用 guest 内 Agent，具体约束见
-`platform/sandbox-kata/README.md`。测试 Pod 临时创建并在验收后删除，普通 Pod 的身份
-声明保留。
+`spire-smoke` namespace、ServiceAccount 和 `sandbox-spire-smoke` ClusterSPIFFEID 是
+普通 Pod 与后续 Kata guest 的回归夹具，稳定身份为
+`spiffe://ddupan.top/sandbox/smoke`。测试 Pod 临时创建并在验收后删除，身份声明保留。

 Kata 阶段使用官方 4.1.0 `kata-deploy` chart 的短生命周期 `job` 模式，逐节点安装并
 重启 K3s。只启用 `kata-clh-runtime-rs`，不创建默认 `kata` 别名；该 handler 的
 `emptyDir` 固定使用 `block-plain`，为 Docker/BuildKit overlay2 与 kind 提供 guest
 内块设备文件系统。详细限制与上线验收见 `platform/sandbox-kata/README.md`。

-Sandbox 的 ESO 通过独立 `auth/kubernetes-sandbox` 向 OpenBao 证明 ServiceAccount
-身份，只能读取共享的 `kv/k8s/opensandbox-api`。它不保存 reviewer JWT 或长期 Bao token；相关
-Terraform 与 Flux 边界见 `platform/sandbox-external-secrets/README.md`。
-
-OpenSandbox 阶段固定官方源码 commit 与 umbrella chart `0.2.2`，只部署 controller、
-ClusterIP server 和 CRD。API key 由 ESO 从 OpenBao 投影，明文不进入 Git。
-
 ## 监控边界

 这里只管理 sandbox LXC 内的 Kubernetes 监控，不负责 PVE 宿主监控。LXC 与宿主共享
@@ -1,17 +0,0 @@
---
-apiVersion: kustomize.toolkit.fluxcd.io/v1
-kind: Kustomization
-metadata:
-  name: external-secrets-operator
-  namespace: flux-system
-spec:
-  dependsOn:
-    - name: monitoring-operator
-  interval: 10m
-  path: ./platform/sandbox-external-secrets/operator
-  prune: true
-  sourceRef:
-    kind: GitRepository
-    name: flux-system
-  timeout: 10m
-  wait: true
@@ -2,16 +2,16 @@
 apiVersion: kustomize.toolkit.fluxcd.io/v1
 kind: Kustomization
 metadata:
-  name: external-secrets
+  name: opensandbox-pools
  namespace: flux-system
 spec:
  dependsOn:
-    - name: external-secrets-operator
+    - name: opensandbox
  interval: 10m
-  path: ./platform/sandbox-external-secrets/config
+  path: ./platform/sandbox-opensandbox-pools
  prune: true
  sourceRef:
    kind: GitRepository
    name: flux-system
-  timeout: 10m
+  timeout: 20m
  wait: true
@@ -6,14 +6,13 @@ metadata:
  namespace: flux-system
 spec:
  dependsOn:
-    - name: external-secrets
    - name: kata
-    - name: monitoring-operator
+    - name: spire-agents
  interval: 10m
  path: ./platform/sandbox-opensandbox
  prune: true
  sourceRef:
    kind: GitRepository
    name: flux-system
-  timeout: 15m
+  timeout: 20m
  wait: true
@@ -7,6 +7,5 @@ resources:
  - apps/spire-bootstrap.yaml
  - apps/spire-agents.yaml
  - apps/kata.yaml
-  - apps/external-secrets-operator.yaml
-  - apps/external-secrets.yaml
  - apps/opensandbox.yaml
+  - apps/opensandbox-pools.yaml
@@ -51,7 +51,7 @@ few things Terraform must not own.**
 | secrets engine mounts (`kv`, `pki`, `ssh-client-signer`) | the daemon, Raft, TLS files, systemd |
 | PKI role, issuing/CRL URLs, cluster paths, **ACME** | `bao operator init` / unseal (manual, PGP-wrapped) |
 | SSH signing role (`ai-agent`) | **PKI root CA + SSH CA signing key** |
-| OIDC/Kubernetes auth mount、config 与 role | OIDC **client secret** (`auth/oidc/config`) |
+| OIDC auth *mount* and *role* | OIDC **client secret** (`auth/oidc/config`) |
 | all policies | snapshot token + script + systemd timer |
 |  | host-level CA trust distribution (`openbao_ssh_ca_trust`) |

@@ -92,13 +92,6 @@ then `VAULT_ADDR`/`VAULT_TOKEN`), mirroring how `smtp-relay/terraform` uses
 native `openbao/openbao` provider is published only to the OpenTofu registry and
 cannot be resolved by the HashiCorp `terraform` CLI.

-Sandbox 集群使用独立的 `auth/kubernetes-sandbox`。其 API 地址、公开 Kubernetes CA、
-ESO role 与只读 `kv/k8s/opensandbox-api` policy 全部由 Terraform 管理；CA 位于
-`terraform/certs/sandbox-kubernetes-ca.crt`。集群重建并轮换 CA 后，先更新该文件并
-apply，再让 Flux 恢复 ESO reconciliation。该 backend 不保存 reviewer JWT，而是使用
-ESO 的短期登录 JWT 执行 TokenReview。该组资源已于 2026-09-18 apply，随后复验 plan
-为 zero-diff。
-
 ## DNS

 `bao.ad.ddupan.top` is an **internal-only** name — not a public Cloudflare record and
@@ -59,33 +59,3 @@ resource "vault_kubernetes_auth_backend_role" "external_secrets" {
  # in a long TTL and every extra hour is a longer-lived credential in memory.
  token_ttl = 3600
 }
-
-# A Kubernetes auth mount can validate identities from only the API server it is
-# configured against. The sandbox cluster therefore cannot reuse auth/kubernetes,
-# whose TokenReview endpoint belongs to homelab.
-resource "vault_auth_backend" "sandbox_kubernetes" {
-  type = "kubernetes"
-  path = "kubernetes-sandbox"
-}
-
-resource "vault_kubernetes_auth_backend_config" "sandbox" {
-  backend              = vault_auth_backend.sandbox_kubernetes.path
-  kubernetes_host      = "https://10.60.0.13:6443"
-  kubernetes_ca_cert   = file("${path.module}/certs/sandbox-kubernetes-ca.crt")
-  disable_local_ca_jwt = true
-
-  # Deliberately omit token_reviewer_jwt. OpenBao uses the login JWT for
-  # TokenReview; the sandbox external-secrets ServiceAccount is bound only to
-  # system:auth-delegator and all issued JWTs remain short-lived.
-}
-
-resource "vault_kubernetes_auth_backend_role" "sandbox_external_secrets" {
-  backend   = vault_auth_backend.sandbox_kubernetes.path
-  role_name = "external-secrets"
-
-  bound_service_account_names      = ["external-secrets"]
-  bound_service_account_namespaces = ["external-secrets"]
-
-  token_policies = [vault_policy.sandbox_external_secrets.name]
-  token_ttl      = 3600
-}
@@ -1,10 +0,0 @@
-----BEGIN CERTIFICATE-----
-MIIBdjCCAR2gAwIBAgIBADAKBggqhkjOPQQDAjAjMSEwHwYDVQQDDBhrM3Mtc2Vy
-dmVyLWNhQDE3ODk2NTI4NTcwHhcNMjYwOTE3MTI0NzM3WhcNMzYwOTE0MTI0NzM3
-WjAjMSEwHwYDVQQDDBhrM3Mtc2VydmVyLWNhQDE3ODk2NTI4NTcwWTATBgcqhkjO
-PQIBBggqhkjOPQMBBwNCAAR4SbqzTXZnlZdUPz7viN6+dYbB1Maw44Qiepn9r5XG
-sOzkYkN8t1aG3Ugo8TqQ3xJaKkM89n1Rluj0vbOhiNajo0IwQDAOBgNVHQ8BAf8E
-BAMCAqQwDwYDVR0TAQH/BAUwAwEB/zAdBgNVHQ4EFgQUyLSGoKAKAJuiniuRdBLG
-XYaDQC8wCgYIKoZIzj0EAwIDRwAwRAIgFkVzyUZexk/ynnxBEOg+3foJv3WKqAei
-hTSRjO1gL0UCIFbBKR7BMrJJAgW3DJFeeBM+b+tTg93jNx55qZACbFOL
-----END CERTIFICATE-----
@@ -35,10 +35,3 @@ resource "vault_policy" "external_secrets" {
  name   = "external-secrets"
  policy = file("${path.module}/policies/external-secrets.hcl")
 }
-
-# The sandbox cluster has its own Kubernetes auth backend and a deliberately
-# narrower KV view than the homelab ESO instance.
-resource "vault_policy" "sandbox_external_secrets" {
-  name   = "sandbox-external-secrets"
-  policy = file("${path.module}/policies/sandbox-external-secrets.hcl")
-}
@@ -1,9 +0,0 @@
-# Read only the shared OpenSandbox control-plane API key. The same Bao object is
-# consumed by the server in sandbox and, later, by the scheduler in homelab.
-path "kv/data/k8s/opensandbox-api" {
-  capabilities = ["read"]
-}
-
-path "kv/metadata/k8s/opensandbox-api" {
-  capabilities = ["read"]
-}
@@ -18,8 +18,7 @@ Flux 管理以下 Kubernetes 资源：
 - Kata Containers 和 CI 专用的 `block-plain` RuntimeClass；
 - SPIRE Agent、SPIFFE CSI Driver 与 workload identity 声明；
 - vmagent、kube-state-metrics、kubelet/cAdvisor scrape 配置和告警；
- sandbox External Secrets Operator、OpenSandbox controller/server；CI Pool 与 runner
-  调度器由 runner 项目接入。
+- OpenSandbox operator/server、`ci-pod` 与 `ci-vm` Pools。

 同一个对象只能有一个 owner。Ansible 不直接部署上述集群内 workload；Flux 不管理
 LXC、K3s datastore 或 K3s 本身。
@@ -107,9 +106,7 @@ ansible-playbook verify.yml
 当前已经声明 LXC 生命周期、最小 OS baseline、PostgreSQL 和 K3s，包括系统级
 homelab CA trust。Flux `v2.9.5` controllers 与 root sync 也由 Ansible 通过 K3s
 server manifests 管理；root 使用 homelab CA 访问公开 Gitea 仓库，不保存 Git token。
-集群内 workload 由 `clusters/sandbox/` 分阶段纳入 Flux。root Kustomization 的健康检查
-timeout 为 40 分钟，用于覆盖 Kata 等首次安装时会逐节点重启 K3s 的子
-Kustomization；各子项仍保留自己的更短 timeout，故障会在对应子项先行暴露。
+集群内 workload 由 `clusters/sandbox/` 分阶段纳入 Flux。

 ## SPIRE 跨集群 bootstrap

@@ -35,5 +35,5 @@ spec:
  sourceRef:
    kind: GitRepository
    name: flux-system
-  timeout: 40m
+  timeout: 3m
  wait: true
@@ -1,53 +0,0 @@
-# Sandbox External Secrets Operator
-
-本目录在 sandbox 集群部署独立的 External Secrets Operator `2.8.0`，并通过
-`ClusterSecretStore/openbao` 读取 OpenBao KV v2 中共享的 OpenSandbox API key。它不复用
-homelab 集群的 ESO Pod、ServiceAccount 或 Kubernetes auth backend。
-
-## 当前状态
-
- OpenBao `auth/kubernetes-sandbox`、backend config、`external-secrets` role 与
-  `sandbox-external-secrets` policy 已于 2026-09-18 由 Terraform 创建；apply 后 plan
-  为 zero-diff；
- `kv/k8s/opensandbox-api` 已由本机 `spiffe://ddupan.top/dev/panxiao81` 身份生成并写入，
-  值未输出或落盘；
- sandbox ESO operator、`ClusterSecretStore/openbao` 与 OpenSandbox `ExternalSecret`
-  由 Flux 管理；
- homelab runner 对同一 key 的投影不在本目录，留给 runner 项目管理。
-
-OpenBao 的 `auth/kubernetes-sandbox`、对应 role、policy、sandbox API 地址与公开 CA
-完全由 `infrastructure/openbao/terraform/` 管理。CA 文件提交到 Git 是刻意设计：CA
-是公开信任材料，版本化后集群重建造成的 trust root 变化会产生可审计的 Terraform diff。
-
-ESO 使用 TokenRequest 生成短期 ServiceAccount JWT。OpenBao 未配置长期
-`token_reviewer_jwt`，而是使用登录 JWT 调用 sandbox TokenReview；因此
-`external-secrets` ServiceAccount 仅额外绑定内建 `system:auth-delegator`。
-
-## 重建顺序
-
-1. 在 OpenBao 写入 OpenSandbox API key：
-
-   ```bash
-   openssl rand -hex 32 | bao kv put kv/k8s/opensandbox-api api_key=-
-   ```
-
-2. 在 `infrastructure/openbao/terraform` 执行 `terraform plan` 和 `terraform apply`，
-   创建 `kubernetes-sandbox` auth mount、backend config、role 与只允许读取
-   `kv/k8s/opensandbox-api` 的最小权限 policy。
-3. 合并 Flux 变更。依次等待 `flux-system/external-secrets-operator`、
-   `flux-system/external-secrets` Ready，再等待 `flux-system/opensandbox` 滚动完成。
-
-operator 与配置拆成两个 Flux Kustomization，确保全新集群先安装 CRD，再声明
-`ClusterSecretStore`；不要为了减少目录而把两层重新合并。
-
-## 验收
-
-```bash
-kubectl get clustersecretstore openbao
-kubectl -n opensandbox-system get externalsecret opensandbox-api-key
-kubectl -n opensandbox-system get secret opensandbox-api-key
-```
-
-只检查 Secret 是否存在及 key 名，不输出 `data`。`ClusterSecretStore` 或
-`ExternalSecret` 不 Ready 时，先检查 `auth/kubernetes-sandbox`，不要临时创建静态
-Bao token Secret。
@@ -1,13 +0,0 @@
---
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRoleBinding
-metadata:
-  name: sandbox-external-secrets-token-review
-roleRef:
-  apiGroup: rbac.authorization.k8s.io
-  kind: ClusterRole
-  name: system:auth-delegator
-subjects:
-  - kind: ServiceAccount
-    name: external-secrets
-    namespace: external-secrets
@@ -1,18 +0,0 @@
---
-apiVersion: external-secrets.io/v1
-kind: ClusterSecretStore
-metadata:
-  name: openbao
-spec:
-  provider:
-    vault:
-      server: https://bao.ad.ddupan.top:8200
-      path: kv
-      version: v2
-      auth:
-        kubernetes:
-          mountPath: kubernetes-sandbox
-          role: external-secrets
-          serviceAccountRef:
-            name: external-secrets
-            namespace: external-secrets
@@ -1,35 +0,0 @@
---
-apiVersion: helm.toolkit.fluxcd.io/v2
-kind: HelmRelease
-metadata:
-  name: external-secrets
-  namespace: external-secrets
-spec:
-  chart:
-    spec:
-      chart: external-secrets
-      interval: 1h
-      sourceRef:
-        kind: HelmRepository
-        name: external-secrets
-      version: 2.8.0
-  driftDetection:
-    mode: enabled
-  install:
-    crds: CreateReplace
-    strategy:
-      name: RetryOnFailure
-      retryInterval: 5m
-  interval: 30m
-  releaseName: external-secrets
-  targetNamespace: external-secrets
-  timeout: 10m
-  upgrade:
-    crds: CreateReplace
-    strategy:
-      name: RetryOnFailure
-      retryInterval: 5m
-  valuesFrom:
-    - kind: ConfigMap
-      name: external-secrets-values
-      valuesKey: values.yaml
@@ -1,8 +0,0 @@
---
-apiVersion: kustomize.config.k8s.io/v1beta1
-kind: Kustomization
-resources:
-  - namespace.yaml
-  - repository.yaml
-  - values.yaml
-  - helmrelease.yaml
@@ -1,5 +0,0 @@
---
-apiVersion: v1
-kind: Namespace
-metadata:
-  name: external-secrets
@@ -1,9 +0,0 @@
---
-apiVersion: source.toolkit.fluxcd.io/v1
-kind: HelmRepository
-metadata:
-  name: external-secrets
-  namespace: external-secrets
-spec:
-  interval: 1h
-  url: https://charts.external-secrets.io
@@ -1,31 +0,0 @@
---
-apiVersion: v1
-kind: ConfigMap
-metadata:
-  name: external-secrets-values
-  namespace: external-secrets
-data:
-  values.yaml: |
-    replicaCount: 1
-
-    webhook:
-      replicaCount: 1
-      resources:
-        requests: {cpu: 10m, memory: 32Mi}
-        limits: {memory: 128Mi}
-
-    certController:
-      replicaCount: 1
-      resources:
-        requests: {cpu: 10m, memory: 32Mi}
-        limits: {memory: 128Mi}
-
-    resources:
-      requests: {cpu: 10m, memory: 64Mi}
-      limits: {memory: 256Mi}
-
-    serviceAccount:
-      create: true
-      name: external-secrets
-
-    installCRDs: true
@@ -28,32 +28,6 @@ Pod 删除后的 backing-file/VMM 回收和真实构建基准必须作为上线
 由 `VMPodScrape` 写入中央 VictoriaMetrics。它不拥有 Kubernetes API 凭据或 host 写
 权限。

-## Guest 内 SPIFFE 身份
-
-Kata guest 不能直接使用 node SPIRE Agent 的 CSI socket。Unix socket 的路径即使通过
-virtio-fs 出现在 guest 中，连接也不能跨 VM 边界。CI Pod 应以 native sidecar 在同一
-guest 内启动临时 SPIRE Agent，并满足以下约束：
-
-1. 外层 Pod 挂载 `audience=spire-server` 的 Pod-bound projected ServiceAccount token；
-2. 内层 Agent 使用 `k8s_psat` 向中央 Server attestation，Server cluster profile 必须
-   启用 `use_pod_uid_for_agent_id`，使 Agent ID 包含 Pod UID；
-3. 调度器为该具体 Agent 创建 registration entry；SPIFFE ID 使用仓库和任务名等稳定的
-   业务语义，Pod UID 只作为 parent binding，不进入业务身份；
-4. Agent 与 workload 通过 guest 内 `emptyDir` 上的 Unix socket 通信；Pod 必须设置
-   `shareProcessNamespace: true`，否则 `unix` workload attestor 无法从共享 `/proc`
-   解析客户端的 `SO_PEERCRED` PID；
-5. 调度器必须在 registration entry 已同步到 Agent 后才放行 workload。Pod 删除后同步
-   删除 entry，并 evict 对应的临时 Agent 记录。
-
-2026-09-17 的 live PoC 已验证：以 Pod UID 作为 parent 的临时 Agent 成功注册，
-`unix:uid:2000` workload 从 guest-local socket 获得
-`spiffe://ddupan.top/ci/poc/task/build` X.509-SVID，Pod 正常退出。PoC 资源随后全部清理，
-中央 SPIRE 恢复声明式配置。
-
-SPIRE 1.15.3 已支持 `use_pod_uid_for_agent_id`，但当前使用的 hardened chart 0.30.2
-尚未把它暴露到 values/template。正式部署应先给 chart 补齐该字段并向上游提交，随后
-采用包含修复的 release；不得把手工修改 Server ConfigMap 作为运行方案。
-
 ## 上线验收

 Flux reconciliation 完成后至少确认：
@@ -61,8 +35,8 @@ Flux reconciliation 完成后至少确认：
 1. 两个节点重新回到 Ready，`RuntimeClass/kata-clh-runtime-rs` 存在；
 2. Kata Pod 内核与 LXC host 内核不同，且 `/dev/kvm` 可用；
 3. Cloud Hypervisor API 的 `vm.info.config.memory.shared` 为 `true`；
-4. Kata Pod 内层 Agent 的 ID 包含该 Pod UID；只有调度器创建的业务 entry 所匹配的
-   workload UID 能从 guest-local socket 获得预期 SPIFFE ID；
+4. `spire-smoke` ServiceAccount 的 Kata Pod 可获得
+   `spiffe://ddupan.top/sandbox/smoke`，错误 ServiceAccount 无法获得身份；
 5. block-backed `emptyDir` 上 Docker 使用 `overlay2`，BuildKit 与 kind smoke test
   通过；kind 的 dockerd bootstrap 需要先在 guest 内执行
   `mknod /dev/kmsg c 1 11`；
@@ -1,6 +1,4 @@
---
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 resources:
-  - auth-delegator.yaml
-  - clustersecretstore.yaml
+  - pools.yaml
@@ -0,0 +1,123 @@
+---
+apiVersion: sandbox.opensandbox.io/v1alpha1
+kind: Pool
+metadata:
+  name: ci-pod
+  namespace: opensandbox
+spec:
+  template:
+    metadata:
+      labels:
+        ci.ddupan.top/backend: pod
+    spec:
+      containers:
+        - name: sandbox
+          image: sandbox-registry.cn-zhangjiakou.cr.aliyuncs.com/opensandbox/code-interpreter:v1.1.0
+          command: [/opt/opensandbox/task-executor]
+          args: [-listen-addr=0.0.0.0:5758, -log-dir=/tmp]
+          env:
+            - name: SANDBOX_MAIN_CONTAINER
+              value: sandbox
+            - name: EXECD_ENVS
+              value: /opt/opensandbox/.env
+            - name: EXECD
+              value: /opt/opensandbox/execd
+          resources:
+            requests:
+              cpu: 100m
+              memory: 256Mi
+            limits:
+              cpu: "2"
+              memory: 4Gi
+          volumeMounts:
+            - name: opensandbox-bin
+              mountPath: /opt/opensandbox
+            - name: sandbox-storage
+              mountPath: /var/lib/sandbox
+      initContainers:
+        - name: task-executor-installer
+          image: sandbox-registry.cn-zhangjiakou.cr.aliyuncs.com/opensandbox/task-executor:v0.1.0
+          command: [/bin/sh, -c]
+          args: [cp /workspace/server /opt/opensandbox/task-executor && chmod 0755 /opt/opensandbox/task-executor]
+          volumeMounts:
+            - name: opensandbox-bin
+              mountPath: /opt/opensandbox
+        - name: execd-installer
+          image: sandbox-registry.cn-zhangjiakou.cr.aliyuncs.com/opensandbox/execd:v1.0.22
+          command: [/bin/sh, -c]
+          args: [cp ./execd /opt/opensandbox/execd && cp ./bootstrap.sh /opt/opensandbox/bootstrap.sh && chmod 0755 /opt/opensandbox/execd /opt/opensandbox/bootstrap.sh]
+          volumeMounts:
+            - name: opensandbox-bin
+              mountPath: /opt/opensandbox
+      volumes:
+        - name: opensandbox-bin
+          emptyDir: {}
+        - name: sandbox-storage
+          emptyDir: {}
+  capacitySpec:
+    bufferMax: 1
+    bufferMin: 0
+    poolMax: 4
+    poolMin: 0
+---
+apiVersion: sandbox.opensandbox.io/v1alpha1
+kind: Pool
+metadata:
+  name: ci-vm
+  namespace: opensandbox
+spec:
+  template:
+    metadata:
+      labels:
+        ci.ddupan.top/backend: vm
+    spec:
+      runtimeClassName: kata-clh-runtime-rs
+      containers:
+        - name: sandbox
+          image: sandbox-registry.cn-zhangjiakou.cr.aliyuncs.com/opensandbox/code-interpreter:v1.1.0
+          command: [/opt/opensandbox/task-executor]
+          args: [-listen-addr=0.0.0.0:5758, -log-dir=/tmp]
+          env:
+            - name: SANDBOX_MAIN_CONTAINER
+              value: sandbox
+            - name: EXECD_ENVS
+              value: /opt/opensandbox/.env
+            - name: EXECD
+              value: /opt/opensandbox/execd
+          resources:
+            requests:
+              cpu: 250m
+              memory: 512Mi
+            limits:
+              cpu: "4"
+              memory: 8Gi
+          volumeMounts:
+            - name: opensandbox-bin
+              mountPath: /opt/opensandbox
+            - name: sandbox-storage
+              mountPath: /var/lib/sandbox
+      initContainers:
+        - name: task-executor-installer
+          image: sandbox-registry.cn-zhangjiakou.cr.aliyuncs.com/opensandbox/task-executor:v0.1.0
+          command: [/bin/sh, -c]
+          args: [cp /workspace/server /opt/opensandbox/task-executor && chmod 0755 /opt/opensandbox/task-executor]
+          volumeMounts:
+            - name: opensandbox-bin
+              mountPath: /opt/opensandbox
+        - name: execd-installer
+          image: sandbox-registry.cn-zhangjiakou.cr.aliyuncs.com/opensandbox/execd:v1.0.22
+          command: [/bin/sh, -c]
+          args: [cp ./execd /opt/opensandbox/execd && cp ./bootstrap.sh /opt/opensandbox/bootstrap.sh && chmod 0755 /opt/opensandbox/execd /opt/opensandbox/bootstrap.sh]
+          volumeMounts:
+            - name: opensandbox-bin
+              mountPath: /opt/opensandbox
+      volumes:
+        - name: opensandbox-bin
+          emptyDir: {}
+        - name: sandbox-storage
+          emptyDir: {}
+  capacitySpec:
+    bufferMax: 1
+    bufferMin: 0
+    poolMax: 2
+    poolMin: 0
@@ -1,42 +1,19 @@
-# Sandbox OpenSandbox
+# OpenSandbox

-本目录在独立 sandbox k3s 集群部署 OpenSandbox controller、server 与 CRD。Flux 从
-上游 commit `8f01e935c2cabba778cf37a152033fae062fa0f4` 构建官方 umbrella chart
-`0.2.2`；该源码渲染结果已与 release `opensandbox-0.2.2.tgz` 对比一致。不要改为跟随
-浮动 branch 或 tag。
+Flux installs the upstream all-in-one OpenSandbox chart pinned to
+`helm/opensandbox/0.2.2` (`8f01e935`). The API is cluster-internal and intentionally runs a
+single replica until shared server state and HA behaviour have been validated.

-server 只提供集群内 `opensandbox-server.opensandbox-system.svc:80` ClusterIP，不部署
-Gateway、Ingress 或 LoadBalancer。sandbox workload 位于 `opensandbox` namespace，
-默认使用 `kata-clh-runtime-rs`；CI Pool、runner 镜像、动态 SPIFFE registration 均由
-runner 项目后续声明，本目录不预制。
+`ci-pod` uses `runc`; `ci-vm` uses the separately managed
+`kata-clh-runtime-rs` RuntimeClass. Both Pools start at zero and create capacity
+on demand. They currently use the upstream interpreter image to validate the
+Lifecycle API and Pool allocation independently of the CI scheduler cutover.

-## API 认证
+The dynamic runner worker, runner image, guest-local SPIRE Agent and Docker
+sidecar are introduced only after this layer is Ready. In particular, do not
+mount the host SPIFFE CSI socket into `ci-vm`: Unix sockets do not cross the
+Kata VM boundary.

-`ExternalSecret/opensandbox-api-key` 从 OpenBao
-`kv/k8s/opensandbox-api:api_key` 投影同名 Secret。这个路径不归属于某个 Kubernetes
-集群：sandbox server 与 homelab runner 调度器分别通过自己的 Bao 身份读取。server 只通过
-`secretKeyRef` 读取：
-
-```yaml
- name: OPENSANDBOX_SERVER_API_KEY
-  valueFrom:
-    secretKeyRef:
-      name: opensandbox-api-key
-      key: api-key
-```
-
-仓库与 Helm values 均不保存 API key。OpenSandbox 不支持更丰富的原生 workload
-authentication；runner 后续读取同一 Bao 路径并在请求头中使用 API key。
-
-## 验收
-
-合并后等待 `flux-system/opensandbox` 与 `opensandbox-system/opensandbox` Ready，并确认：
-
-```bash
-kubectl get crd batchsandboxes.sandbox.opensandbox.io pools.sandbox.opensandbox.io
-kubectl -n opensandbox-system get deploy,pod,svc
-kubectl get runtimeclass kata-clh-runtime-rs
-```
-
-控制面上线不创建 CI Pool，也不产生 sandbox workload。首个 runner 集成应另行提交 Pool
-与完整的 Lifecycle API smoke test。
+Smoke test both backends through the same API by creating sandboxes with
+`extensions.poolRef` set to `ci-pod` and `ci-vm`, then confirm their
+BatchSandboxes, Pods and VMMs disappear after deletion.
@@ -1,19 +0,0 @@
---
-apiVersion: external-secrets.io/v1
-kind: ExternalSecret
-metadata:
-  name: opensandbox-api-key
-  namespace: opensandbox-system
-spec:
-  refreshInterval: 1h
-  secretStoreRef:
-    kind: ClusterSecretStore
-    name: openbao
-  target:
-    name: opensandbox-api-key
-    creationPolicy: Owner
-  data:
-    - secretKey: api-key
-      remoteRef:
-        key: k8s/opensandbox-api
-        property: api_key
@@ -1,4 +1,3 @@
---
 apiVersion: helm.toolkit.fluxcd.io/v2
 kind: HelmRelease
 metadata:
@@ -16,7 +15,6 @@ spec:
  driftDetection:
    mode: enabled
  install:
-    crds: CreateReplace
    strategy:
      name: RetryOnFailure
      retryInterval: 5m
@@ -25,11 +23,9 @@ spec:
  targetNamespace: opensandbox-system
  timeout: 15m
  upgrade:
-    crds: CreateReplace
    strategy:
      name: RetryOnFailure
      retryInterval: 5m
  valuesFrom:
    - kind: ConfigMap
      name: opensandbox-values
-      valuesKey: values.yaml
@@ -1,11 +1,7 @@
---
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 resources:
-  - namespace.yaml
-  - external-secret.yaml
+  - namespaces.yaml
  - repository.yaml
-  - template.yaml
  - values.yaml
  - helmrelease.yaml
-  - monitor-scrape.yaml
@@ -1,16 +0,0 @@
---
-apiVersion: operator.victoriametrics.com/v1beta1
-kind: VMPodScrape
-metadata:
-  name: opensandbox-controller
-  namespace: monitoring
-spec:
-  namespaceSelector:
-    matchNames:
-      - opensandbox-system
-  podMetricsEndpoints:
-    - interval: 30s
-      port: metrics
-  selector:
-    matchLabels:
-      app.kubernetes.io/name: opensandbox
@@ -1,4 +1,3 @@
---
 apiVersion: source.toolkit.fluxcd.io/v1
 kind: GitRepository
 metadata:
@@ -7,5 +6,6 @@ metadata:
 spec:
  interval: 1h
  ref:
-    commit: 8f01e935c2cabba778cf37a152033fae062fa0f4
-  url: https://github.com/opensandbox-group/OpenSandbox.git
+    tag: helm/opensandbox/0.2.2
+  timeout: 60s
+  url: https://github.com/alibaba/OpenSandbox.git
@@ -1,17 +0,0 @@
---
-apiVersion: v1
-kind: ConfigMap
-metadata:
-  name: opensandbox-batchsandbox-template
-  namespace: opensandbox-system
-data:
-  batchsandbox-template.yaml: |
-    metadata:
-      labels:
-        ddupan.top/workload-class: sandbox
-    spec:
-      replicas: 1
-      template:
-        spec:
-          restartPolicy: Never
-          terminationGracePeriodSeconds: 30
@@ -1,4 +1,3 @@
---
 apiVersion: v1
 kind: ConfigMap
 metadata:
@@ -8,43 +7,35 @@ data:
  values.yaml: |
    opensandbox-controller:
      controller:
+        logLevel: info
+        replicaCount: 1
        metrics:
          enabled: true
-          port: 8080
          secure: false
+          port: 8080
+        resources:
+          requests:
+            cpu: 25m
+            memory: 64Mi
+          limits:
+            cpu: 500m
+            memory: 256Mi

    opensandbox-server:
      server:
        replicaCount: 1
-        env:
-          - name: OPENSANDBOX_SERVER_API_KEY
-            valueFrom:
-              secretKeyRef:
-                name: opensandbox-api-key
-                key: api-key
        resources:
-          limits:
-            cpu: "1"
-            memory: 1Gi
          requests:
            cpu: 100m
            memory: 256Mi
-        volumeMounts:
-          - name: batchsandbox-template
-            mountPath: /etc/opensandbox/batchsandbox-template.yaml
-            subPath: batchsandbox-template.yaml
-            readOnly: true
-        volumes:
-          - name: batchsandbox-template
-            configMap:
-              name: opensandbox-batchsandbox-template
-
+          limits:
+            cpu: "1"
+            memory: 1Gi
      configToml: |
        [server]
        host = "0.0.0.0"
        port = 80
        api_key = ""
-        max_sandbox_timeout_seconds = 86400

        [log]
        level = "INFO"
@@ -53,10 +44,6 @@ data:
        type = "kubernetes"
        execd_image = "sandbox-registry.cn-zhangjiakou.cr.aliyuncs.com/opensandbox/execd:v1.0.22"

-        [storage]
-        allowed_host_paths = []
-        volume_default_size = "1Gi"
-
        [kubernetes]
        kubeconfig_path = ""
        namespace = "opensandbox"
@@ -65,17 +52,11 @@ data:
        informer_watch_timeout_seconds = 60
        snapshot_create_timeout_seconds = 900
        workload_provider = "batchsandbox"
-        image_pull_policy = "IfNotPresent"
-        batchsandbox_template_file = "/etc/opensandbox/batchsandbox-template.yaml"
+        batchsandbox_template_file = "/etc/opensandbox/example.batchsandbox-template.yaml"

        [egress]
        image = "sandbox-registry.cn-zhangjiakou.cr.aliyuncs.com/opensandbox/egress:v1.1.6"
        mode = "dns+nft"
-        disable_ipv6 = true
-
-        [secure_runtime]
-        type = "kata"
-        k8s_runtime_class = "kata-clh-runtime-rs"

    opensandbox-node-agent:
      enabled: false