From 6eb4875db649e0c52f51c462131169c27f5b64e5 Mon Sep 17 00:00:00 2001 From: panxiao81 Date: Fri, 18 Sep 2026 18:33:00 +0000 Subject: [PATCH] =?UTF-8?q?=E6=8E=A5=E5=85=A5=20OpenSandbox=20=E5=8A=A8?= =?UTF-8?q?=E6=80=81=20Runner?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- clusters/sandbox/apps/ci-runners.yaml | 18 ++ clusters/sandbox/kustomization.yaml | 1 + .../roles/vyos_router/defaults/main.yml | 2 + .../roles/vyos_router/templates/vyos.conf.j2 | 12 + platform/dynamic-runner/README.md | 63 ++-- platform/dynamic-runner/deployment.yaml | 49 +-- platform/dynamic-runner/external-secret.yaml | 10 +- platform/dynamic-runner/kustomization.yaml | 1 - platform/dynamic-runner/service.yaml | 2 + platform/sandbox-ci-runners/README.md | 60 ++++ platform/sandbox-ci-runners/agent-config.yaml | 31 ++ .../identity-controller.yaml | 41 +++ .../sandbox-ci-runners/kustomization.yaml | 8 + platform/sandbox-ci-runners/pools.yaml | 295 ++++++++++++++++++ platform/sandbox-ci-runners/rbac.yaml | 87 ++++++ platform/sandbox-ci-runners/service.yaml | 15 + platform/sandbox-opensandbox/namespace.yaml | 4 + platform/spire/values.yaml | 11 + 18 files changed, 626 insertions(+), 84 deletions(-) create mode 100644 clusters/sandbox/apps/ci-runners.yaml create mode 100644 platform/sandbox-ci-runners/README.md create mode 100644 platform/sandbox-ci-runners/agent-config.yaml create mode 100644 platform/sandbox-ci-runners/identity-controller.yaml create mode 100644 platform/sandbox-ci-runners/kustomization.yaml create mode 100644 platform/sandbox-ci-runners/pools.yaml create mode 100644 platform/sandbox-ci-runners/rbac.yaml create mode 100644 platform/sandbox-ci-runners/service.yaml diff --git a/clusters/sandbox/apps/ci-runners.yaml b/clusters/sandbox/apps/ci-runners.yaml new file mode 100644 index 0000000..f1c68a5 --- /dev/null +++ b/clusters/sandbox/apps/ci-runners.yaml @@ -0,0 +1,18 @@ +--- +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: ci-runners + namespace: flux-system +spec: + dependsOn: + - name: opensandbox + - name: spire-agents + interval: 10m + path: ./platform/sandbox-ci-runners + prune: true + sourceRef: + kind: GitRepository + name: flux-system + timeout: 20m + wait: true diff --git a/clusters/sandbox/kustomization.yaml b/clusters/sandbox/kustomization.yaml index 10118f1..6ff91b9 100644 --- a/clusters/sandbox/kustomization.yaml +++ b/clusters/sandbox/kustomization.yaml @@ -10,3 +10,4 @@ resources: - apps/external-secrets-operator.yaml - apps/external-secrets.yaml - apps/opensandbox.yaml + - apps/ci-runners.yaml diff --git a/infrastructure/proxmox/ansible/roles/vyos_router/defaults/main.yml b/infrastructure/proxmox/ansible/roles/vyos_router/defaults/main.yml index 2cea4c4..f9f35e0 100644 --- a/infrastructure/proxmox/ansible/roles/vyos_router/defaults/main.yml +++ b/infrastructure/proxmox/ansible/roles/vyos_router/defaults/main.yml @@ -83,6 +83,8 @@ vyos_postgresql_primary_address: "10.60.0.11" vyos_sandbox_k3s_api_address: "10.60.0.13" vyos_sandbox_k3s_api_interface: eth1 vyos_sandbox_k3s_api_port: 6443 +vyos_opensandbox_api_port: 8080 +vyos_opensandbox_node_port: 30080 vyos_sandbox_k3s_servers: - name: sandbox1 address: "10.60.0.11" diff --git a/infrastructure/proxmox/ansible/roles/vyos_router/templates/vyos.conf.j2 b/infrastructure/proxmox/ansible/roles/vyos_router/templates/vyos.conf.j2 index 2fec2f9..1621fd4 100644 --- a/infrastructure/proxmox/ansible/roles/vyos_router/templates/vyos.conf.j2 +++ b/infrastructure/proxmox/ansible/roles/vyos_router/templates/vyos.conf.j2 @@ -15,6 +15,18 @@ set interfaces ethernet {{ i.iface }} address {{ i.address }} set interfaces ethernet {{ i.iface }} description '{{ i.description }}' {% endfor %} +{# OpenSandbox stays on a NodePort; VyOS owns the stable routed frontend. #} +set load-balancing haproxy service opensandbox-api listen-address {{ vyos_sandbox_k3s_api_address }} +set load-balancing haproxy service opensandbox-api port {{ vyos_opensandbox_api_port }} +set load-balancing haproxy service opensandbox-api mode 'tcp' +set load-balancing haproxy service opensandbox-api backend 'opensandbox-api' +set load-balancing haproxy backend opensandbox-api mode 'tcp' +{% for server in vyos_sandbox_k3s_servers %} +set load-balancing haproxy backend opensandbox-api server {{ server.name }} address {{ server.address }} +set load-balancing haproxy backend opensandbox-api server {{ server.name }} port {{ vyos_opensandbox_node_port }} +set load-balancing haproxy backend opensandbox-api server {{ server.name }} check +{% endfor %} + {# --- default route out; OSPF carries the rest --- #} set protocols static route 0.0.0.0/0 next-hop {{ vyos_lan_gateway }} diff --git a/platform/dynamic-runner/README.md b/platform/dynamic-runner/README.md index a38e948..ab1501c 100644 --- a/platform/dynamic-runner/README.md +++ b/platform/dynamic-runner/README.md @@ -1,51 +1,40 @@ # Gitea dynamic runner controller -此目录只管理 homelab 中的 controller 部署。controller、worker、Cloud Hypervisor -launcher 和 guest runner 的源码与发布位于独立仓库 -`panxiao81/gitea-dynamic-runner`。 - -当前 bootstrap controller 接收 Gitea `workflow_job` webhook，将 `[self-hosted, pod]` 和 -`[self-hosted, vm]` 的 queued job 分别发布到 NATS。Pod worker 在本集群创建一次性 -privileged host runner；Docker、BuildKit 和 kind 由 workflow 自行 setup。内部 -endpoint： +controller 接收 Gitea `workflow_job` webhook，对带 `[self-hosted, pod]` 或 +`[self-hosted, vm]` 的 queued job 直接调用 OpenSandbox Lifecycle API。链路不使用 +NATS、消息队列或平台侧 webhook worker。 ```text -http://dynamic-runner-controller.dynamic-runner.svc.cluster.local:8787/webhook +Gitea -> dynamic-runner-controller -> http://10.60.0.13:8080/v1/sandboxes + -> ci-pod / ci-vm Pool ``` -OpenBao 路径： +`10.60.0.13:8080` 是 VyOS 上的内网 HAProxy frontend，后端是 sandbox 节点的 +`30080` NodePort。它只依赖现有跨网段路由，不暴露公网，也不经过 Cloudflare +Tunnel。controller 自身的 `192.168.10.127:8787` 仅供 Gitea webhook 和 sandbox +通过一次性 nonce 领取 registration token。 -- `kv/k8s/nats.ci_producer_password`：已有 NATS producer 密码。 -- `kv/k8s/nats.ci_worker_password`：已有 NATS worker 密码。 -- `kv/k8s/dynamic-runner.webhook_secret`：Gitea webhook HMAC secret。 -- `kv/k8s/gitea-runner.token`：现有 instance runner registration token。 +## Secret 边界 -首期 controller 与 runner 镜像由 laptop 本机构建后导入 k3s containerd，作为 CI -发布链路建立前的 bootstrap。部署使用 `imagePullPolicy: Never`。正式发布 workflow -获得专用 SPIFFE ID 后，必须将 image 改为 zot digest 并移除本地导入步骤。 +homelab 的 `ExternalSecret/dynamic-runner` 从既有 `ClusterSecretStore/openbao` 读取： -## 身份绑定 +- `kv/k8s/opensandbox-api:api_key` -> `opensandbox-api-key`； +- `kv/k8s/dynamic-runner:webhook_secret` -> `webhook-secret`； +- `kv/k8s/gitea-runner:token` -> `token`。 -queued webhook 只负责创建没有业务身份的 Pod。runner 实际领取任务后，Gitea 的 -`in_progress` webhook 会携带实际 `runner_name`；controller 将 binding 消息发布到 -NATS，Pod worker 再给对应 Pod 添加： +API key 以只读文件挂载，controller 通过 `OPEN-SANDBOX-API-KEY` header 使用。不要把 +key 复制到 Git、Lifecycle 请求、BatchSandbox 或 sandbox 集群 Secret。本目录不创建 +Bao token，也不拥有 sandbox 平台侧的 ESO/Bao Terraform。 -```text -ci.ddupan.top/identity-bound=true -ci.ddupan.top/spiffe-path=// -``` +## 调度与身份 -`ClusterSPIFFEID/gitea-dynamic-runner` 只匹配已经绑定的 Pod，并签发 -`spiffe://ddupan.top/ci///`。runner 的 job-start hook 在 -SVID 可用之前不会放行第一步，因此不能根据 queued 事件错配身份。 +Lifecycle 请求只按 `ci-pod` / `ci-vm` 选择 Pool，并携带稳定的 +`spiffe://ddupan.top/ci///`。job ID 只用于日志和诊断 metadata。 +runner labels 始终以 `self-hosted` 开头。 -每个 runner Pod 使用 `gitea-dynamic-runner` ServiceAccount。该 ServiceAccount 没有 -Kubernetes API 权限；只有 `dynamic-runner-pod-worker` ServiceAccount 能在本 namespace -create/get/patch/delete Pod。 +registration token 通过 controller 内存中的单次 nonce URL 投递。成功领取后 nonce +立即失效；沙箱结束、创建失败或超时时也会撤销。OpenSandbox DELETE 是最终的正常 +回收路径，OpenSandbox timeout 是 controller 异常退出时的兜底。 -长期实现将由兼容 Gitea RunnerService 的 scheduler 直接领取 task，再交给 Pod/VM -executor；届时删除 webhook、临时 runner 注册和 identity binding 消息。跟踪见 -`panxiao81/gitea-dynamic-runner` issue #7。 - -Gitea webhook 只订阅 `workflow_job`，content type 使用 JSON，secret 与 Bao 中值 -一致。不要启用 `send_everything`，否则 controller 会收到无关仓库事件。 +完整部署和验收步骤见 +[`platform/sandbox-ci-runners/README.md`](../sandbox-ci-runners/README.md)。 diff --git a/platform/dynamic-runner/deployment.yaml b/platform/dynamic-runner/deployment.yaml index 259d0af..1989060 100644 --- a/platform/dynamic-runner/deployment.yaml +++ b/platform/dynamic-runner/deployment.yaml @@ -15,44 +15,21 @@ spec: spec: serviceAccountName: dynamic-runner-controller automountServiceAccountToken: false - initContainers: - - name: fetch-internal-ca - image: curlimages/curl:8.16.0@sha256:463eaf6072688fe96ac64fa623fe73e1dbe25d8ad6c34404a669ad3ce1f104b6 - args: - - --fail - - --silent - - --show-error - - --output - - /trust/ca.pem - - https://bao.ad.ddupan.top:8200/v1/pki/ca/pem - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: [ALL] - readOnlyRootFilesystem: true - runAsNonRoot: true - runAsUser: 101 - runAsGroup: 102 - seccompProfile: - type: RuntimeDefault - volumeMounts: - - name: trust - mountPath: /trust containers: - name: controller - # Bootstrap import on laptop. Replace with a zot digest after the - # repository's image publishing workflow has a dedicated identity. - image: gitea-dynamic-runner-controller:0.3.0-bootstrap - imagePullPolicy: Never + image: zot.ad.ddupan.top/panxiao81/gitea-dynamic-runner-controller@sha256:1e481726cbcf6da50f1ef61aa5a1b070d2859209329842e954463293c27c0e66 + imagePullPolicy: IfNotPresent env: - - name: NATS_URL - value: tls://nats.ad.ddupan.top:4222 - - name: NATS_CA_FILE - value: /run/trust/ca.pem - - name: NATS_PASSWORD_FILE - value: /run/dynamic-runner-secrets/nats-password - name: WEBHOOK_SECRET_FILE value: /run/dynamic-runner-secrets/webhook-secret + - name: REGISTRATION_TOKEN_FILE + value: /run/dynamic-runner-secrets/token + - name: OPENSANDBOX_API + value: http://10.60.0.13:8080 + - name: OPENSANDBOX_API_KEY_FILE + value: /run/dynamic-runner-secrets/opensandbox-api-key + - name: RUNNER_TOKEN_BASE_URL + value: http://192.168.10.127:8787/token ports: - name: http containerPort: 8787 @@ -88,9 +65,6 @@ spec: - name: secret mountPath: /run/dynamic-runner-secrets readOnly: true - - name: trust - mountPath: /run/trust - readOnly: true securityContext: fsGroup: 65532 fsGroupChangePolicy: OnRootMismatch @@ -101,6 +75,3 @@ spec: secret: secretName: dynamic-runner defaultMode: 0400 - - name: trust - emptyDir: - sizeLimit: 1Mi diff --git a/platform/dynamic-runner/external-secret.yaml b/platform/dynamic-runner/external-secret.yaml index 7baa41d..60afce9 100644 --- a/platform/dynamic-runner/external-secret.yaml +++ b/platform/dynamic-runner/external-secret.yaml @@ -12,14 +12,10 @@ spec: creationPolicy: Owner name: dynamic-runner data: - - secretKey: nats-password + - secretKey: opensandbox-api-key remoteRef: - key: k8s/nats - property: ci_producer_password - - secretKey: nats-worker-password - remoteRef: - key: k8s/nats - property: ci_worker_password + key: k8s/opensandbox-api + property: api_key - secretKey: webhook-secret remoteRef: key: k8s/dynamic-runner diff --git a/platform/dynamic-runner/kustomization.yaml b/platform/dynamic-runner/kustomization.yaml index 6bf792e..00d940d 100644 --- a/platform/dynamic-runner/kustomization.yaml +++ b/platform/dynamic-runner/kustomization.yaml @@ -6,5 +6,4 @@ resources: - rbac.yaml - clusterspiffeid.yaml - deployment.yaml - - pod-worker-deployment.yaml - service.yaml diff --git a/platform/dynamic-runner/service.yaml b/platform/dynamic-runner/service.yaml index e12f4b8..33df06c 100644 --- a/platform/dynamic-runner/service.yaml +++ b/platform/dynamic-runner/service.yaml @@ -4,6 +4,8 @@ metadata: name: dynamic-runner-controller namespace: dynamic-runner spec: + type: LoadBalancer + loadBalancerIP: 192.168.10.127 selector: app.kubernetes.io/name: dynamic-runner-controller ports: diff --git a/platform/sandbox-ci-runners/README.md b/platform/sandbox-ci-runners/README.md new file mode 100644 index 0000000..6974ff7 --- /dev/null +++ b/platform/sandbox-ci-runners/README.md @@ -0,0 +1,60 @@ +# OpenSandbox Gitea runner Pools + +本目录部署 `ci-vm` 和 `ci-pod` 两个零预热 Pool、sandbox 内的 SPIFFE identity +controller，以及仅供内网 VyOS 转发的 OpenSandbox NodePort。`ci-vm` 使用 +`kata-clh-runtime-rs`，`ci-pod` 使用默认 runtime；两者均为单任务、用后删除。 + +Pool 中 task-executor 接收 Lifecycle API 下发的进程环境。guest-local SPIRE Agent 用 +Pod-bound PSAT 向中央 SPIRE 注册；identity controller 从 BatchSandbox allocation +取得真实 Pod UID，再创建精确的 `ClusterStaticEntry`。runner 只有拿到请求中的完整 +repository/task SVID 后才领取一次性 Gitea registration token。 + +## 部署依赖 + +- OpenSandbox chart 和 CRD 已 Ready； +- RuntimeClass `kata-clh-runtime-rs` 已存在； +- 中央 SPIRE 已发布 `ConfigMap/opensandbox/spire-bundle-pem`； +- VyOS `10.60.0.13:8080` 转发 sandbox1/2 的 NodePort `30080`； +- runner/controller 镜像均使用 Zot digest，而不是可变 tag。 + +本目录不读取 OpenBao，也不修改 OpenSandbox 平台侧 ExternalSecret、ClusterSecretStore +或 Bao policy。OpenSandbox API key 只存在于平台 server Secret 和 homelab controller +Secret，两边由各自身份读取同一 Bao 资源。 + +## 上线验收 + +先确认 Secret 和 API 认证，命令不得输出 key： + +```bash +kubectl -n dynamic-runner wait externalsecret/dynamic-runner --for=condition=Ready --timeout=2m +kubectl -n dynamic-runner exec deploy/dynamic-runner-controller -- \ + wget -qO- http://10.60.0.13:8080/health +``` + +然后触发一个 `runs-on: [self-hosted, vm]` 的最小 workflow，并同时观察： + +```bash +kubectl -n dynamic-runner logs deploy/dynamic-runner-controller -f +kubectl -n opensandbox get pool,batchsandbox,pod -w +kubectl get clusterstaticentry -l app.kubernetes.io/component=opensandbox-identity -w +kubectl -n opensandbox logs deploy/opensandbox-identity -f +``` + +合格证据必须同时包含： + +1. Lifecycle create 成功并选择 `ci-vm`； +2. 分配 Pod 的 `runtimeClassName` 为 `kata-clh-runtime-rs`； +3. entry 的 parent ID 包含该 Pod 的 UID，SPIFFE ID 使用 repository/task； +4. Gitea 显示临时 runner 带 `self-hosted,vm` labels 并完成真实任务； +5. 任务后 BatchSandbox、Pod、ClusterStaticEntry 和临时 runner 均消失。 + +清理超时对象时只调用 Lifecycle DELETE，不直接删除 Pool Pod。若 controller 已不可用， +可从 OpenSandbox API 按 metadata 定位 sandbox 后执行 DELETE；不要绕过 API 伪造状态。 + +## 故障定位 + +- `401`：检查 homelab ExternalSecret Ready 和文件挂载，不打印 Secret； +- `PoolCapacityExhausted`：检查 `ci-vm` 的 `poolMax` 及残留 BatchSandbox； +- runner 等待 SVID：核对 allocation Pod UID、ClusterStaticEntry parentID、guest Agent 日志； +- runner 等待 token：核对 `192.168.10.127:8787` 的 sandbox 到 homelab 路由； +- Docker 任务失败：检查 `docker` sidecar 和 `/run/docker/docker.sock` 的 group 2000。 diff --git a/platform/sandbox-ci-runners/agent-config.yaml b/platform/sandbox-ci-runners/agent-config.yaml new file mode 100644 index 0000000..976a895 --- /dev/null +++ b/platform/sandbox-ci-runners/agent-config.yaml @@ -0,0 +1,31 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: gitea-ci-spire-agent + namespace: opensandbox +data: + agent.conf: | + agent { + data_dir = "/run/spire/data" + log_level = "INFO" + server_address = "spire-server.ad.ddupan.top" + server_port = "8081" + socket_path = "/run/spire/agent-sockets/spire-agent.sock" + trust_bundle_path = "/run/spire/bundle/bundle.pem" + trust_domain = "ddupan.top" + } + + plugins { + NodeAttestor "k8s_psat" { + plugin_data { + cluster = "sandbox-kata" + token_path = "/run/spire/tokens/token" + } + } + KeyManager "memory" { + plugin_data {} + } + WorkloadAttestor "unix" { + plugin_data {} + } + } diff --git a/platform/sandbox-ci-runners/identity-controller.yaml b/platform/sandbox-ci-runners/identity-controller.yaml new file mode 100644 index 0000000..90aae47 --- /dev/null +++ b/platform/sandbox-ci-runners/identity-controller.yaml @@ -0,0 +1,41 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: opensandbox-identity + namespace: opensandbox +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: opensandbox-identity + template: + metadata: + labels: + app.kubernetes.io/name: opensandbox-identity + spec: + serviceAccountName: opensandbox-identity + containers: + - name: controller + image: zot.ad.ddupan.top/panxiao81/gitea-dynamic-runner-controller@sha256:1e481726cbcf6da50f1ef61aa5a1b070d2859209329842e954463293c27c0e66 + imagePullPolicy: IfNotPresent + command: [/venv/bin/gitea-dynamic-runner-opensandbox-identity] + resources: + requests: + cpu: 10m + memory: 32Mi + limits: + cpu: 100m + memory: 96Mi + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: [ALL] + readOnlyRootFilesystem: true + runAsNonRoot: true + runAsUser: 65532 + runAsGroup: 65532 + seccompProfile: + type: RuntimeDefault + securityContext: + seccompProfile: + type: RuntimeDefault diff --git a/platform/sandbox-ci-runners/kustomization.yaml b/platform/sandbox-ci-runners/kustomization.yaml new file mode 100644 index 0000000..0d00954 --- /dev/null +++ b/platform/sandbox-ci-runners/kustomization.yaml @@ -0,0 +1,8 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - service.yaml + - agent-config.yaml + - rbac.yaml + - identity-controller.yaml + - pools.yaml diff --git a/platform/sandbox-ci-runners/pools.yaml b/platform/sandbox-ci-runners/pools.yaml new file mode 100644 index 0000000..a0daee7 --- /dev/null +++ b/platform/sandbox-ci-runners/pools.yaml @@ -0,0 +1,295 @@ +--- +apiVersion: sandbox.opensandbox.io/v1alpha1 +kind: Pool +metadata: + name: ci-vm + namespace: opensandbox + labels: + app.kubernetes.io/name: gitea-ci-vm + app.kubernetes.io/component: runner-pool +spec: + capacitySpec: + bufferMax: 0 + bufferMin: 0 + poolMax: 2 + poolMin: 0 + recycleStrategy: + type: Delete + template: + metadata: + labels: + app.kubernetes.io/name: gitea-ci-vm + ci.ddupan.top/backend: vm + spec: + runtimeClassName: kata-clh-runtime-rs + serviceAccountName: gitea-ci + restartPolicy: Never + terminationGracePeriodSeconds: 30 + shareProcessNamespace: true + securityContext: + fsGroup: 2000 + fsGroupChangePolicy: OnRootMismatch + initContainers: + - name: task-executor-installer + image: sandbox-registry.cn-zhangjiakou.cr.aliyuncs.com/opensandbox/task-executor:v0.1.0 + command: [/bin/sh, -c] + args: + - cp /workspace/server /opt/opensandbox/task-executor && chmod 0755 /opt/opensandbox/task-executor + volumeMounts: + - name: opensandbox-bin + mountPath: /opt/opensandbox + - name: execd-installer + image: sandbox-registry.cn-zhangjiakou.cr.aliyuncs.com/opensandbox/execd:v1.0.22 + command: [/bin/sh, -c] + args: + - cp ./execd /opt/opensandbox/execd && cp ./bootstrap.sh /opt/opensandbox/bootstrap.sh && chmod 0755 /opt/opensandbox/execd /opt/opensandbox/bootstrap.sh + volumeMounts: + - name: opensandbox-bin + mountPath: /opt/opensandbox + containers: + - name: sandbox + image: zot.ad.ddupan.top/panxiao81/gitea-dynamic-runner-runner@sha256:a45875fd2d0e67429b0b7bc3914735581669c705bb4e1a05d712134d2bceb86f + imagePullPolicy: IfNotPresent + command: [/opt/opensandbox/task-executor] + args: [-listen-addr=0.0.0.0:5758, -log-dir=/tmp] + env: + - name: SANDBOX_MAIN_CONTAINER + value: sandbox + - name: EXECD_ENVS + value: /opt/opensandbox/.env + - name: EXECD + value: /opt/opensandbox/execd + - name: GITEA_INSTANCE_URL + value: https://git.ddupan.top + - name: HOME + value: /data + - name: DOCKER_HOST + value: unix:///run/docker/docker.sock + ports: + - name: task-executor + containerPort: 5758 + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: [ALL] + runAsNonRoot: true + runAsUser: 2000 + runAsGroup: 2000 + volumeMounts: + - name: opensandbox-bin + mountPath: /opt/opensandbox + - name: runner-data + mountPath: /data + - name: docker-socket + mountPath: /run/docker + - name: spire-socket + mountPath: /run/spire/agent-sockets + - name: spire-agent + image: ghcr.io/spiffe/spire-agent:1.15.3@sha256:41b0dcd8b258a69db9e2768292a060766fb76fd866e4bc925849981ea1b825ff + args: [-config, /run/spire/config/agent.conf] + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: [ALL] + volumeMounts: + - name: spire-config + mountPath: /run/spire/config + readOnly: true + - name: spire-bundle + mountPath: /run/spire/bundle + readOnly: true + - name: spire-token + mountPath: /run/spire/tokens + readOnly: true + - name: spire-data + mountPath: /run/spire/data + - name: spire-socket + mountPath: /run/spire/agent-sockets + - name: docker + image: docker.io/library/docker:29.1.5-dind + command: [/bin/sh, -c] + args: + - test -e /dev/kmsg || mknod /dev/kmsg c 1 11; exec dockerd --host=unix:///run/docker/docker.sock --group=2000 --storage-driver=overlay2 + securityContext: + privileged: true + volumeMounts: + - name: docker-socket + mountPath: /run/docker + - name: docker-data + mountPath: /var/lib/docker + volumes: + - name: opensandbox-bin + emptyDir: {} + - name: runner-data + emptyDir: {} + - name: docker-data + emptyDir: {} + - name: docker-socket + emptyDir: + medium: Memory + - name: spire-data + emptyDir: + medium: Memory + - name: spire-socket + emptyDir: + medium: Memory + - name: spire-config + configMap: + name: gitea-ci-spire-agent + - name: spire-bundle + configMap: + name: spire-bundle-pem + - name: spire-token + projected: + sources: + - serviceAccountToken: + audience: spire-server + expirationSeconds: 3600 + path: token +--- +apiVersion: sandbox.opensandbox.io/v1alpha1 +kind: Pool +metadata: + name: ci-pod + namespace: opensandbox + labels: + app.kubernetes.io/name: gitea-ci-pod + app.kubernetes.io/component: runner-pool +spec: + capacitySpec: + bufferMax: 0 + bufferMin: 0 + poolMax: 4 + poolMin: 0 + recycleStrategy: + type: Delete + template: + metadata: + labels: + app.kubernetes.io/name: gitea-ci-pod + ci.ddupan.top/backend: pod + spec: + serviceAccountName: gitea-ci + restartPolicy: Never + terminationGracePeriodSeconds: 30 + shareProcessNamespace: true + securityContext: + fsGroup: 2000 + fsGroupChangePolicy: OnRootMismatch + initContainers: + - name: task-executor-installer + image: sandbox-registry.cn-zhangjiakou.cr.aliyuncs.com/opensandbox/task-executor:v0.1.0 + command: [/bin/sh, -c] + args: + - cp /workspace/server /opt/opensandbox/task-executor && chmod 0755 /opt/opensandbox/task-executor + volumeMounts: + - name: opensandbox-bin + mountPath: /opt/opensandbox + - name: execd-installer + image: sandbox-registry.cn-zhangjiakou.cr.aliyuncs.com/opensandbox/execd:v1.0.22 + command: [/bin/sh, -c] + args: + - cp ./execd /opt/opensandbox/execd && cp ./bootstrap.sh /opt/opensandbox/bootstrap.sh && chmod 0755 /opt/opensandbox/execd /opt/opensandbox/bootstrap.sh + volumeMounts: + - name: opensandbox-bin + mountPath: /opt/opensandbox + containers: + - name: sandbox + image: zot.ad.ddupan.top/panxiao81/gitea-dynamic-runner-runner@sha256:a45875fd2d0e67429b0b7bc3914735581669c705bb4e1a05d712134d2bceb86f + imagePullPolicy: IfNotPresent + command: [/opt/opensandbox/task-executor] + args: [-listen-addr=0.0.0.0:5758, -log-dir=/tmp] + env: + - name: SANDBOX_MAIN_CONTAINER + value: sandbox + - name: EXECD_ENVS + value: /opt/opensandbox/.env + - name: EXECD + value: /opt/opensandbox/execd + - name: GITEA_INSTANCE_URL + value: https://git.ddupan.top + - name: HOME + value: /data + - name: DOCKER_HOST + value: unix:///run/docker/docker.sock + ports: + - name: task-executor + containerPort: 5758 + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: [ALL] + runAsNonRoot: true + runAsUser: 2000 + runAsGroup: 2000 + volumeMounts: + - name: opensandbox-bin + mountPath: /opt/opensandbox + - name: runner-data + mountPath: /data + - name: docker-socket + mountPath: /run/docker + - name: spire-socket + mountPath: /run/spire/agent-sockets + - name: spire-agent + image: ghcr.io/spiffe/spire-agent:1.15.3@sha256:41b0dcd8b258a69db9e2768292a060766fb76fd866e4bc925849981ea1b825ff + args: [-config, /run/spire/config/agent.conf] + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: [ALL] + volumeMounts: + - name: spire-config + mountPath: /run/spire/config + readOnly: true + - name: spire-bundle + mountPath: /run/spire/bundle + readOnly: true + - name: spire-token + mountPath: /run/spire/tokens + readOnly: true + - name: spire-data + mountPath: /run/spire/data + - name: spire-socket + mountPath: /run/spire/agent-sockets + - name: docker + image: docker.io/library/docker:29.1.5-dind + command: [/bin/sh, -c] + args: + - test -e /dev/kmsg || mknod /dev/kmsg c 1 11; exec dockerd --host=unix:///run/docker/docker.sock --group=2000 --storage-driver=overlay2 + securityContext: + privileged: true + volumeMounts: + - name: docker-socket + mountPath: /run/docker + - name: docker-data + mountPath: /var/lib/docker + volumes: + - name: opensandbox-bin + emptyDir: {} + - name: runner-data + emptyDir: {} + - name: docker-data + emptyDir: {} + - name: docker-socket + emptyDir: + medium: Memory + - name: spire-data + emptyDir: + medium: Memory + - name: spire-socket + emptyDir: + medium: Memory + - name: spire-config + configMap: + name: gitea-ci-spire-agent + - name: spire-bundle + configMap: + name: spire-bundle-pem + - name: spire-token + projected: + sources: + - serviceAccountToken: + audience: spire-server + expirationSeconds: 3600 + path: token diff --git a/platform/sandbox-ci-runners/rbac.yaml b/platform/sandbox-ci-runners/rbac.yaml new file mode 100644 index 0000000..1c03ffa --- /dev/null +++ b/platform/sandbox-ci-runners/rbac.yaml @@ -0,0 +1,87 @@ +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: gitea-ci + namespace: opensandbox +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: opensandbox-identity + namespace: opensandbox +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: opensandbox-identity + namespace: opensandbox +rules: + - apiGroups: [""] + resources: [pods] + verbs: [get, list, watch] + - apiGroups: [sandbox.opensandbox.io] + resources: [batchsandboxes] + verbs: [get, list, watch] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: opensandbox-identity + namespace: opensandbox +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: opensandbox-identity +subjects: + - kind: ServiceAccount + name: opensandbox-identity + namespace: opensandbox +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: opensandbox-identity +rules: + - apiGroups: [spire.spiffe.io] + resources: [clusterstaticentries] + verbs: [create, delete, get, list, watch] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: opensandbox-identity +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: opensandbox-identity +subjects: + - kind: ServiceAccount + name: opensandbox-identity + namespace: opensandbox +--- +# The central SPIRE external controller uses this existing sandbox credential +# to publish a PEM bundle for guest-local Agents. It cannot read Secrets here. +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: spire-runner-bundle-publisher + namespace: opensandbox +rules: + - apiGroups: [""] + resources: [configmaps] + verbs: [create, delete, get, list, patch, update, watch] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: spire-runner-bundle-publisher + namespace: opensandbox +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: spire-runner-bundle-publisher +subjects: + - kind: ServiceAccount + name: spire-controller-manager + namespace: spire-system diff --git a/platform/sandbox-ci-runners/service.yaml b/platform/sandbox-ci-runners/service.yaml new file mode 100644 index 0000000..2609c39 --- /dev/null +++ b/platform/sandbox-ci-runners/service.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: Service +metadata: + name: opensandbox-server-internal + namespace: opensandbox-system +spec: + type: NodePort + selector: + app.kubernetes.io/instance: opensandbox + app.kubernetes.io/name: opensandbox-server + ports: + - name: http + port: 80 + targetPort: http + nodePort: 30080 diff --git a/platform/sandbox-opensandbox/namespace.yaml b/platform/sandbox-opensandbox/namespace.yaml index b41233e..47a5c42 100644 --- a/platform/sandbox-opensandbox/namespace.yaml +++ b/platform/sandbox-opensandbox/namespace.yaml @@ -8,3 +8,7 @@ apiVersion: v1 kind: Namespace metadata: name: opensandbox + labels: + pod-security.kubernetes.io/enforce: privileged + pod-security.kubernetes.io/audit: restricted + pod-security.kubernetes.io/warn: restricted diff --git a/platform/spire/values.yaml b/platform/spire/values.yaml index 2675a90..d628ae2 100644 --- a/platform/spire/values.yaml +++ b/platform/spire/values.yaml @@ -43,6 +43,10 @@ spire-server: externalSecret: name: spire-external-kubeconfigs key: sandbox-controller + sandbox-runner-bundle: + externalSecret: + name: spire-external-kubeconfigs + key: sandbox-controller nodeAttestor: externalK8sPSAT: enabled: true @@ -55,6 +59,7 @@ spire-server: kubeConfigName: sandbox serviceAccountAllowList: - spire-smoke:spire-smoke + - opensandbox:gitea-ci usePodUIDForAgentID: true externalControllerManagers: enabled: true @@ -71,6 +76,12 @@ spire-server: configMapName: spire-bundle configMapKey: bundle.spiffe format: spiffe + sandbox-runner-bundle: + kubeConfigName: sandbox-runner-bundle + namespace: opensandbox + configMapName: spire-bundle-pem + configMapKey: bundle.pem + format: pem persistence: # PostgreSQL stores registrations, but the disk KeyManager still needs durable # storage for the trust-domain signing keys.