From cc4405f7886ca7194924ff18cd490aff1c1fefde Mon Sep 17 00:00:00 2001 From: panxiao81 Date: Mon, 21 Sep 2026 11:05:03 +0000 Subject: [PATCH 1/5] =?UTF-8?q?ci:=20=E4=BF=9D=E7=95=99=20kind=20=E5=A4=B1?= =?UTF-8?q?=E8=B4=A5=E7=8E=B0=E5=9C=BA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .gitea/workflows/vm-kind-smoke.yml | 46 ++++++++++++++++++++++++++---- 1 file changed, 41 insertions(+), 5 deletions(-) diff --git a/.gitea/workflows/vm-kind-smoke.yml b/.gitea/workflows/vm-kind-smoke.yml index f2a5b20..1d683fe 100644 --- a/.gitea/workflows/vm-kind-smoke.yml +++ b/.gitea/workflows/vm-kind-smoke.yml @@ -1,5 +1,7 @@ +--- name: VM kind smoke +# yamllint disable-line rule:truthy on: workflow_dispatch: @@ -15,17 +17,51 @@ jobs: run: | set -euo pipefail version=v0.33.0 + base_url="https://kind.sigs.k8s.io/dl/${version}" curl --fail --location --silent --show-error \ - --output /tmp/kind "https://kind.sigs.k8s.io/dl/${version}/kind-linux-amd64" + --output /tmp/kind "${base_url}/kind-linux-amd64" curl --fail --location --silent --show-error \ - --output /tmp/kind.sha256sum "https://kind.sigs.k8s.io/dl/${version}/kind-linux-amd64.sha256sum" - printf '%s %s\n' "$(cut -d ' ' -f1 /tmp/kind.sha256sum)" /tmp/kind | sha256sum --check + --output /tmp/kind.sha256sum \ + "${base_url}/kind-linux-amd64.sha256sum" + checksum=$(cut -d ' ' -f1 /tmp/kind.sha256sum) + printf '%s %s\n' "$checksum" /tmp/kind | sha256sum --check chmod 0755 /tmp/kind - name: Create and delete kind cluster shell: bash run: | set -euo pipefail - trap '/tmp/kind delete cluster --name smoke' EXIT - /tmp/kind create cluster --name smoke --wait 180s + diagnose_and_cleanup() { + status=$? + node=smoke-control-plane + if (( status != 0 )) && docker inspect "$node" >/dev/null 2>&1; then + echo '::group::kind node inspect' + docker inspect "$node" + echo '::endgroup::' + echo '::group::kind node logs' + docker logs "$node" 2>&1 || true + echo '::endgroup::' + echo '::group::kind node guest state' + docker exec "$node" bash -c ' + set +e + echo "### pid 1" + ps -p 1 -o pid,ppid,user,stat,comm,args + cat /proc/1/status + echo "### cgroup" + cat /proc/1/cgroup + findmnt -R /sys/fs/cgroup + stat -fc "%T %a" /sys/fs/cgroup + echo "### systemd" + systemctl --no-pager --failed + systemctl --no-pager status \ + multi-user.target containerd.service kubelet.service + journalctl --no-pager -b -n 500 + ' 2>&1 || true + echo '::endgroup::' + fi + /tmp/kind delete cluster --name smoke || true + exit "$status" + } + trap diagnose_and_cleanup EXIT + /tmp/kind create cluster --name smoke --wait 180s --retain /tmp/kind get clusters | grep -Fx smoke -- 2.54.0 From 8fa8e463200e2ec1d4be03a32f64cf380936a83c Mon Sep 17 00:00:00 2001 From: panxiao81 Date: Mon, 21 Sep 2026 11:09:55 +0000 Subject: [PATCH 2/5] =?UTF-8?q?ci:=20=E8=BE=93=E5=87=BA=20DinD=20cgroup=20?= =?UTF-8?q?=E5=B1=82=E7=BA=A7?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .gitea/workflows/vm-kind-smoke.yml | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/.gitea/workflows/vm-kind-smoke.yml b/.gitea/workflows/vm-kind-smoke.yml index 1d683fe..a3d2b97 100644 --- a/.gitea/workflows/vm-kind-smoke.yml +++ b/.gitea/workflows/vm-kind-smoke.yml @@ -10,7 +10,21 @@ jobs: runs-on: [self-hosted, vm-dev] steps: - name: Verify Docker - run: docker info + shell: bash + run: | + set -euo pipefail + docker info + echo '### runner cgroup' + cat /proc/self/cgroup + cat /sys/fs/cgroup/cgroup.type + cat /sys/fs/cgroup/cgroup.controllers + cat /sys/fs/cgroup/cgroup.subtree_control + echo '### nested private cgroup namespace' + docker run --rm --privileged --cgroupns=private alpine:3.22 \ + sh -c 'cat /proc/self/cgroup; cat /sys/fs/cgroup/cgroup.type' + echo '### nested host cgroup namespace' + docker run --rm --privileged --cgroupns=host alpine:3.22 \ + sh -c 'cat /proc/self/cgroup; cat /sys/fs/cgroup/cgroup.type' - name: Install kind shell: bash -- 2.54.0 From b614ef4c2fc92f5d64f6f2a856291e26a489c393 Mon Sep 17 00:00:00 2001 From: panxiao81 Date: Mon, 21 Sep 2026 11:24:40 +0000 Subject: [PATCH 3/5] =?UTF-8?q?ci:=20=E6=94=B6=E9=9B=86=20kind=20CRI=20?= =?UTF-8?q?=E5=AE=B9=E5=99=A8=E6=97=A5=E5=BF=97?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .gitea/workflows/vm-kind-smoke.yml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/.gitea/workflows/vm-kind-smoke.yml b/.gitea/workflows/vm-kind-smoke.yml index a3d2b97..9d83654 100644 --- a/.gitea/workflows/vm-kind-smoke.yml +++ b/.gitea/workflows/vm-kind-smoke.yml @@ -69,6 +69,16 @@ jobs: systemctl --no-pager --failed systemctl --no-pager status \ multi-user.target containerd.service kubelet.service + echo "### CRI containers" + endpoint=unix:///run/containerd/containerd.sock + crictl --runtime-endpoint "$endpoint" ps --all + for id in $( + crictl --runtime-endpoint "$endpoint" ps --all --quiet + ); do + echo "### CRI container $id" + crictl --runtime-endpoint "$endpoint" inspect "$id" + crictl --runtime-endpoint "$endpoint" logs "$id" + done journalctl --no-pager -b -n 500 ' 2>&1 || true echo '::endgroup::' -- 2.54.0 From 51b940468edecbc655fa82770c14322fcc119be4 Mon Sep 17 00:00:00 2001 From: panxiao81 Date: Mon, 21 Sep 2026 11:41:16 +0000 Subject: [PATCH 4/5] =?UTF-8?q?ci:=20=E6=94=B6=E9=9B=86=20nested=20contain?= =?UTF-8?q?erd=20=E9=98=BB=E5=A1=9E=E7=8E=B0=E5=9C=BA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .gitea/workflows/vm-kind-smoke.yml | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/.gitea/workflows/vm-kind-smoke.yml b/.gitea/workflows/vm-kind-smoke.yml index 9d83654..d6fa4fd 100644 --- a/.gitea/workflows/vm-kind-smoke.yml +++ b/.gitea/workflows/vm-kind-smoke.yml @@ -79,6 +79,17 @@ jobs: crictl --runtime-endpoint "$endpoint" inspect "$id" crictl --runtime-endpoint "$endpoint" logs "$id" done + echo "### containerd metadata" + timeout 10 ctr --namespace k8s.io containers list + timeout 10 ctr --namespace k8s.io snapshots list + echo "### runtime process stacks" + ps -e -o pid,ppid,stat,wchan:32,comm,args + for pid in $(pidof containerd containerd-shim-runc-v2); do + echo "### kernel stack $pid" + cat "/proc/$pid/stack" + done + kill -USR1 "$(pidof containerd)" + sleep 2 journalctl --no-pager -b -n 500 ' 2>&1 || true echo '::endgroup::' -- 2.54.0 From fef7e5a21403e4a2ea6793fb53482512827d0a7f Mon Sep 17 00:00:00 2001 From: panxiao81 Date: Mon, 21 Sep 2026 12:00:55 +0000 Subject: [PATCH 5/5] =?UTF-8?q?ci:=20=E7=94=B1=20VM=20=E4=BB=BB=E5=8A=A1?= =?UTF-8?q?=E5=90=AF=E5=8A=A8=E6=9C=AC=E5=9C=B0=20Docker=20daemon?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .gitea/workflows/vm-kind-smoke.yml | 39 ++++++++++++++++++++++++++++++ docs/opensandbox-runner.md | 8 +++--- 2 files changed, 44 insertions(+), 3 deletions(-) diff --git a/.gitea/workflows/vm-kind-smoke.yml b/.gitea/workflows/vm-kind-smoke.yml index d6fa4fd..387a240 100644 --- a/.gitea/workflows/vm-kind-smoke.yml +++ b/.gitea/workflows/vm-kind-smoke.yml @@ -9,6 +9,45 @@ jobs: kind: runs-on: [self-hosted, vm-dev] steps: + - name: Start job-local Docker + shell: bash + run: | + set -euo pipefail + sudo install -d /var/lib/docker + sudo truncate -s 20G /tmp/docker-data.img + sudo mkfs.ext4 -F /tmp/docker-data.img + sudo mount -o loop /tmp/docker-data.img /var/lib/docker + + # A nested systemd needs a domain cgroup namespace. This is the + # cgroup v2 nesting initialization performed by the official DinD + # entrypoint, kept here because the shared runner image deliberately + # does not carry a second DinD-specific entrypoint. + if [[ -f /sys/fs/cgroup/cgroup.controllers ]]; then + sudo sh -c ' + mkdir -p /sys/fs/cgroup/init + while read -r pid; do + printf "%s\n" "$pid" \ + >/sys/fs/cgroup/init/cgroup.procs 2>/dev/null || true + done /sys/fs/cgroup/cgroup.subtree_control + ' + fi + + sudo nohup dockerd \ + --host=unix:///var/run/docker.sock \ + --storage-driver=overlay2 \ + >/tmp/dockerd.log 2>&1 & + for _ in {1..60}; do + if docker info >/dev/null 2>&1; then + exit 0 + fi + sleep 1 + done + cat /tmp/dockerd.log + exit 1 + - name: Verify Docker shell: bash run: | diff --git a/docs/opensandbox-runner.md b/docs/opensandbox-runner.md index 7919a19..6bfba8d 100644 --- a/docs/opensandbox-runner.md +++ b/docs/opensandbox-runner.md @@ -40,9 +40,11 @@ UID attestation 的临时 Agent 失去父级。 `ci-vm` 使用 `kata-clh-runtime-rs`；`ci-pod` 使用默认 runc。两者都要求： -- runner 镜像包含 Gitea Runner、Node.js action userspace、SPIRE CLI 和 identity gate； -- runner UID 2000，SPIRE Agent 与 privileged dockerd 使用不同 UID； -- Docker socket 通过 group 2000 共享，Docker 数据仅存在于 sandbox emptyDir； +- runner 镜像包含 Gitea Runner、Node.js action userspace、SPIRE CLI、Docker 工具和 + identity gate；Pod 与 VM backend 使用同一个镜像； +- runner UID 2000；SPIRE Agent 独立运行，需要 Docker 的 workflow 通过 sudo 在 + privileged executor 内启动 job-local daemon； +- Kata VM 中 Docker 数据使用 guest 内的 loop-backed ext4，并随 sandbox 一起删除； - `self-hosted` 必须是所有 runner labels 的前缀； - ephemeral/once runner 完成一项任务后退出。 -- 2.54.0