diff --git a/.gitea/workflows/vm-kind-smoke.yml b/.gitea/workflows/vm-kind-smoke.yml index f2a5b20..387a240 100644 --- a/.gitea/workflows/vm-kind-smoke.yml +++ b/.gitea/workflows/vm-kind-smoke.yml @@ -1,5 +1,7 @@ +--- name: VM kind smoke +# yamllint disable-line rule:truthy on: workflow_dispatch: @@ -7,25 +9,133 @@ jobs: kind: runs-on: [self-hosted, vm-dev] steps: + - name: Start job-local Docker + shell: bash + run: | + set -euo pipefail + sudo install -d /var/lib/docker + sudo truncate -s 20G /tmp/docker-data.img + sudo mkfs.ext4 -F /tmp/docker-data.img + sudo mount -o loop /tmp/docker-data.img /var/lib/docker + + # A nested systemd needs a domain cgroup namespace. This is the + # cgroup v2 nesting initialization performed by the official DinD + # entrypoint, kept here because the shared runner image deliberately + # does not carry a second DinD-specific entrypoint. + if [[ -f /sys/fs/cgroup/cgroup.controllers ]]; then + sudo sh -c ' + mkdir -p /sys/fs/cgroup/init + while read -r pid; do + printf "%s\n" "$pid" \ + >/sys/fs/cgroup/init/cgroup.procs 2>/dev/null || true + done /sys/fs/cgroup/cgroup.subtree_control + ' + fi + + sudo nohup dockerd \ + --host=unix:///var/run/docker.sock \ + --storage-driver=overlay2 \ + >/tmp/dockerd.log 2>&1 & + for _ in {1..60}; do + if docker info >/dev/null 2>&1; then + exit 0 + fi + sleep 1 + done + cat /tmp/dockerd.log + exit 1 + - name: Verify Docker - run: docker info + shell: bash + run: | + set -euo pipefail + docker info + echo '### runner cgroup' + cat /proc/self/cgroup + cat /sys/fs/cgroup/cgroup.type + cat /sys/fs/cgroup/cgroup.controllers + cat /sys/fs/cgroup/cgroup.subtree_control + echo '### nested private cgroup namespace' + docker run --rm --privileged --cgroupns=private alpine:3.22 \ + sh -c 'cat /proc/self/cgroup; cat /sys/fs/cgroup/cgroup.type' + echo '### nested host cgroup namespace' + docker run --rm --privileged --cgroupns=host alpine:3.22 \ + sh -c 'cat /proc/self/cgroup; cat /sys/fs/cgroup/cgroup.type' - name: Install kind shell: bash run: | set -euo pipefail version=v0.33.0 + base_url="https://kind.sigs.k8s.io/dl/${version}" curl --fail --location --silent --show-error \ - --output /tmp/kind "https://kind.sigs.k8s.io/dl/${version}/kind-linux-amd64" + --output /tmp/kind "${base_url}/kind-linux-amd64" curl --fail --location --silent --show-error \ - --output /tmp/kind.sha256sum "https://kind.sigs.k8s.io/dl/${version}/kind-linux-amd64.sha256sum" - printf '%s %s\n' "$(cut -d ' ' -f1 /tmp/kind.sha256sum)" /tmp/kind | sha256sum --check + --output /tmp/kind.sha256sum \ + "${base_url}/kind-linux-amd64.sha256sum" + checksum=$(cut -d ' ' -f1 /tmp/kind.sha256sum) + printf '%s %s\n' "$checksum" /tmp/kind | sha256sum --check chmod 0755 /tmp/kind - name: Create and delete kind cluster shell: bash run: | set -euo pipefail - trap '/tmp/kind delete cluster --name smoke' EXIT - /tmp/kind create cluster --name smoke --wait 180s + diagnose_and_cleanup() { + status=$? + node=smoke-control-plane + if (( status != 0 )) && docker inspect "$node" >/dev/null 2>&1; then + echo '::group::kind node inspect' + docker inspect "$node" + echo '::endgroup::' + echo '::group::kind node logs' + docker logs "$node" 2>&1 || true + echo '::endgroup::' + echo '::group::kind node guest state' + docker exec "$node" bash -c ' + set +e + echo "### pid 1" + ps -p 1 -o pid,ppid,user,stat,comm,args + cat /proc/1/status + echo "### cgroup" + cat /proc/1/cgroup + findmnt -R /sys/fs/cgroup + stat -fc "%T %a" /sys/fs/cgroup + echo "### systemd" + systemctl --no-pager --failed + systemctl --no-pager status \ + multi-user.target containerd.service kubelet.service + echo "### CRI containers" + endpoint=unix:///run/containerd/containerd.sock + crictl --runtime-endpoint "$endpoint" ps --all + for id in $( + crictl --runtime-endpoint "$endpoint" ps --all --quiet + ); do + echo "### CRI container $id" + crictl --runtime-endpoint "$endpoint" inspect "$id" + crictl --runtime-endpoint "$endpoint" logs "$id" + done + echo "### containerd metadata" + timeout 10 ctr --namespace k8s.io containers list + timeout 10 ctr --namespace k8s.io snapshots list + echo "### runtime process stacks" + ps -e -o pid,ppid,stat,wchan:32,comm,args + for pid in $(pidof containerd containerd-shim-runc-v2); do + echo "### kernel stack $pid" + cat "/proc/$pid/stack" + done + kill -USR1 "$(pidof containerd)" + sleep 2 + journalctl --no-pager -b -n 500 + ' 2>&1 || true + echo '::endgroup::' + fi + /tmp/kind delete cluster --name smoke || true + exit "$status" + } + trap diagnose_and_cleanup EXIT + /tmp/kind create cluster --name smoke --wait 180s --retain /tmp/kind get clusters | grep -Fx smoke diff --git a/docs/opensandbox-runner.md b/docs/opensandbox-runner.md index 7919a19..6bfba8d 100644 --- a/docs/opensandbox-runner.md +++ b/docs/opensandbox-runner.md @@ -40,9 +40,11 @@ UID attestation 的临时 Agent 失去父级。 `ci-vm` 使用 `kata-clh-runtime-rs`；`ci-pod` 使用默认 runc。两者都要求： -- runner 镜像包含 Gitea Runner、Node.js action userspace、SPIRE CLI 和 identity gate； -- runner UID 2000，SPIRE Agent 与 privileged dockerd 使用不同 UID； -- Docker socket 通过 group 2000 共享，Docker 数据仅存在于 sandbox emptyDir； +- runner 镜像包含 Gitea Runner、Node.js action userspace、SPIRE CLI、Docker 工具和 + identity gate；Pod 与 VM backend 使用同一个镜像； +- runner UID 2000；SPIRE Agent 独立运行，需要 Docker 的 workflow 通过 sudo 在 + privileged executor 内启动 job-local daemon； +- Kata VM 中 Docker 数据使用 guest 内的 loop-backed ext4，并随 sandbox 一起删除； - `self-hosted` 必须是所有 runner labels 的前缀； - ephemeral/once runner 完成一项任务后退出。