ci: 为 Kata nested kubelet 补充 kmsg 设备

.gitea/workflows/vm-kind-smoke.yml

@@ -1,5 +1,7 @@
+---
 name: VM kind smoke
 
+# yamllint disable-line rule:truthy
 on:
   workflow_dispatch:
 
@@ -7,25 +9,146 @@ jobs:
   kind:
     runs-on: [self-hosted, vm-dev]
     steps:
+      - name: Start job-local Docker
+        shell: bash
+        run: |
+          set -euo pipefail
+          if [[ ! -e /dev/kmsg ]]; then
+            sudo mknod /dev/kmsg c 1 11
+          fi
+          sudo install -d /var/lib/docker
+          sudo truncate -s 20G /tmp/docker-data.img
+          sudo mkfs.ext4 -F /tmp/docker-data.img
+          if [[ ! -e /dev/loop-control ]]; then
+            sudo mknod /dev/loop-control c 10 237
+          fi
+          for minor in {0..7}; do
+            if [[ ! -e "/dev/loop${minor}" ]]; then
+              sudo mknod "/dev/loop${minor}" b 7 "$minor"
+            fi
+          done
+          loop_device=$(sudo losetup --find --show /tmp/docker-data.img)
+          sudo mount "$loop_device" /var/lib/docker
+          findmnt /var/lib/docker
+
+          # A nested systemd needs a domain cgroup namespace. This is the
+          # cgroup v2 nesting initialization performed by the official DinD
+          # entrypoint, kept here because the shared runner image deliberately
+          # does not carry a second DinD-specific entrypoint.
+          if [[ -f /sys/fs/cgroup/cgroup.controllers ]]; then
+            sudo sh -c '
+              mkdir -p /sys/fs/cgroup/init
+              while read -r pid; do
+                printf "%s\n" "$pid" \
+                  >/sys/fs/cgroup/init/cgroup.procs 2>/dev/null || true
+              done </sys/fs/cgroup/cgroup.procs
+              sed -e "s/ / +/g" -e "s/^/+/" \
+                /sys/fs/cgroup/cgroup.controllers \
+                >/sys/fs/cgroup/cgroup.subtree_control
+            '
+          fi
+
+          sudo nohup dockerd \
+            --host=unix:///var/run/docker.sock \
+            --storage-driver=overlay2 \
+            >/tmp/dockerd.log 2>&1 &
+          for _ in {1..60}; do
+            if docker info >/dev/null 2>&1; then
+              exit 0
+            fi
+            sleep 1
+          done
+          cat /tmp/dockerd.log
+          exit 1
+
       - name: Verify Docker
-        run: docker info
+        shell: bash
+        run: |
+          set -euo pipefail
+          docker info
+          echo '### runner cgroup'
+          cat /proc/self/cgroup
+          cat /sys/fs/cgroup/cgroup.type
+          cat /sys/fs/cgroup/cgroup.controllers
+          cat /sys/fs/cgroup/cgroup.subtree_control
+          echo '### nested private cgroup namespace'
+          docker run --rm --privileged --cgroupns=private alpine:3.22 \
+            sh -c 'cat /proc/self/cgroup; cat /sys/fs/cgroup/cgroup.type'
+          echo '### nested host cgroup namespace'
+          docker run --rm --privileged --cgroupns=host alpine:3.22 \
+            sh -c 'cat /proc/self/cgroup; cat /sys/fs/cgroup/cgroup.type'
 
       - name: Install kind
         shell: bash
         run: |
           set -euo pipefail
           version=v0.33.0
+          base_url="https://kind.sigs.k8s.io/dl/${version}"
           curl --fail --location --silent --show-error \
-            --output /tmp/kind "https://kind.sigs.k8s.io/dl/${version}/kind-linux-amd64"
+            --output /tmp/kind "${base_url}/kind-linux-amd64"
           curl --fail --location --silent --show-error \
-            --output /tmp/kind.sha256sum "https://kind.sigs.k8s.io/dl/${version}/kind-linux-amd64.sha256sum"
-          printf '%s  %s\n' "$(cut -d ' ' -f1 /tmp/kind.sha256sum)" /tmp/kind | sha256sum --check
+            --output /tmp/kind.sha256sum \
+            "${base_url}/kind-linux-amd64.sha256sum"
+          checksum=$(cut -d ' ' -f1 /tmp/kind.sha256sum)
+          printf '%s  %s\n' "$checksum" /tmp/kind | sha256sum --check
           chmod 0755 /tmp/kind
 
       - name: Create and delete kind cluster
         shell: bash
         run: |
           set -euo pipefail
-          trap '/tmp/kind delete cluster --name smoke' EXIT
-          /tmp/kind create cluster --name smoke --wait 180s
+          diagnose_and_cleanup() {
+            status=$?
+            node=smoke-control-plane
+            if (( status != 0 )) && docker inspect "$node" >/dev/null 2>&1; then
+              echo '::group::kind node inspect'
+              docker inspect "$node"
+              echo '::endgroup::'
+              echo '::group::kind node logs'
+              docker logs "$node" 2>&1 || true
+              echo '::endgroup::'
+              echo '::group::kind node guest state'
+              docker exec "$node" bash -c '
+                set +e
+                echo "### pid 1"
+                ps -p 1 -o pid,ppid,user,stat,comm,args
+                cat /proc/1/status
+                echo "### cgroup"
+                cat /proc/1/cgroup
+                findmnt -R /sys/fs/cgroup
+                stat -fc "%T %a" /sys/fs/cgroup
+                echo "### systemd"
+                systemctl --no-pager --failed
+                systemctl --no-pager status \
+                  multi-user.target containerd.service kubelet.service
+                echo "### CRI containers"
+                endpoint=unix:///run/containerd/containerd.sock
+                crictl --runtime-endpoint "$endpoint" ps --all
+                for id in $(
+                  crictl --runtime-endpoint "$endpoint" ps --all --quiet
+                ); do
+                  echo "### CRI container $id"
+                  crictl --runtime-endpoint "$endpoint" inspect "$id"
+                  crictl --runtime-endpoint "$endpoint" logs "$id"
+                done
+                echo "### containerd metadata"
+                timeout 10 ctr --namespace k8s.io containers list
+                timeout 10 ctr --namespace k8s.io snapshots list
+                echo "### runtime process stacks"
+                ps -e -o pid,ppid,stat,wchan:32,comm,args
+                for pid in $(pidof containerd containerd-shim-runc-v2); do
+                  echo "### kernel stack $pid"
+                  cat "/proc/$pid/stack"
+                done
+                kill -USR1 "$(pidof containerd)"
+                sleep 2
+                journalctl --no-pager -b -n 500
+              ' 2>&1 || true
+              echo '::endgroup::'
+            fi
+            /tmp/kind delete cluster --name smoke || true
+            exit "$status"
+          }
+          trap diagnose_and_cleanup EXIT
+          /tmp/kind create cluster --name smoke --wait 180s --retain
           /tmp/kind get clusters | grep -Fx smoke

docs/opensandbox-runner.md

@@ -40,9 +40,11 @@ UID attestation 的临时 Agent 失去父级。
 
 `ci-vm` 使用 `kata-clh-runtime-rs`；`ci-pod` 使用默认 runc。两者都要求：
 
-- runner 镜像包含 Gitea Runner、Node.js action userspace、SPIRE CLI 和 identity gate；
-- runner UID 2000，SPIRE Agent 与 privileged dockerd 使用不同 UID；
-- Docker socket 通过 group 2000 共享，Docker 数据仅存在于 sandbox emptyDir；
+- runner 镜像包含 Gitea Runner、Node.js action userspace、SPIRE CLI、Docker 工具和
+  identity gate；Pod 与 VM backend 使用同一个镜像；
+- runner UID 2000；SPIRE Agent 独立运行，需要 Docker 的 workflow 通过 sudo 在
+  privileged executor 内启动 job-local daemon；
+- Kata VM 中 Docker 数据使用 guest 内的 loop-backed ext4，并随 sandbox 一起删除；
 - `self-hosted` 必须是所有 runner labels 的前缀；
 - ephemeral/once runner 完成一项任务后退出。