fix(runner): redirect dockerd logs as root

.gitea/workflows/pod-smoke.yml

@@ -1,5 +1,7 @@
+---
 name: dynamic Pod smoke test
 
+# yamllint disable-line rule:truthy
 on:
   workflow_dispatch:
 
@@ -17,3 +19,24 @@ jobs:
             -socketPath /run/spire/agent-sockets/spire-agent.sock \
             >/dev/null
           test "$(id -u)" = 2000
+
+      - name: Build and run image
+        shell: bash
+        run: |
+          set -euo pipefail
+          findmnt /var/lib/docker
+          context=$(mktemp -d)
+          cleanup() {
+            docker image rm --force pod-docker-smoke:test \
+              >/dev/null 2>&1 || true
+            rm -rf -- "$context"
+          }
+          trap cleanup EXIT
+          printf '%s\n' \
+            'FROM alpine:3.22' \
+            'RUN printf pod-docker-ok >/result' \
+            >"$context/Dockerfile"
+          docker build --tag pod-docker-smoke:test "$context"
+          output=$(docker run --rm pod-docker-smoke:test cat /result)
+          test "$output" = pod-docker-ok
+          test "$(docker info --format '{{.Driver}}')" = overlay2

.gitea/workflows/publish-images.yml

@@ -34,17 +34,27 @@ jobs:
     steps:
       - uses: actions/checkout@v4
 
+      - uses: actions/setup-go@v5
+        with:
+          go-version-file: go.mod
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: '3.12'
+
       - name: Test source
         shell: bash
         run: |
           set -euo pipefail
           go test ./...
           go vet ./...
-          python3 -m pip install --break-system-packages -e '.[test]'
+          python3 -m venv /tmp/test-venv
+          source /tmp/test-venv/bin/activate
+          python3 -m pip install -e '.[test]'
           pytest -q
           python3 -m compileall -q src tests
-          apt-get update
+          sudo apt-get update
-          apt-get install --yes --no-install-recommends shellcheck
+          sudo apt-get install --yes --no-install-recommends shellcheck
           shellcheck scripts/*
 
       - name: Build and publish
@@ -53,6 +63,10 @@ jobs:
           set -euo pipefail
           set +x
 
+          # Bootstrap the image that first introduces automatic Docker setup.
+          # Once deployed, the job-started hook makes this an idempotent no-op.
+          sudo scripts/setup-job-docker
+
           : "${GITHUB_SHA:?GITHUB_SHA is required}"
           image_tag="sha-${GITHUB_SHA}"
           docker_config=$(mktemp -d)

.gitea/workflows/vm-kind-smoke.yml

@@ -1,31 +1,131 @@
+---
 name: VM kind smoke
 
+# yamllint disable-line rule:truthy
 on:
   workflow_dispatch:
 
 jobs:
   kind:
-    runs-on: [self-hosted, vm-dev]
+    runs-on: [self-hosted, vm]
     steps:
+      - name: Prepare nested kubelet device
+        shell: bash
+        run: |
+          set -euo pipefail
+          if [[ ! -e /dev/kmsg ]]; then
+            sudo mknod /dev/kmsg c 1 11
+          fi
+
       - name: Verify Docker
-        run: docker info
+        shell: bash
+        run: |
+          set -euo pipefail
+          findmnt /var/lib/docker
+          docker info
+          echo '### runner cgroup'
+          cat /proc/self/cgroup
+          cat /sys/fs/cgroup/cgroup.type
+          cat /sys/fs/cgroup/cgroup.controllers
+          cat /sys/fs/cgroup/cgroup.subtree_control
+          echo '### nested private cgroup namespace'
+          docker run --rm --privileged --cgroupns=private alpine:3.22 \
+            sh -c 'cat /proc/self/cgroup; cat /sys/fs/cgroup/cgroup.type'
+          echo '### nested host cgroup namespace'
+          docker run --rm --privileged --cgroupns=host alpine:3.22 \
+            sh -c 'cat /proc/self/cgroup; cat /sys/fs/cgroup/cgroup.type'
 
       - name: Install kind
         shell: bash
         run: |
           set -euo pipefail
           version=v0.33.0
+          base_url="https://kind.sigs.k8s.io/dl/${version}"
           curl --fail --location --silent --show-error \
-            --output /tmp/kind "https://kind.sigs.k8s.io/dl/${version}/kind-linux-amd64"
+            --output /tmp/kind "${base_url}/kind-linux-amd64"
           curl --fail --location --silent --show-error \
-            --output /tmp/kind.sha256sum "https://kind.sigs.k8s.io/dl/${version}/kind-linux-amd64.sha256sum"
+            --output /tmp/kind.sha256sum \
-          printf '%s  %s\n' "$(cut -d ' ' -f1 /tmp/kind.sha256sum)" /tmp/kind | sha256sum --check
+            "${base_url}/kind-linux-amd64.sha256sum"
+          checksum=$(cut -d ' ' -f1 /tmp/kind.sha256sum)
+          printf '%s  %s\n' "$checksum" /tmp/kind | sha256sum --check
           chmod 0755 /tmp/kind
 
       - name: Create and delete kind cluster
         shell: bash
         run: |
           set -euo pipefail
-          trap '/tmp/kind delete cluster --name smoke' EXIT
+          diagnose_and_cleanup() {
-          /tmp/kind create cluster --name smoke --wait 180s
+            status=$?
+            node=smoke-control-plane
+            if (( status != 0 )) && docker inspect "$node" >/dev/null 2>&1; then
+              echo '::group::kind node inspect'
+              docker inspect "$node"
+              echo '::endgroup::'
+              echo '::group::kind node logs'
+              docker logs "$node" 2>&1 || true
+              echo '::endgroup::'
+              echo '::group::kind node guest state'
+              docker exec "$node" bash -c '
+                set +e
+                echo "### pid 1"
+                ps -p 1 -o pid,ppid,user,stat,comm,args
+                cat /proc/1/status
+                echo "### cgroup"
+                cat /proc/1/cgroup
+                findmnt -R /sys/fs/cgroup
+                stat -fc "%T %a" /sys/fs/cgroup
+                echo "### systemd"
+                systemctl --no-pager --failed
+                systemctl --no-pager status \
+                  multi-user.target containerd.service kubelet.service
+                echo "### CRI containers"
+                endpoint=unix:///run/containerd/containerd.sock
+                crictl --runtime-endpoint "$endpoint" ps --all
+                for id in $(
+                  crictl --runtime-endpoint "$endpoint" ps --all --quiet
+                ); do
+                  echo "### CRI container $id"
+                  crictl --runtime-endpoint "$endpoint" inspect "$id"
+                  crictl --runtime-endpoint "$endpoint" logs "$id"
+                done
+                echo "### containerd metadata"
+                timeout 10 ctr --namespace k8s.io containers list
+                timeout 10 ctr --namespace k8s.io snapshots list
+                echo "### runtime process stacks"
+                ps -e -o pid,ppid,stat,wchan:32,comm,args
+                for pid in $(pidof containerd containerd-shim-runc-v2); do
+                  echo "### kernel stack $pid"
+                  cat "/proc/$pid/stack"
+                done
+                kill -USR1 "$(pidof containerd)"
+                sleep 2
+                journalctl --no-pager -b -n 500
+              ' 2>&1 || true
+              echo '::endgroup::'
+            fi
+            /tmp/kind delete cluster --name smoke || true
+            exit "$status"
+          }
+          trap diagnose_and_cleanup EXIT
+          # Nested Kata + Docker + kind cold starts can take longer than
+          # kubeadm's one-minute API-call default even after the static pods
+          # have been accepted. Give the API server enough time to become
+          # responsive before kubeadm creates its initial RBAC objects.
+          cat >/tmp/kind-config.yml <<'EOF'
+          kind: Cluster
+          apiVersion: kind.x-k8s.io/v1alpha4
+          nodes:
+            - role: control-plane
+              kubeadmConfigPatches:
+                - |
+                  apiVersion: kubeadm.k8s.io/v1beta4
+                  kind: InitConfiguration
+                  timeouts:
+                    kubernetesAPICall: 5m0s
+          EOF
+          /tmp/kind create cluster \
+            --name smoke \
+            --config /tmp/kind-config.yml \
+            --wait 300s \
+            --retain
           /tmp/kind get clusters | grep -Fx smoke

.gitea/workflows/vm-runtime-smoke.yml

@@ -7,14 +7,15 @@ on:
 
 jobs:
   runtime:
-    runs-on: [self-hosted, vm-dev]
+    runs-on: [self-hosted, vm]
     steps:
       - name: Verify workload identity socket
         shell: bash
         run: |
           set -euo pipefail
           : "${SPIFFE_ENDPOINT_SOCKET:?SPIFFE_ENDPOINT_SOCKET is required}"
-          test -S "$SPIFFE_ENDPOINT_SOCKET"
+          socket_path=${SPIFFE_ENDPOINT_SOCKET#unix://}
+          test -S "$socket_path"
 
       - name: Verify Docker daemon
         shell: bash

README.md

@@ -36,8 +36,9 @@ runs-on: [self-hosted, vm]
   entry；不持有 OpenSandbox API key、Gitea token 或 Bao 凭据。身份与 Pool 契约见
   [`docs/opensandbox-runner.md`](docs/opensandbox-runner.md)。
 - Pod executor：在 Kubernetes 中创建一次性 privileged Pod；Pod 内的 workflow 使用
-  host executor，Docker、BuildKit 和 kind 等工具由 pipeline 按需 setup。Runner 固定在
+  host executor。Runner 固定在支持原生 job hooks 的 3.x 版本，在 workflow 第一步前
-  支持原生 job hooks 的 3.x 版本，在 workflow 第一步前等待实际任务对应的 SVID。
+  等待实际任务对应的 SVID，并启动 job-local Docker daemon；workflow 可直接使用与
+  GitHub-hosted runner 相同的 Docker/BuildKit action。
 - `jwt-broker`：早期共享 Kubernetes runner 的过渡实验；目标架构不部署它，每个
   动态 Pod 或 VM 直接取得自己的 SPIFFE 身份。
 

container/runner.Dockerfile

@@ -29,6 +29,7 @@ COPY --from=spire /opt/spire/bin/spire-agent /opt/spire/bin/spire-agent
 COPY config/runner.yaml /etc/gitea-runner/config.yaml
 COPY --chmod=0755 scripts/gitea-job-started /usr/local/libexec/gitea-job-started
 COPY --chmod=0755 scripts/gitea-opensandbox-runner /usr/local/libexec/gitea-opensandbox-runner
+COPY --chmod=0755 scripts/setup-job-docker /usr/local/libexec/setup-job-docker
 
 VOLUME ["/data"]
 ENV HOME=/home/runner

docs/opensandbox-runner.md

@@ -40,9 +40,11 @@ UID attestation 的临时 Agent 失去父级。
 
 `ci-vm` 使用 `kata-clh-runtime-rs`；`ci-pod` 使用默认 runc。两者都要求：
 
-- runner 镜像包含 Gitea Runner、Node.js action userspace、SPIRE CLI 和 identity gate；
+- runner 镜像包含 Gitea Runner、Node.js action userspace、SPIRE CLI、Docker 工具和
-- runner UID 2000，SPIRE Agent 与 privileged dockerd 使用不同 UID；
+  identity gate；Pod 与 VM backend 使用同一个镜像；
-- Docker socket 通过 group 2000 共享，Docker 数据仅存在于 sandbox emptyDir；
+- runner UID 2000；SPIRE Agent 独立运行；job-started hook 在第一步 workflow 之前
+  启动 job-local Docker daemon，业务 workflow 不负责 runner 基础设施初始化；
+- Kata VM 中 Docker 数据使用 guest 内的 loop-backed ext4，并随 sandbox 一起删除；
 - `self-hosted` 必须是所有 runner labels 的前缀；
 - ephemeral/once runner 完成一项任务后退出。
 

internal/podbackend/client.go

@@ -8,6 +8,7 @@ import (
 
 	corev1 "k8s.io/api/core/v1"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	"k8s.io/apimachinery/pkg/api/resource"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 	"k8s.io/apimachinery/pkg/runtime/schema"
@@ -76,16 +77,25 @@ func (c *Client) CreatePod(ctx context.Context, manifest PodManifest) (Pod, erro
 			Containers: []corev1.Container{{
 				Name: "executor", Image: manifest.Image, Args: manifest.Args, Env: environment,
 				SecurityContext: &corev1.SecurityContext{Privileged: boolPointer(true)},
-				VolumeMounts: []corev1.VolumeMount{{
+				VolumeMounts: []corev1.VolumeMount{
-					Name: "spire-agent-socket", MountPath: "/run/spire/agent-sockets", ReadOnly: true,
+					{Name: "spire-agent-socket", MountPath: "/run/spire/agent-sockets", ReadOnly: true},
-				}},
+					{Name: "docker-data", MountPath: "/var/lib/docker"},
-			}},
+				},
-			Volumes: []corev1.Volume{{
-				Name: "spire-agent-socket",
-				VolumeSource: corev1.VolumeSource{CSI: &corev1.CSIVolumeSource{
-					Driver: "csi.spiffe.io", ReadOnly: boolPointer(true),
-				}},
 			}},
+			Volumes: []corev1.Volume{
+				{
+					Name: "spire-agent-socket",
+					VolumeSource: corev1.VolumeSource{CSI: &corev1.CSIVolumeSource{
+						Driver: "csi.spiffe.io", ReadOnly: boolPointer(true),
+					}},
+				},
+				{
+					Name: "docker-data",
+					VolumeSource: corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{
+						SizeLimit: resourceQuantity("20Gi"),
+					}},
+				},
+			},
 		},
 	}
 	created, err := c.Kubernetes.CoreV1().Pods(manifest.Namespace).Create(ctx, document, metav1.CreateOptions{})
@@ -169,6 +179,11 @@ func podFromKubernetes(pod corev1.Pod) Pod {
 
 func boolPointer(value bool) *bool { return &value }
 
+func resourceQuantity(value string) *resource.Quantity {
+	quantity := resource.MustParse(value)
+	return &quantity
+}
+
 func stringMap(values map[string]string) map[string]any {
 	result := make(map[string]any, len(values))
 	for key, value := range values {

internal/podbackend/client_test.go

@@ -37,6 +37,12 @@ func TestClientPodLifecycleUsesTypedClient(t *testing.T) {
 	if got := pod.Spec.Containers[0].Env; len(got) != 1 || got[0].Name != "CI_RUNNER_CAPABILITY" || got[0].Value != "capability" {
 		t.Fatalf("environment = %#v", got)
 	}
+	if got := pod.Spec.Containers[0].VolumeMounts; len(got) != 2 || got[1].Name != "docker-data" || got[1].MountPath != "/var/lib/docker" {
+		t.Fatalf("volume mounts = %#v", got)
+	}
+	if got := pod.Spec.Volumes; len(got) != 2 || got[1].EmptyDir == nil || got[1].EmptyDir.SizeLimit == nil || got[1].EmptyDir.SizeLimit.String() != "20Gi" {
+		t.Fatalf("volumes = %#v", got)
+	}
 	if err := client.LabelPod(context.Background(), "gitea-actions", created.Name, map[string]string{terminalLabel: "true"}); err != nil {
 		t.Fatal(err)
 	}

scripts/gitea-job-started

@@ -10,6 +10,7 @@ while [ "$(date +%s)" -lt "$deadline" ]; do
         -audience ci-job-ready \
         -socketPath "$socket" \
         >/dev/null 2>&1; then
+        /usr/local/libexec/setup-job-docker
         exit 0
     fi
     sleep 1

scripts/setup-job-docker

@@ -0,0 +1,60 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+if docker info >/dev/null 2>&1; then
+  printf '%s\n' 'job Docker daemon is already ready'
+  exit 0
+fi
+
+storage_size=${DOCKER_DATA_SIZE:-20G}
+storage_driver=${DOCKER_STORAGE_DRIVER:-overlay2}
+wait_seconds=${DOCKER_START_WAIT_SECONDS:-60}
+
+sudo install -d /var/lib/docker
+if ! mountpoint --quiet /var/lib/docker; then
+  printf '%s\n' "preparing ${storage_size} loop-backed Docker storage"
+  if [[ ! -e /dev/loop-control ]]; then
+    sudo mknod /dev/loop-control c 10 237
+  fi
+  for minor in {0..7}; do
+    if [[ ! -e "/dev/loop${minor}" ]]; then
+      sudo mknod "/dev/loop${minor}" b 7 "$minor"
+    fi
+  done
+  sudo truncate -s "$storage_size" /tmp/docker-data.img
+  sudo mkfs.ext4 -F /tmp/docker-data.img
+  loop_device=$(sudo losetup --find --show /tmp/docker-data.img)
+  sudo mount "$loop_device" /var/lib/docker
+else
+  printf '%s\n' 'using mounted Docker storage at /var/lib/docker'
+fi
+
+if [[ -f /sys/fs/cgroup/cgroup.controllers ]]; then
+  sudo sh -c '
+    mkdir -p /sys/fs/cgroup/init
+    while read -r pid; do
+      printf "%s\n" "$pid" \
+        >/sys/fs/cgroup/init/cgroup.procs 2>/dev/null || true
+    done </sys/fs/cgroup/cgroup.procs
+    sed -e "s/ / +/g" -e "s/^/+" \
+      /sys/fs/cgroup/cgroup.controllers \
+      >/sys/fs/cgroup/cgroup.subtree_control
+  '
+fi
+
+sudo sh -c 'exec nohup dockerd "$@" >/tmp/dockerd.log 2>&1' sh \
+  --host=unix:///var/run/docker.sock \
+  --storage-driver="$storage_driver" &
+printf '%s\n' 'waiting for job Docker daemon'
+for ((attempt = 0; attempt < wait_seconds; attempt++)); do
+  if docker info >/dev/null 2>&1; then
+    findmnt /var/lib/docker
+    docker info --format \
+      '{{json .ServerVersion}} {{json .Driver}} {{json .CgroupVersion}}'
+    exit 0
+  fi
+  sleep 1
+done
+
+cat /tmp/dockerd.log
+exit 1