From adb5af14860f9bf45d7a46ed2506618a63115509 Mon Sep 17 00:00:00 2001 From: panxiao81 Date: Mon, 21 Sep 2026 15:43:26 +0000 Subject: [PATCH 1/3] =?UTF-8?q?fix:=20=E4=B8=BA=E9=95=9C=E5=83=8F=E5=8F=91?= =?UTF-8?q?=E5=B8=83=E5=87=86=E5=A4=87=E6=9E=84=E5=BB=BA=E5=B7=A5=E5=85=B7?= =?UTF-8?q?=E9=93=BE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .gitea/workflows/publish-images.yml | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/.gitea/workflows/publish-images.yml b/.gitea/workflows/publish-images.yml index 0cb4bd2..bb09832 100644 --- a/.gitea/workflows/publish-images.yml +++ b/.gitea/workflows/publish-images.yml @@ -34,17 +34,27 @@ jobs: steps: - uses: actions/checkout@v4 + - uses: actions/setup-go@v5 + with: + go-version-file: go.mod + + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + - name: Test source shell: bash run: | set -euo pipefail go test ./... go vet ./... - python3 -m pip install --break-system-packages -e '.[test]' + python3 -m venv /tmp/test-venv + source /tmp/test-venv/bin/activate + python3 -m pip install -e '.[test]' pytest -q python3 -m compileall -q src tests - apt-get update - apt-get install --yes --no-install-recommends shellcheck + sudo apt-get update + sudo apt-get install --yes --no-install-recommends shellcheck shellcheck scripts/* - name: Build and publish From d776fa71e9ae25ac7d8006e80a84baf9ffb53619 Mon Sep 17 00:00:00 2001 From: panxiao81 Date: Mon, 21 Sep 2026 15:53:03 +0000 Subject: [PATCH 2/3] feat(runner): provide Docker before workflow steps --- .gitea/workflows/pod-smoke.yml | 19 +-------- .gitea/workflows/publish-images.yml | 4 ++ .gitea/workflows/vm-kind-smoke.yml | 49 ++------------------- .gitea/workflows/vm-runtime-smoke.yml | 2 +- README.md | 5 ++- container/runner.Dockerfile | 1 + docs/opensandbox-runner.md | 4 +- scripts/gitea-job-started | 1 + scripts/setup-job-docker | 61 +++++++++++++++++++++++++++ 9 files changed, 77 insertions(+), 69 deletions(-) create mode 100755 scripts/setup-job-docker diff --git a/.gitea/workflows/pod-smoke.yml b/.gitea/workflows/pod-smoke.yml index eb73c7f..6dd0a6c 100644 --- a/.gitea/workflows/pod-smoke.yml +++ b/.gitea/workflows/pod-smoke.yml @@ -20,28 +20,11 @@ jobs: >/dev/null test "$(id -u)" = 2000 - - name: Start job-local Docker - shell: bash - run: | - set -euo pipefail - findmnt /var/lib/docker - sudo nohup dockerd \ - --host=unix:///var/run/docker.sock \ - --storage-driver=overlay2 \ - >/tmp/dockerd.log 2>&1 & - for _ in {1..60}; do - if docker info >/dev/null 2>&1; then - exit 0 - fi - sleep 1 - done - cat /tmp/dockerd.log - exit 1 - - name: Build and run image shell: bash run: | set -euo pipefail + findmnt /var/lib/docker context=$(mktemp -d) cleanup() { docker image rm --force pod-docker-smoke:test \ diff --git a/.gitea/workflows/publish-images.yml b/.gitea/workflows/publish-images.yml index bb09832..b3cdc75 100644 --- a/.gitea/workflows/publish-images.yml +++ b/.gitea/workflows/publish-images.yml @@ -63,6 +63,10 @@ jobs: set -euo pipefail set +x + # Bootstrap the image that first introduces automatic Docker setup. + # Once deployed, the job-started hook makes this an idempotent no-op. + sudo scripts/setup-job-docker + : "${GITHUB_SHA:?GITHUB_SHA is required}" image_tag="sha-${GITHUB_SHA}" docker_config=$(mktemp -d) diff --git a/.gitea/workflows/vm-kind-smoke.yml b/.gitea/workflows/vm-kind-smoke.yml index 188e28b..2e7a055 100644 --- a/.gitea/workflows/vm-kind-smoke.yml +++ b/.gitea/workflows/vm-kind-smoke.yml @@ -7,64 +7,21 @@ on: jobs: kind: - runs-on: [self-hosted, vm-dev] + runs-on: [self-hosted, vm] steps: - - name: Start job-local Docker + - name: Prepare nested kubelet device shell: bash run: | set -euo pipefail if [[ ! -e /dev/kmsg ]]; then sudo mknod /dev/kmsg c 1 11 fi - sudo install -d /var/lib/docker - sudo truncate -s 20G /tmp/docker-data.img - sudo mkfs.ext4 -F /tmp/docker-data.img - if [[ ! -e /dev/loop-control ]]; then - sudo mknod /dev/loop-control c 10 237 - fi - for minor in {0..7}; do - if [[ ! -e "/dev/loop${minor}" ]]; then - sudo mknod "/dev/loop${minor}" b 7 "$minor" - fi - done - loop_device=$(sudo losetup --find --show /tmp/docker-data.img) - sudo mount "$loop_device" /var/lib/docker - findmnt /var/lib/docker - - # A nested systemd needs a domain cgroup namespace. This is the - # cgroup v2 nesting initialization performed by the official DinD - # entrypoint, kept here because the shared runner image deliberately - # does not carry a second DinD-specific entrypoint. - if [[ -f /sys/fs/cgroup/cgroup.controllers ]]; then - sudo sh -c ' - mkdir -p /sys/fs/cgroup/init - while read -r pid; do - printf "%s\n" "$pid" \ - >/sys/fs/cgroup/init/cgroup.procs 2>/dev/null || true - done /sys/fs/cgroup/cgroup.subtree_control - ' - fi - - sudo nohup dockerd \ - --host=unix:///var/run/docker.sock \ - --storage-driver=overlay2 \ - >/tmp/dockerd.log 2>&1 & - for _ in {1..60}; do - if docker info >/dev/null 2>&1; then - exit 0 - fi - sleep 1 - done - cat /tmp/dockerd.log - exit 1 - name: Verify Docker shell: bash run: | set -euo pipefail + findmnt /var/lib/docker docker info echo '### runner cgroup' cat /proc/self/cgroup diff --git a/.gitea/workflows/vm-runtime-smoke.yml b/.gitea/workflows/vm-runtime-smoke.yml index 76153c4..288f98c 100644 --- a/.gitea/workflows/vm-runtime-smoke.yml +++ b/.gitea/workflows/vm-runtime-smoke.yml @@ -7,7 +7,7 @@ on: jobs: runtime: - runs-on: [self-hosted, vm-dev] + runs-on: [self-hosted, vm] steps: - name: Verify workload identity socket shell: bash diff --git a/README.md b/README.md index 2762fc0..2b86479 100644 --- a/README.md +++ b/README.md @@ -36,8 +36,9 @@ runs-on: [self-hosted, vm] entry；不持有 OpenSandbox API key、Gitea token 或 Bao 凭据。身份与 Pool 契约见 [`docs/opensandbox-runner.md`](docs/opensandbox-runner.md)。 - Pod executor：在 Kubernetes 中创建一次性 privileged Pod；Pod 内的 workflow 使用 - host executor，Docker、BuildKit 和 kind 等工具由 pipeline 按需 setup。Runner 固定在 - 支持原生 job hooks 的 3.x 版本，在 workflow 第一步前等待实际任务对应的 SVID。 + host executor。Runner 固定在支持原生 job hooks 的 3.x 版本，在 workflow 第一步前 + 等待实际任务对应的 SVID，并启动 job-local Docker daemon；workflow 可直接使用与 + GitHub-hosted runner 相同的 Docker/BuildKit action。 - `jwt-broker`：早期共享 Kubernetes runner 的过渡实验；目标架构不部署它，每个 动态 Pod 或 VM 直接取得自己的 SPIFFE 身份。 diff --git a/container/runner.Dockerfile b/container/runner.Dockerfile index c9e7182..0408ba6 100644 --- a/container/runner.Dockerfile +++ b/container/runner.Dockerfile @@ -29,6 +29,7 @@ COPY --from=spire /opt/spire/bin/spire-agent /opt/spire/bin/spire-agent COPY config/runner.yaml /etc/gitea-runner/config.yaml COPY --chmod=0755 scripts/gitea-job-started /usr/local/libexec/gitea-job-started COPY --chmod=0755 scripts/gitea-opensandbox-runner /usr/local/libexec/gitea-opensandbox-runner +COPY --chmod=0755 scripts/setup-job-docker /usr/local/libexec/setup-job-docker VOLUME ["/data"] ENV HOME=/home/runner diff --git a/docs/opensandbox-runner.md b/docs/opensandbox-runner.md index 6bfba8d..d651941 100644 --- a/docs/opensandbox-runner.md +++ b/docs/opensandbox-runner.md @@ -42,8 +42,8 @@ UID attestation 的临时 Agent 失去父级。 - runner 镜像包含 Gitea Runner、Node.js action userspace、SPIRE CLI、Docker 工具和 identity gate；Pod 与 VM backend 使用同一个镜像； -- runner UID 2000；SPIRE Agent 独立运行，需要 Docker 的 workflow 通过 sudo 在 - privileged executor 内启动 job-local daemon； +- runner UID 2000；SPIRE Agent 独立运行；job-started hook 在第一步 workflow 之前 + 启动 job-local Docker daemon，业务 workflow 不负责 runner 基础设施初始化； - Kata VM 中 Docker 数据使用 guest 内的 loop-backed ext4，并随 sandbox 一起删除； - `self-hosted` 必须是所有 runner labels 的前缀； - ephemeral/once runner 完成一项任务后退出。 diff --git a/scripts/gitea-job-started b/scripts/gitea-job-started index b6b3b83..59cb557 100644 --- a/scripts/gitea-job-started +++ b/scripts/gitea-job-started @@ -10,6 +10,7 @@ while [ "$(date +%s)" -lt "$deadline" ]; do -audience ci-job-ready \ -socketPath "$socket" \ >/dev/null 2>&1; then + /usr/local/libexec/setup-job-docker exit 0 fi sleep 1 diff --git a/scripts/setup-job-docker b/scripts/setup-job-docker new file mode 100755 index 0000000..fef16b0 --- /dev/null +++ b/scripts/setup-job-docker @@ -0,0 +1,61 @@ +#!/usr/bin/env bash +set -euo pipefail + +if docker info >/dev/null 2>&1; then + printf '%s\n' 'job Docker daemon is already ready' + exit 0 +fi + +storage_size=${DOCKER_DATA_SIZE:-20G} +storage_driver=${DOCKER_STORAGE_DRIVER:-overlay2} +wait_seconds=${DOCKER_START_WAIT_SECONDS:-60} + +sudo install -d /var/lib/docker +if ! mountpoint --quiet /var/lib/docker; then + printf '%s\n' "preparing ${storage_size} loop-backed Docker storage" + if [[ ! -e /dev/loop-control ]]; then + sudo mknod /dev/loop-control c 10 237 + fi + for minor in {0..7}; do + if [[ ! -e "/dev/loop${minor}" ]]; then + sudo mknod "/dev/loop${minor}" b 7 "$minor" + fi + done + sudo truncate -s "$storage_size" /tmp/docker-data.img + sudo mkfs.ext4 -F /tmp/docker-data.img + loop_device=$(sudo losetup --find --show /tmp/docker-data.img) + sudo mount "$loop_device" /var/lib/docker +else + printf '%s\n' 'using mounted Docker storage at /var/lib/docker' +fi + +if [[ -f /sys/fs/cgroup/cgroup.controllers ]]; then + sudo sh -c ' + mkdir -p /sys/fs/cgroup/init + while read -r pid; do + printf "%s\n" "$pid" \ + >/sys/fs/cgroup/init/cgroup.procs 2>/dev/null || true + done /sys/fs/cgroup/cgroup.subtree_control + ' +fi + +sudo nohup dockerd \ + --host=unix:///var/run/docker.sock \ + --storage-driver="$storage_driver" \ + >/tmp/dockerd.log 2>&1 & +printf '%s\n' 'waiting for job Docker daemon' +for ((attempt = 0; attempt < wait_seconds; attempt++)); do + if docker info >/dev/null 2>&1; then + findmnt /var/lib/docker + docker info --format \ + '{{json .ServerVersion}} {{json .Driver}} {{json .CgroupVersion}}' + exit 0 + fi + sleep 1 +done + +cat /tmp/dockerd.log +exit 1 From 0bf39b4751f34744140df4cbef52b33a2ab291b8 Mon Sep 17 00:00:00 2001 From: panxiao81 Date: Mon, 21 Sep 2026 16:20:02 +0000 Subject: [PATCH 3/3] fix(runner): redirect dockerd logs as root --- scripts/setup-job-docker | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/scripts/setup-job-docker b/scripts/setup-job-docker index fef16b0..16b1c2d 100755 --- a/scripts/setup-job-docker +++ b/scripts/setup-job-docker @@ -42,10 +42,9 @@ if [[ -f /sys/fs/cgroup/cgroup.controllers ]]; then ' fi -sudo nohup dockerd \ +sudo sh -c 'exec nohup dockerd "$@" >/tmp/dockerd.log 2>&1' sh \ --host=unix:///var/run/docker.sock \ - --storage-driver="$storage_driver" \ - >/tmp/dockerd.log 2>&1 & + --storage-driver="$storage_driver" & printf '%s\n' 'waiting for job Docker daemon' for ((attempt = 0; attempt < wait_seconds; attempt++)); do if docker info >/dev/null 2>&1; then