diff --git a/.gitea/workflows/pod-smoke.yml b/.gitea/workflows/pod-smoke.yml index eb73c7f..6dd0a6c 100644 --- a/.gitea/workflows/pod-smoke.yml +++ b/.gitea/workflows/pod-smoke.yml @@ -20,28 +20,11 @@ jobs: >/dev/null test "$(id -u)" = 2000 - - name: Start job-local Docker - shell: bash - run: | - set -euo pipefail - findmnt /var/lib/docker - sudo nohup dockerd \ - --host=unix:///var/run/docker.sock \ - --storage-driver=overlay2 \ - >/tmp/dockerd.log 2>&1 & - for _ in {1..60}; do - if docker info >/dev/null 2>&1; then - exit 0 - fi - sleep 1 - done - cat /tmp/dockerd.log - exit 1 - - name: Build and run image shell: bash run: | set -euo pipefail + findmnt /var/lib/docker context=$(mktemp -d) cleanup() { docker image rm --force pod-docker-smoke:test \ diff --git a/.gitea/workflows/publish-images.yml b/.gitea/workflows/publish-images.yml index bb09832..b3cdc75 100644 --- a/.gitea/workflows/publish-images.yml +++ b/.gitea/workflows/publish-images.yml @@ -63,6 +63,10 @@ jobs: set -euo pipefail set +x + # Bootstrap the image that first introduces automatic Docker setup. + # Once deployed, the job-started hook makes this an idempotent no-op. + sudo scripts/setup-job-docker + : "${GITHUB_SHA:?GITHUB_SHA is required}" image_tag="sha-${GITHUB_SHA}" docker_config=$(mktemp -d) diff --git a/.gitea/workflows/vm-kind-smoke.yml b/.gitea/workflows/vm-kind-smoke.yml index 188e28b..2e7a055 100644 --- a/.gitea/workflows/vm-kind-smoke.yml +++ b/.gitea/workflows/vm-kind-smoke.yml @@ -7,64 +7,21 @@ on: jobs: kind: - runs-on: [self-hosted, vm-dev] + runs-on: [self-hosted, vm] steps: - - name: Start job-local Docker + - name: Prepare nested kubelet device shell: bash run: | set -euo pipefail if [[ ! -e /dev/kmsg ]]; then sudo mknod /dev/kmsg c 1 11 fi - sudo install -d /var/lib/docker - sudo truncate -s 20G /tmp/docker-data.img - sudo mkfs.ext4 -F /tmp/docker-data.img - if [[ ! -e /dev/loop-control ]]; then - sudo mknod /dev/loop-control c 10 237 - fi - for minor in {0..7}; do - if [[ ! -e "/dev/loop${minor}" ]]; then - sudo mknod "/dev/loop${minor}" b 7 "$minor" - fi - done - loop_device=$(sudo losetup --find --show /tmp/docker-data.img) - sudo mount "$loop_device" /var/lib/docker - findmnt /var/lib/docker - - # A nested systemd needs a domain cgroup namespace. This is the - # cgroup v2 nesting initialization performed by the official DinD - # entrypoint, kept here because the shared runner image deliberately - # does not carry a second DinD-specific entrypoint. - if [[ -f /sys/fs/cgroup/cgroup.controllers ]]; then - sudo sh -c ' - mkdir -p /sys/fs/cgroup/init - while read -r pid; do - printf "%s\n" "$pid" \ - >/sys/fs/cgroup/init/cgroup.procs 2>/dev/null || true - done /sys/fs/cgroup/cgroup.subtree_control - ' - fi - - sudo nohup dockerd \ - --host=unix:///var/run/docker.sock \ - --storage-driver=overlay2 \ - >/tmp/dockerd.log 2>&1 & - for _ in {1..60}; do - if docker info >/dev/null 2>&1; then - exit 0 - fi - sleep 1 - done - cat /tmp/dockerd.log - exit 1 - name: Verify Docker shell: bash run: | set -euo pipefail + findmnt /var/lib/docker docker info echo '### runner cgroup' cat /proc/self/cgroup diff --git a/.gitea/workflows/vm-runtime-smoke.yml b/.gitea/workflows/vm-runtime-smoke.yml index 76153c4..288f98c 100644 --- a/.gitea/workflows/vm-runtime-smoke.yml +++ b/.gitea/workflows/vm-runtime-smoke.yml @@ -7,7 +7,7 @@ on: jobs: runtime: - runs-on: [self-hosted, vm-dev] + runs-on: [self-hosted, vm] steps: - name: Verify workload identity socket shell: bash diff --git a/README.md b/README.md index 2762fc0..2b86479 100644 --- a/README.md +++ b/README.md @@ -36,8 +36,9 @@ runs-on: [self-hosted, vm] entry；不持有 OpenSandbox API key、Gitea token 或 Bao 凭据。身份与 Pool 契约见 [`docs/opensandbox-runner.md`](docs/opensandbox-runner.md)。 - Pod executor：在 Kubernetes 中创建一次性 privileged Pod；Pod 内的 workflow 使用 - host executor，Docker、BuildKit 和 kind 等工具由 pipeline 按需 setup。Runner 固定在 - 支持原生 job hooks 的 3.x 版本，在 workflow 第一步前等待实际任务对应的 SVID。 + host executor。Runner 固定在支持原生 job hooks 的 3.x 版本，在 workflow 第一步前 + 等待实际任务对应的 SVID，并启动 job-local Docker daemon；workflow 可直接使用与 + GitHub-hosted runner 相同的 Docker/BuildKit action。 - `jwt-broker`：早期共享 Kubernetes runner 的过渡实验；目标架构不部署它，每个 动态 Pod 或 VM 直接取得自己的 SPIFFE 身份。 diff --git a/container/runner.Dockerfile b/container/runner.Dockerfile index c9e7182..0408ba6 100644 --- a/container/runner.Dockerfile +++ b/container/runner.Dockerfile @@ -29,6 +29,7 @@ COPY --from=spire /opt/spire/bin/spire-agent /opt/spire/bin/spire-agent COPY config/runner.yaml /etc/gitea-runner/config.yaml COPY --chmod=0755 scripts/gitea-job-started /usr/local/libexec/gitea-job-started COPY --chmod=0755 scripts/gitea-opensandbox-runner /usr/local/libexec/gitea-opensandbox-runner +COPY --chmod=0755 scripts/setup-job-docker /usr/local/libexec/setup-job-docker VOLUME ["/data"] ENV HOME=/home/runner diff --git a/docs/opensandbox-runner.md b/docs/opensandbox-runner.md index 6bfba8d..d651941 100644 --- a/docs/opensandbox-runner.md +++ b/docs/opensandbox-runner.md @@ -42,8 +42,8 @@ UID attestation 的临时 Agent 失去父级。 - runner 镜像包含 Gitea Runner、Node.js action userspace、SPIRE CLI、Docker 工具和 identity gate；Pod 与 VM backend 使用同一个镜像； -- runner UID 2000；SPIRE Agent 独立运行，需要 Docker 的 workflow 通过 sudo 在 - privileged executor 内启动 job-local daemon； +- runner UID 2000；SPIRE Agent 独立运行；job-started hook 在第一步 workflow 之前 + 启动 job-local Docker daemon，业务 workflow 不负责 runner 基础设施初始化； - Kata VM 中 Docker 数据使用 guest 内的 loop-backed ext4，并随 sandbox 一起删除； - `self-hosted` 必须是所有 runner labels 的前缀； - ephemeral/once runner 完成一项任务后退出。 diff --git a/scripts/gitea-job-started b/scripts/gitea-job-started index b6b3b83..59cb557 100644 --- a/scripts/gitea-job-started +++ b/scripts/gitea-job-started @@ -10,6 +10,7 @@ while [ "$(date +%s)" -lt "$deadline" ]; do -audience ci-job-ready \ -socketPath "$socket" \ >/dev/null 2>&1; then + /usr/local/libexec/setup-job-docker exit 0 fi sleep 1 diff --git a/scripts/setup-job-docker b/scripts/setup-job-docker new file mode 100755 index 0000000..fef16b0 --- /dev/null +++ b/scripts/setup-job-docker @@ -0,0 +1,61 @@ +#!/usr/bin/env bash +set -euo pipefail + +if docker info >/dev/null 2>&1; then + printf '%s\n' 'job Docker daemon is already ready' + exit 0 +fi + +storage_size=${DOCKER_DATA_SIZE:-20G} +storage_driver=${DOCKER_STORAGE_DRIVER:-overlay2} +wait_seconds=${DOCKER_START_WAIT_SECONDS:-60} + +sudo install -d /var/lib/docker +if ! mountpoint --quiet /var/lib/docker; then + printf '%s\n' "preparing ${storage_size} loop-backed Docker storage" + if [[ ! -e /dev/loop-control ]]; then + sudo mknod /dev/loop-control c 10 237 + fi + for minor in {0..7}; do + if [[ ! -e "/dev/loop${minor}" ]]; then + sudo mknod "/dev/loop${minor}" b 7 "$minor" + fi + done + sudo truncate -s "$storage_size" /tmp/docker-data.img + sudo mkfs.ext4 -F /tmp/docker-data.img + loop_device=$(sudo losetup --find --show /tmp/docker-data.img) + sudo mount "$loop_device" /var/lib/docker +else + printf '%s\n' 'using mounted Docker storage at /var/lib/docker' +fi + +if [[ -f /sys/fs/cgroup/cgroup.controllers ]]; then + sudo sh -c ' + mkdir -p /sys/fs/cgroup/init + while read -r pid; do + printf "%s\n" "$pid" \ + >/sys/fs/cgroup/init/cgroup.procs 2>/dev/null || true + done /sys/fs/cgroup/cgroup.subtree_control + ' +fi + +sudo nohup dockerd \ + --host=unix:///var/run/docker.sock \ + --storage-driver="$storage_driver" \ + >/tmp/dockerd.log 2>&1 & +printf '%s\n' 'waiting for job Docker daemon' +for ((attempt = 0; attempt < wait_seconds; attempt++)); do + if docker info >/dev/null 2>&1; then + findmnt /var/lib/docker + docker info --format \ + '{{json .ServerVersion}} {{json .Driver}} {{json .CgroupVersion}}' + exit 0 + fi + sleep 1 +done + +cat /tmp/dockerd.log +exit 1