feat(runner): provide Docker before workflow steps
This commit is contained in:
@@ -20,28 +20,11 @@ jobs:
|
||||
>/dev/null
|
||||
test "$(id -u)" = 2000
|
||||
|
||||
- name: Start job-local Docker
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
findmnt /var/lib/docker
|
||||
sudo nohup dockerd \
|
||||
--host=unix:///var/run/docker.sock \
|
||||
--storage-driver=overlay2 \
|
||||
>/tmp/dockerd.log 2>&1 &
|
||||
for _ in {1..60}; do
|
||||
if docker info >/dev/null 2>&1; then
|
||||
exit 0
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
cat /tmp/dockerd.log
|
||||
exit 1
|
||||
|
||||
- name: Build and run image
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
findmnt /var/lib/docker
|
||||
context=$(mktemp -d)
|
||||
cleanup() {
|
||||
docker image rm --force pod-docker-smoke:test \
|
||||
|
||||
@@ -63,6 +63,10 @@ jobs:
|
||||
set -euo pipefail
|
||||
set +x
|
||||
|
||||
# Bootstrap the image that first introduces automatic Docker setup.
|
||||
# Once deployed, the job-started hook makes this an idempotent no-op.
|
||||
sudo scripts/setup-job-docker
|
||||
|
||||
: "${GITHUB_SHA:?GITHUB_SHA is required}"
|
||||
image_tag="sha-${GITHUB_SHA}"
|
||||
docker_config=$(mktemp -d)
|
||||
|
||||
@@ -7,64 +7,21 @@ on:
|
||||
|
||||
jobs:
|
||||
kind:
|
||||
runs-on: [self-hosted, vm-dev]
|
||||
runs-on: [self-hosted, vm]
|
||||
steps:
|
||||
- name: Start job-local Docker
|
||||
- name: Prepare nested kubelet device
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
if [[ ! -e /dev/kmsg ]]; then
|
||||
sudo mknod /dev/kmsg c 1 11
|
||||
fi
|
||||
sudo install -d /var/lib/docker
|
||||
sudo truncate -s 20G /tmp/docker-data.img
|
||||
sudo mkfs.ext4 -F /tmp/docker-data.img
|
||||
if [[ ! -e /dev/loop-control ]]; then
|
||||
sudo mknod /dev/loop-control c 10 237
|
||||
fi
|
||||
for minor in {0..7}; do
|
||||
if [[ ! -e "/dev/loop${minor}" ]]; then
|
||||
sudo mknod "/dev/loop${minor}" b 7 "$minor"
|
||||
fi
|
||||
done
|
||||
loop_device=$(sudo losetup --find --show /tmp/docker-data.img)
|
||||
sudo mount "$loop_device" /var/lib/docker
|
||||
findmnt /var/lib/docker
|
||||
|
||||
# A nested systemd needs a domain cgroup namespace. This is the
|
||||
# cgroup v2 nesting initialization performed by the official DinD
|
||||
# entrypoint, kept here because the shared runner image deliberately
|
||||
# does not carry a second DinD-specific entrypoint.
|
||||
if [[ -f /sys/fs/cgroup/cgroup.controllers ]]; then
|
||||
sudo sh -c '
|
||||
mkdir -p /sys/fs/cgroup/init
|
||||
while read -r pid; do
|
||||
printf "%s\n" "$pid" \
|
||||
>/sys/fs/cgroup/init/cgroup.procs 2>/dev/null || true
|
||||
done </sys/fs/cgroup/cgroup.procs
|
||||
sed -e "s/ / +/g" -e "s/^/+/" \
|
||||
/sys/fs/cgroup/cgroup.controllers \
|
||||
>/sys/fs/cgroup/cgroup.subtree_control
|
||||
'
|
||||
fi
|
||||
|
||||
sudo nohup dockerd \
|
||||
--host=unix:///var/run/docker.sock \
|
||||
--storage-driver=overlay2 \
|
||||
>/tmp/dockerd.log 2>&1 &
|
||||
for _ in {1..60}; do
|
||||
if docker info >/dev/null 2>&1; then
|
||||
exit 0
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
cat /tmp/dockerd.log
|
||||
exit 1
|
||||
|
||||
- name: Verify Docker
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
findmnt /var/lib/docker
|
||||
docker info
|
||||
echo '### runner cgroup'
|
||||
cat /proc/self/cgroup
|
||||
|
||||
@@ -7,7 +7,7 @@ on:
|
||||
|
||||
jobs:
|
||||
runtime:
|
||||
runs-on: [self-hosted, vm-dev]
|
||||
runs-on: [self-hosted, vm]
|
||||
steps:
|
||||
- name: Verify workload identity socket
|
||||
shell: bash
|
||||
|
||||
@@ -36,8 +36,9 @@ runs-on: [self-hosted, vm]
|
||||
entry;不持有 OpenSandbox API key、Gitea token 或 Bao 凭据。身份与 Pool 契约见
|
||||
[`docs/opensandbox-runner.md`](docs/opensandbox-runner.md)。
|
||||
- Pod executor:在 Kubernetes 中创建一次性 privileged Pod;Pod 内的 workflow 使用
|
||||
host executor,Docker、BuildKit 和 kind 等工具由 pipeline 按需 setup。Runner 固定在
|
||||
支持原生 job hooks 的 3.x 版本,在 workflow 第一步前等待实际任务对应的 SVID。
|
||||
host executor。Runner 固定在支持原生 job hooks 的 3.x 版本,在 workflow 第一步前
|
||||
等待实际任务对应的 SVID,并启动 job-local Docker daemon;workflow 可直接使用与
|
||||
GitHub-hosted runner 相同的 Docker/BuildKit action。
|
||||
- `jwt-broker`:早期共享 Kubernetes runner 的过渡实验;目标架构不部署它,每个
|
||||
动态 Pod 或 VM 直接取得自己的 SPIFFE 身份。
|
||||
|
||||
|
||||
@@ -29,6 +29,7 @@ COPY --from=spire /opt/spire/bin/spire-agent /opt/spire/bin/spire-agent
|
||||
COPY config/runner.yaml /etc/gitea-runner/config.yaml
|
||||
COPY --chmod=0755 scripts/gitea-job-started /usr/local/libexec/gitea-job-started
|
||||
COPY --chmod=0755 scripts/gitea-opensandbox-runner /usr/local/libexec/gitea-opensandbox-runner
|
||||
COPY --chmod=0755 scripts/setup-job-docker /usr/local/libexec/setup-job-docker
|
||||
|
||||
VOLUME ["/data"]
|
||||
ENV HOME=/home/runner
|
||||
|
||||
@@ -42,8 +42,8 @@ UID attestation 的临时 Agent 失去父级。
|
||||
|
||||
- runner 镜像包含 Gitea Runner、Node.js action userspace、SPIRE CLI、Docker 工具和
|
||||
identity gate;Pod 与 VM backend 使用同一个镜像;
|
||||
- runner UID 2000;SPIRE Agent 独立运行,需要 Docker 的 workflow 通过 sudo 在
|
||||
privileged executor 内启动 job-local daemon;
|
||||
- runner UID 2000;SPIRE Agent 独立运行;job-started hook 在第一步 workflow 之前
|
||||
启动 job-local Docker daemon,业务 workflow 不负责 runner 基础设施初始化;
|
||||
- Kata VM 中 Docker 数据使用 guest 内的 loop-backed ext4,并随 sandbox 一起删除;
|
||||
- `self-hosted` 必须是所有 runner labels 的前缀;
|
||||
- ephemeral/once runner 完成一项任务后退出。
|
||||
|
||||
@@ -10,6 +10,7 @@ while [ "$(date +%s)" -lt "$deadline" ]; do
|
||||
-audience ci-job-ready \
|
||||
-socketPath "$socket" \
|
||||
>/dev/null 2>&1; then
|
||||
/usr/local/libexec/setup-job-docker
|
||||
exit 0
|
||||
fi
|
||||
sleep 1
|
||||
|
||||
Executable
+61
@@ -0,0 +1,61 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
if docker info >/dev/null 2>&1; then
|
||||
printf '%s\n' 'job Docker daemon is already ready'
|
||||
exit 0
|
||||
fi
|
||||
|
||||
storage_size=${DOCKER_DATA_SIZE:-20G}
|
||||
storage_driver=${DOCKER_STORAGE_DRIVER:-overlay2}
|
||||
wait_seconds=${DOCKER_START_WAIT_SECONDS:-60}
|
||||
|
||||
sudo install -d /var/lib/docker
|
||||
if ! mountpoint --quiet /var/lib/docker; then
|
||||
printf '%s\n' "preparing ${storage_size} loop-backed Docker storage"
|
||||
if [[ ! -e /dev/loop-control ]]; then
|
||||
sudo mknod /dev/loop-control c 10 237
|
||||
fi
|
||||
for minor in {0..7}; do
|
||||
if [[ ! -e "/dev/loop${minor}" ]]; then
|
||||
sudo mknod "/dev/loop${minor}" b 7 "$minor"
|
||||
fi
|
||||
done
|
||||
sudo truncate -s "$storage_size" /tmp/docker-data.img
|
||||
sudo mkfs.ext4 -F /tmp/docker-data.img
|
||||
loop_device=$(sudo losetup --find --show /tmp/docker-data.img)
|
||||
sudo mount "$loop_device" /var/lib/docker
|
||||
else
|
||||
printf '%s\n' 'using mounted Docker storage at /var/lib/docker'
|
||||
fi
|
||||
|
||||
if [[ -f /sys/fs/cgroup/cgroup.controllers ]]; then
|
||||
sudo sh -c '
|
||||
mkdir -p /sys/fs/cgroup/init
|
||||
while read -r pid; do
|
||||
printf "%s\n" "$pid" \
|
||||
>/sys/fs/cgroup/init/cgroup.procs 2>/dev/null || true
|
||||
done </sys/fs/cgroup/cgroup.procs
|
||||
sed -e "s/ / +/g" -e "s/^/+" \
|
||||
/sys/fs/cgroup/cgroup.controllers \
|
||||
>/sys/fs/cgroup/cgroup.subtree_control
|
||||
'
|
||||
fi
|
||||
|
||||
sudo nohup dockerd \
|
||||
--host=unix:///var/run/docker.sock \
|
||||
--storage-driver="$storage_driver" \
|
||||
>/tmp/dockerd.log 2>&1 &
|
||||
printf '%s\n' 'waiting for job Docker daemon'
|
||||
for ((attempt = 0; attempt < wait_seconds; attempt++)); do
|
||||
if docker info >/dev/null 2>&1; then
|
||||
findmnt /var/lib/docker
|
||||
docker info --format \
|
||||
'{{json .ServerVersion}} {{json .Driver}} {{json .CgroupVersion}}'
|
||||
exit 0
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
|
||||
cat /tmp/dockerd.log
|
||||
exit 1
|
||||
Reference in New Issue
Block a user