From 3641e6ffb3beab556f1f1a0895ad9fb309e70bea Mon Sep 17 00:00:00 2001 From: panxiao81 Date: Wed, 16 Sep 2026 16:38:03 +0000 Subject: [PATCH 01/11] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=20LXC=20=E5=86=85=20mi?= =?UTF-8?q?croVM=20Runner=20=E5=90=AF=E5=8A=A8=E9=93=BE=E8=B7=AF?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 6 +++++- config/runner-vm-bootstrap.yaml | 11 +++++++++++ scripts/gitea-microvm-guest-runner | 3 ++- scripts/microvm-runner-launch | 11 +++++++---- src/gitea_dynamic_runner/worker.py | 11 +++++++++++ 5 files changed, 36 insertions(+), 6 deletions(-) create mode 100644 config/runner-vm-bootstrap.yaml diff --git a/README.md b/README.md index e292367..55e0c87 100644 --- a/README.md +++ b/README.md @@ -20,7 +20,7 @@ runs-on: [self-hosted, vm] - `controller`：接收 Gitea `workflow_job` webhook，将指定 label 的 queued job 发布到 NATS JetStream。 - `worker`：领取任务、限制并发，并通过 Pod 或 microVM backend 创建一次性环境。 -- `microvm-runner-launch`：为每个任务创建 COW disk、NoCloud seed 和 TAP，运行 +- `microvm-runner-launch`：为每个任务复制 flat qcow2 root disk（支持时使用 reflink）、创建 NoCloud seed 和 TAP，运行 Cloud Hypervisor，退出后完整清理。 - `guest-runner`：在 guest 中领取一次性 runner registration token，注册 ephemeral runner，执行一个 job 后关机。 @@ -51,6 +51,10 @@ pytest - NATS 密码、webhook secret 和 Gitea registration token 只从文件读取。 - registration token 不写入 seed image；worker 通过单次 nonce endpoint 交给 guest。 +- guest 启动时从仅监听 microVM bridge 的 worker endpoint 获取固定版本 Runner 和配置 + 资产；基础镜像无需为 Runner 发布而重做。 +- `runner-vm-bootstrap.yaml` 暂时只验证 VM 调度和生命周期，不提供 SPIFFE + identity；VM agent attestation 完成前不得将它当作身份链路验证结果。 - guest runner 使用 `--ephemeral`，每台 VM 只执行一个 job。 - launcher 只接受 UUID instance ID 和 URL-safe nonce，所有临时文件都位于独立目录。 - base image 不得包含 runner identity、registration token、SSH 密码或 host key。 diff --git a/config/runner-vm-bootstrap.yaml b/config/runner-vm-bootstrap.yaml new file mode 100644 index 0000000..93333fe --- /dev/null +++ b/config/runner-vm-bootstrap.yaml @@ -0,0 +1,11 @@ +runner: + capacity: 1 + timeout: 3h + shutdown_timeout: 1m + +host: + workdir_parent: /workspace + +container: + require_docker: false + valid_volumes: [] diff --git a/scripts/gitea-microvm-guest-runner b/scripts/gitea-microvm-guest-runner index 7d8d7ba..cee427f 100755 --- a/scripts/gitea-microvm-guest-runner +++ b/scripts/gitea-microvm-guest-runner @@ -6,6 +6,7 @@ instance=${2:?Gitea instance is required} runner_name=${3:?runner name is required} runner_labels=${4:?runner labels are required} token_file=/run/gitea-runner-registration-token +config_file=${GITEA_RUNNER_CONFIG_FILE:-/etc/gitea-runner/config-vm-bootstrap.yaml} cleanup() { rm -f -- "$token_file" @@ -25,4 +26,4 @@ gitea-runner register \ --labels "$runner_labels" \ --token-file "$token_file" rm -f -- "$token_file" -gitea-runner daemon +gitea-runner daemon --config "$config_file" --once diff --git a/scripts/microvm-runner-launch b/scripts/microvm-runner-launch index b84d0ce..558f232 100755 --- a/scripts/microvm-runner-launch +++ b/scripts/microvm-runner-launch @@ -39,7 +39,9 @@ trap cleanup EXIT INT TERM test -r "$base_image" test -r "$firmware" install -d -m 0700 "$state_root/instances" "$vm_dir" -qemu-img create -q -f qcow2 -F qcow2 -b "$base_image" "$overlay" +# Cloud Hypervisor cannot open qcow2 backing chains. Keep each disposable +# root disk flat; reflink-capable storage still makes this copy cheap. +cp --reflink=auto --sparse=always "$base_image" "$overlay" cat >"$vm_dir/meta-data" <"$vm_dir/user-data" < web.Response: return web.Response(body=value, headers={"Cache-Control": "no-store"}) +async def guest_assets(_: web.Request) -> web.FileResponse: + return web.FileResponse( + GUEST_ASSETS, + headers={"Cache-Control": "public, immutable"}, + ) + + async def heartbeat(message: object, stop: asyncio.Event) -> None: while True: try: @@ -162,6 +172,7 @@ async def consume() -> None: async def main() -> None: app = web.Application() app.router.add_get("/token/{nonce}", token) + app.router.add_get("/assets/guest-assets.tar.gz", guest_assets) runner = web.AppRunner(app) await runner.setup() await web.TCPSite(runner, TOKEN_LISTEN, TOKEN_PORT).start() From ce4c9f13b522029a4db093be3730e4d343ce4044 Mon Sep 17 00:00:00 2001 From: panxiao81 Date: Wed, 16 Sep 2026 16:39:06 +0000 Subject: [PATCH 02/11] =?UTF-8?q?=E6=B7=BB=E5=8A=A0=20microVM=20kind=20?= =?UTF-8?q?=E5=86=92=E7=83=9F=E6=B5=8B=E8=AF=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .gitea/workflows/vm-kind-smoke.yml | 32 ++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) create mode 100644 .gitea/workflows/vm-kind-smoke.yml diff --git a/.gitea/workflows/vm-kind-smoke.yml b/.gitea/workflows/vm-kind-smoke.yml new file mode 100644 index 0000000..0ede1f5 --- /dev/null +++ b/.gitea/workflows/vm-kind-smoke.yml @@ -0,0 +1,32 @@ +name: VM kind smoke + +on: + pull_request: + workflow_dispatch: + +jobs: + kind: + runs-on: [self-hosted, vm] + steps: + - name: Verify Docker + run: docker info + + - name: Install kind + shell: bash + run: | + set -euo pipefail + version=v0.33.0 + curl --fail --location --silent --show-error \ + --output /tmp/kind "https://kind.sigs.k8s.io/dl/${version}/kind-linux-amd64" + curl --fail --location --silent --show-error \ + --output /tmp/kind.sha256sum "https://kind.sigs.k8s.io/dl/${version}/kind-linux-amd64.sha256sum" + printf '%s %s\n' "$(cat /tmp/kind.sha256sum)" /tmp/kind | sha256sum --check + chmod 0755 /tmp/kind + + - name: Create and delete kind cluster + shell: bash + run: | + set -euo pipefail + trap '/tmp/kind delete cluster --name smoke' EXIT + /tmp/kind create cluster --name smoke --wait 180s + test "$(/tmp/kind get clusters)" = smoke From 70a8aa68c649f5d637b025aa1c986e1b2b8af20a Mon Sep 17 00:00:00 2001 From: panxiao81 Date: Wed, 16 Sep 2026 16:42:20 +0000 Subject: [PATCH 03/11] =?UTF-8?q?=E5=B0=86=20VM=20smoke=20=E6=94=B9?= =?UTF-8?q?=E4=B8=BA=E6=89=8B=E5=8A=A8=E8=A7=A6=E5=8F=91?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .gitea/workflows/vm-kind-smoke.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.gitea/workflows/vm-kind-smoke.yml b/.gitea/workflows/vm-kind-smoke.yml index 0ede1f5..c2b95c0 100644 --- a/.gitea/workflows/vm-kind-smoke.yml +++ b/.gitea/workflows/vm-kind-smoke.yml @@ -1,7 +1,6 @@ name: VM kind smoke on: - pull_request: workflow_dispatch: jobs: From ac85d5fe58183f9a48f18e011e80317ca93e0864 Mon Sep 17 00:00:00 2001 From: panxiao81 Date: Wed, 16 Sep 2026 16:44:05 +0000 Subject: [PATCH 04/11] =?UTF-8?q?=E6=A0=A1=E9=AA=8C=20microVM=20Runner=20?= =?UTF-8?q?=E4=BB=BB=E5=8A=A1=E5=AE=8C=E6=88=90=E7=8A=B6=E6=80=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- scripts/gitea-microvm-guest-runner | 1 + scripts/microvm-runner-launch | 27 ++++++++++++++++++--------- 2 files changed, 19 insertions(+), 9 deletions(-) diff --git a/scripts/gitea-microvm-guest-runner b/scripts/gitea-microvm-guest-runner index cee427f..3b4d076 100755 --- a/scripts/gitea-microvm-guest-runner +++ b/scripts/gitea-microvm-guest-runner @@ -27,3 +27,4 @@ gitea-runner register \ --token-file "$token_file" rm -f -- "$token_file" gitea-runner daemon --config "$config_file" --once +echo gitea-runner-job-complete >/dev/ttyS0 diff --git a/scripts/microvm-runner-launch b/scripts/microvm-runner-launch index 558f232..ff3865b 100755 --- a/scripts/microvm-runner-launch +++ b/scripts/microvm-runner-launch @@ -29,8 +29,13 @@ tap="mvr${instance_id%%-*}" overlay="$vm_dir/root.qcow2" seed="$vm_dir/seed.img" serial="$vm_dir/serial.log" +log_dir="$state_root/logs" cleanup() { + if test -f "$serial"; then + install -d -m 0700 "$log_dir" + cp "$serial" "$log_dir/$instance_id.log" + fi ip link delete "$tap" 2>/dev/null || true rm -rf -- "$vm_dir" } @@ -66,12 +71,16 @@ ip tuntap add dev "$tap" mode tap ip link set "$tap" master "$bridge" ip link set "$tap" up -timeout --signal=TERM --kill-after=30s "$vm_timeout" "$cloud_hypervisor" \ - --firmware "$firmware" \ - --cpus "boot=$cpus" \ - --memory "size=$memory" \ - --disk "path=$overlay,image_type=qcow2" \ - --disk "path=$seed,readonly=on,image_type=raw" \ - --net "tap=$tap,mac=$mac" \ - --serial "file=$serial" \ - --console off +if timeout --signal=TERM --kill-after=30s "$vm_timeout" "$cloud_hypervisor" \ + --firmware "$firmware" \ + --cpus "boot=$cpus" \ + --memory "size=$memory" \ + --disk "path=$overlay,image_type=qcow2" \ + --disk "path=$seed,readonly=on,image_type=raw" \ + --net "tap=$tap,mac=$mac" \ + --serial "file=$serial" \ + --console off; then + grep -Fq gitea-runner-job-complete "$serial" +else + exit $? +fi From 02e0b698c58f0e3a7ce6822fcda319d82c0fc631 Mon Sep 17 00:00:00 2001 From: panxiao81 Date: Wed, 16 Sep 2026 16:48:44 +0000 Subject: [PATCH 05/11] =?UTF-8?q?=E4=B8=BA=20cloud-init=20Runner=20?= =?UTF-8?q?=E8=AE=BE=E7=BD=AE=20HOME?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- scripts/gitea-microvm-guest-runner | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/gitea-microvm-guest-runner b/scripts/gitea-microvm-guest-runner index 3b4d076..041e33f 100755 --- a/scripts/gitea-microvm-guest-runner +++ b/scripts/gitea-microvm-guest-runner @@ -7,6 +7,7 @@ runner_name=${3:?runner name is required} runner_labels=${4:?runner labels are required} token_file=/run/gitea-runner-registration-token config_file=${GITEA_RUNNER_CONFIG_FILE:-/etc/gitea-runner/config-vm-bootstrap.yaml} +export HOME=${HOME:-/root} cleanup() { rm -f -- "$token_file" From 8578bee895e96e01a806612d21143fbe7ecceebf Mon Sep 17 00:00:00 2001 From: panxiao81 Date: Wed, 16 Sep 2026 16:50:52 +0000 Subject: [PATCH 06/11] =?UTF-8?q?=E4=BF=AE=E6=AD=A3=20kind=20=E4=BA=8C?= =?UTF-8?q?=E8=BF=9B=E5=88=B6=E6=A0=A1=E9=AA=8C?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .gitea/workflows/vm-kind-smoke.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitea/workflows/vm-kind-smoke.yml b/.gitea/workflows/vm-kind-smoke.yml index c2b95c0..ecad2af 100644 --- a/.gitea/workflows/vm-kind-smoke.yml +++ b/.gitea/workflows/vm-kind-smoke.yml @@ -19,7 +19,7 @@ jobs: --output /tmp/kind "https://kind.sigs.k8s.io/dl/${version}/kind-linux-amd64" curl --fail --location --silent --show-error \ --output /tmp/kind.sha256sum "https://kind.sigs.k8s.io/dl/${version}/kind-linux-amd64.sha256sum" - printf '%s %s\n' "$(cat /tmp/kind.sha256sum)" /tmp/kind | sha256sum --check + printf '%s %s\n' "$(cut -d ' ' -f1 /tmp/kind.sha256sum)" /tmp/kind | sha256sum --check chmod 0755 /tmp/kind - name: Create and delete kind cluster From b25b1fbf6285aae48079c6d5be62770468459305 Mon Sep 17 00:00:00 2001 From: panxiao81 Date: Wed, 16 Sep 2026 17:15:08 +0000 Subject: [PATCH 07/11] =?UTF-8?q?=E9=81=BF=E5=85=8D=20microVM=20=E9=95=9C?= =?UTF-8?q?=E5=83=8F=E5=A4=8D=E5=88=B6=E5=8D=A0=E6=BB=A1=20LXC=20page=20ca?= =?UTF-8?q?che?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 2 +- scripts/microvm-runner-launch | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 55e0c87..04d6668 100644 --- a/README.md +++ b/README.md @@ -20,7 +20,7 @@ runs-on: [self-hosted, vm] - `controller`：接收 Gitea `workflow_job` webhook，将指定 label 的 queued job 发布到 NATS JetStream。 - `worker`：领取任务、限制并发，并通过 Pod 或 microVM backend 创建一次性环境。 -- `microvm-runner-launch`：为每个任务复制 flat qcow2 root disk（支持时使用 reflink）、创建 NoCloud seed 和 TAP，运行 +- `microvm-runner-launch`：为每个任务以 direct I/O 转换出 flat qcow2 root disk、创建 NoCloud seed 和 TAP，运行 Cloud Hypervisor，退出后完整清理。 - `guest-runner`：在 guest 中领取一次性 runner registration token，注册 ephemeral runner，执行一个 job 后关机。 diff --git a/scripts/microvm-runner-launch b/scripts/microvm-runner-launch index ff3865b..64e3a5c 100755 --- a/scripts/microvm-runner-launch +++ b/scripts/microvm-runner-launch @@ -45,8 +45,9 @@ test -r "$base_image" test -r "$firmware" install -d -m 0700 "$state_root/instances" "$vm_dir" # Cloud Hypervisor cannot open qcow2 backing chains. Keep each disposable -# root disk flat; reflink-capable storage still makes this copy cheap. -cp --reflink=auto --sparse=always "$base_image" "$overlay" +# root disk flat and bypass the LXC page cache: caching both a 5 GiB copy and +# guest RAM can otherwise trigger the container memory limit. +qemu-img convert -q -T none -t none -f qcow2 -O qcow2 "$base_image" "$overlay" cat >"$vm_dir/meta-data" < Date: Wed, 16 Sep 2026 17:19:32 +0000 Subject: [PATCH 08/11] =?UTF-8?q?=E4=B8=BA=20microVM=20=E7=A3=81=E7=9B=98?= =?UTF-8?q?=E5=90=AF=E7=94=A8=20direct=20I/O?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- scripts/microvm-runner-launch | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/microvm-runner-launch b/scripts/microvm-runner-launch index 64e3a5c..50c044e 100755 --- a/scripts/microvm-runner-launch +++ b/scripts/microvm-runner-launch @@ -76,8 +76,8 @@ if timeout --signal=TERM --kill-after=30s "$vm_timeout" "$cloud_hypervisor" \ --firmware "$firmware" \ --cpus "boot=$cpus" \ --memory "size=$memory" \ - --disk "path=$overlay,image_type=qcow2" \ - --disk "path=$seed,readonly=on,image_type=raw" \ + --disk "path=$overlay,image_type=qcow2,direct=on,sparse=off" \ + --disk "path=$seed,readonly=on,image_type=raw,direct=on,sparse=off" \ --net "tap=$tap,mac=$mac" \ --serial "file=$serial" \ --console off; then From 6dac9897fd99a9c4c54428a5b026099daf00de47 Mon Sep 17 00:00:00 2001 From: panxiao81 Date: Wed, 16 Sep 2026 17:28:29 +0000 Subject: [PATCH 09/11] =?UTF-8?q?=E7=A8=B3=E5=AE=9A=20kind=20smoke=20?= =?UTF-8?q?=E4=B8=8E=20microVM=20=E5=86=85=E5=AD=98=E9=85=8D=E7=BD=AE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .gitea/workflows/vm-kind-smoke.yml | 2 +- scripts/microvm-runner-launch | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitea/workflows/vm-kind-smoke.yml b/.gitea/workflows/vm-kind-smoke.yml index ecad2af..fdcf7ea 100644 --- a/.gitea/workflows/vm-kind-smoke.yml +++ b/.gitea/workflows/vm-kind-smoke.yml @@ -28,4 +28,4 @@ jobs: set -euo pipefail trap '/tmp/kind delete cluster --name smoke' EXIT /tmp/kind create cluster --name smoke --wait 180s - test "$(/tmp/kind get clusters)" = smoke + /tmp/kind get clusters | grep -Fx smoke diff --git a/scripts/microvm-runner-launch b/scripts/microvm-runner-launch index 50c044e..4a29308 100755 --- a/scripts/microvm-runner-launch +++ b/scripts/microvm-runner-launch @@ -19,7 +19,7 @@ bridge=${RUNNER_BRIDGE:-mvrbr0} cloud_hypervisor=${CLOUD_HYPERVISOR:-/usr/local/bin/cloud-hypervisor} vm_timeout=${RUNNER_VM_TIMEOUT:-3h} cpus=${RUNNER_VM_CPUS:-4} -memory=${RUNNER_VM_MEMORY:-3G} +memory=${RUNNER_VM_MEMORY:-2G} gitea_instance=${GITEA_INSTANCE:-https://git.ddupan.top} runner_labels=${RUNNER_LABELS:-self-hosted:host,vm:host} token_url=${RUNNER_TOKEN_URL:-http://172.30.0.1:8787} From 401c1f9a00901b96e998f43b24d681bc625fc107 Mon Sep 17 00:00:00 2001 From: panxiao81 Date: Wed, 16 Sep 2026 17:38:08 +0000 Subject: [PATCH 10/11] =?UTF-8?q?=E5=85=B1=E4=BA=AB=20guest=20memfd=20?= =?UTF-8?q?=E9=81=BF=E5=85=8D=20LXC=20=E5=86=85=E5=AD=98=E5=8F=8C=E9=87=8D?= =?UTF-8?q?=E8=AE=A1=E8=B4=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- scripts/microvm-runner-launch | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/microvm-runner-launch b/scripts/microvm-runner-launch index 4a29308..299cb0f 100755 --- a/scripts/microvm-runner-launch +++ b/scripts/microvm-runner-launch @@ -75,7 +75,7 @@ ip link set "$tap" up if timeout --signal=TERM --kill-after=30s "$vm_timeout" "$cloud_hypervisor" \ --firmware "$firmware" \ --cpus "boot=$cpus" \ - --memory "size=$memory" \ + --memory "size=$memory,shared=on" \ --disk "path=$overlay,image_type=qcow2,direct=on,sparse=off" \ --disk "path=$seed,readonly=on,image_type=raw,direct=on,sparse=off" \ --net "tap=$tap,mac=$mac" \ From c4aa6ee0af9ce3d24475b4defb671fe89680a333 Mon Sep 17 00:00:00 2001 From: panxiao81 Date: Wed, 16 Sep 2026 17:42:52 +0000 Subject: [PATCH 11/11] =?UTF-8?q?=E8=AE=B0=E5=BD=95=20LXC=20guest=20?= =?UTF-8?q?=E5=86=85=E5=AD=98=E5=85=B1=E4=BA=AB=E8=A6=81=E6=B1=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 3 +++ scripts/microvm-runner-launch | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 04d6668..0e2178b 100644 --- a/README.md +++ b/README.md @@ -58,6 +58,9 @@ pytest - guest runner 使用 `--ephemeral`，每台 VM 只执行一个 job。 - launcher 只接受 UUID instance ID 和 URL-safe nonce，所有临时文件都位于独立目录。 - base image 不得包含 runner identity、registration token、SSH 密码或 host key。 +- LXC 内运行 Cloud Hypervisor 必须为 guest memory 启用 `shared=on`。默认的 private + memfd 映射会在 guest 写入后同时产生 shmem 与 anonymous CoW charge，使 LXC cgroup + 对 guest RAM 接近双倍计费。 homelab 的 Kubernetes、OpenBao、LXC、bridge 和容量配置保留在 `panxiao81/homelab-infra`。 diff --git a/scripts/microvm-runner-launch b/scripts/microvm-runner-launch index 299cb0f..c45956f 100755 --- a/scripts/microvm-runner-launch +++ b/scripts/microvm-runner-launch @@ -19,7 +19,7 @@ bridge=${RUNNER_BRIDGE:-mvrbr0} cloud_hypervisor=${CLOUD_HYPERVISOR:-/usr/local/bin/cloud-hypervisor} vm_timeout=${RUNNER_VM_TIMEOUT:-3h} cpus=${RUNNER_VM_CPUS:-4} -memory=${RUNNER_VM_MEMORY:-2G} +memory=${RUNNER_VM_MEMORY:-3G} gitea_instance=${GITEA_INSTANCE:-https://git.ddupan.top} runner_labels=${RUNNER_LABELS:-self-hosted:host,vm:host} token_url=${RUNNER_TOKEN_URL:-http://172.30.0.1:8787}