diff --git a/.gitea/workflows/vm-kind-smoke.yml b/.gitea/workflows/vm-kind-smoke.yml new file mode 100644 index 0000000..fdcf7ea --- /dev/null +++ b/.gitea/workflows/vm-kind-smoke.yml @@ -0,0 +1,31 @@ +name: VM kind smoke + +on: + workflow_dispatch: + +jobs: + kind: + runs-on: [self-hosted, vm] + steps: + - name: Verify Docker + run: docker info + + - name: Install kind + shell: bash + run: | + set -euo pipefail + version=v0.33.0 + curl --fail --location --silent --show-error \ + --output /tmp/kind "https://kind.sigs.k8s.io/dl/${version}/kind-linux-amd64" + curl --fail --location --silent --show-error \ + --output /tmp/kind.sha256sum "https://kind.sigs.k8s.io/dl/${version}/kind-linux-amd64.sha256sum" + printf '%s %s\n' "$(cut -d ' ' -f1 /tmp/kind.sha256sum)" /tmp/kind | sha256sum --check + chmod 0755 /tmp/kind + + - name: Create and delete kind cluster + shell: bash + run: | + set -euo pipefail + trap '/tmp/kind delete cluster --name smoke' EXIT + /tmp/kind create cluster --name smoke --wait 180s + /tmp/kind get clusters | grep -Fx smoke diff --git a/README.md b/README.md index e292367..0e2178b 100644 --- a/README.md +++ b/README.md @@ -20,7 +20,7 @@ runs-on: [self-hosted, vm] - `controller`:接收 Gitea `workflow_job` webhook,将指定 label 的 queued job 发布到 NATS JetStream。 - `worker`:领取任务、限制并发,并通过 Pod 或 microVM backend 创建一次性环境。 -- `microvm-runner-launch`:为每个任务创建 COW disk、NoCloud seed 和 TAP,运行 +- `microvm-runner-launch`:为每个任务以 direct I/O 转换出 flat qcow2 root disk、创建 NoCloud seed 和 TAP,运行 Cloud Hypervisor,退出后完整清理。 - `guest-runner`:在 guest 中领取一次性 runner registration token,注册 ephemeral runner,执行一个 job 后关机。 @@ -51,9 +51,16 @@ pytest - NATS 密码、webhook secret 和 Gitea registration token 只从文件读取。 - registration token 不写入 seed image;worker 通过单次 nonce endpoint 交给 guest。 +- guest 启动时从仅监听 microVM bridge 的 worker endpoint 获取固定版本 Runner 和配置 + 资产;基础镜像无需为 Runner 发布而重做。 +- `runner-vm-bootstrap.yaml` 暂时只验证 VM 调度和生命周期,不提供 SPIFFE + identity;VM agent attestation 完成前不得将它当作身份链路验证结果。 - guest runner 使用 `--ephemeral`,每台 VM 只执行一个 job。 - launcher 只接受 UUID instance ID 和 URL-safe nonce,所有临时文件都位于独立目录。 - base image 不得包含 runner identity、registration token、SSH 密码或 host key。 +- LXC 内运行 Cloud Hypervisor 必须为 guest memory 启用 `shared=on`。默认的 private + memfd 映射会在 guest 写入后同时产生 shmem 与 anonymous CoW charge,使 LXC cgroup + 对 guest RAM 接近双倍计费。 homelab 的 Kubernetes、OpenBao、LXC、bridge 和容量配置保留在 `panxiao81/homelab-infra`。 diff --git a/config/runner-vm-bootstrap.yaml b/config/runner-vm-bootstrap.yaml new file mode 100644 index 0000000..93333fe --- /dev/null +++ b/config/runner-vm-bootstrap.yaml @@ -0,0 +1,11 @@ +runner: + capacity: 1 + timeout: 3h + shutdown_timeout: 1m + +host: + workdir_parent: /workspace + +container: + require_docker: false + valid_volumes: [] diff --git a/scripts/gitea-microvm-guest-runner b/scripts/gitea-microvm-guest-runner index 7d8d7ba..041e33f 100755 --- a/scripts/gitea-microvm-guest-runner +++ b/scripts/gitea-microvm-guest-runner @@ -6,6 +6,8 @@ instance=${2:?Gitea instance is required} runner_name=${3:?runner name is required} runner_labels=${4:?runner labels are required} token_file=/run/gitea-runner-registration-token +config_file=${GITEA_RUNNER_CONFIG_FILE:-/etc/gitea-runner/config-vm-bootstrap.yaml} +export HOME=${HOME:-/root} cleanup() { rm -f -- "$token_file" @@ -25,4 +27,5 @@ gitea-runner register \ --labels "$runner_labels" \ --token-file "$token_file" rm -f -- "$token_file" -gitea-runner daemon +gitea-runner daemon --config "$config_file" --once +echo gitea-runner-job-complete >/dev/ttyS0 diff --git a/scripts/microvm-runner-launch b/scripts/microvm-runner-launch index b84d0ce..c45956f 100755 --- a/scripts/microvm-runner-launch +++ b/scripts/microvm-runner-launch @@ -29,8 +29,13 @@ tap="mvr${instance_id%%-*}" overlay="$vm_dir/root.qcow2" seed="$vm_dir/seed.img" serial="$vm_dir/serial.log" +log_dir="$state_root/logs" cleanup() { + if test -f "$serial"; then + install -d -m 0700 "$log_dir" + cp "$serial" "$log_dir/$instance_id.log" + fi ip link delete "$tap" 2>/dev/null || true rm -rf -- "$vm_dir" } @@ -39,7 +44,10 @@ trap cleanup EXIT INT TERM test -r "$base_image" test -r "$firmware" install -d -m 0700 "$state_root/instances" "$vm_dir" -qemu-img create -q -f qcow2 -F qcow2 -b "$base_image" "$overlay" +# Cloud Hypervisor cannot open qcow2 backing chains. Keep each disposable +# root disk flat and bypass the LXC page cache: caching both a 5 GiB copy and +# guest RAM can otherwise trigger the container memory limit. +qemu-img convert -q -T none -t none -f qcow2 -O qcow2 "$base_image" "$overlay" cat >"$vm_dir/meta-data" <"$vm_dir/user-data" < web.Response: return web.Response(body=value, headers={"Cache-Control": "no-store"}) +async def guest_assets(_: web.Request) -> web.FileResponse: + return web.FileResponse( + GUEST_ASSETS, + headers={"Cache-Control": "public, immutable"}, + ) + + async def heartbeat(message: object, stop: asyncio.Event) -> None: while True: try: @@ -162,6 +172,7 @@ async def consume() -> None: async def main() -> None: app = web.Application() app.router.add_get("/token/{nonce}", token) + app.router.add_get("/assets/guest-assets.tar.gz", guest_assets) runner = web.AppRunner(app) await runner.setup() await web.TCPSite(runner, TOKEN_LISTEN, TOKEN_PORT).start()