停用镜像自动发布流水线

Reviewed-on: #17

.gitea/workflows/pod-smoke.yml

@@ -0,0 +1,19 @@
+name: dynamic Pod smoke test
+
+on:
+  workflow_dispatch:
+
+jobs:
+  pod-smoke:
+    name: pod-smoke
+    runs-on: [self-hosted, pod]
+    timeout-minutes: 5
+    steps:
+      - name: Verify disposable host environment
+        run: |
+          test -S /run/spire/agent-sockets/spire-agent.sock
+          /opt/spire/bin/spire-agent api fetch jwt \
+            -audience ci-smoke \
+            -socketPath /run/spire/agent-sockets/spire-agent.sock \
+            >/dev/null
+          test "$(id -u)" = 0

.gitea/workflows/vm-kind-smoke.yml

@@ -0,0 +1,31 @@
+name: VM kind smoke
+
+on:
+  workflow_dispatch:
+
+jobs:
+  kind:
+    runs-on: [self-hosted, vm]
+    steps:
+      - name: Verify Docker
+        run: docker info
+
+      - name: Install kind
+        shell: bash
+        run: |
+          set -euo pipefail
+          version=v0.33.0
+          curl --fail --location --silent --show-error \
+            --output /tmp/kind "https://kind.sigs.k8s.io/dl/${version}/kind-linux-amd64"
+          curl --fail --location --silent --show-error \
+            --output /tmp/kind.sha256sum "https://kind.sigs.k8s.io/dl/${version}/kind-linux-amd64.sha256sum"
+          printf '%s  %s\n' "$(cut -d ' ' -f1 /tmp/kind.sha256sum)" /tmp/kind | sha256sum --check
+          chmod 0755 /tmp/kind
+
+      - name: Create and delete kind cluster
+        shell: bash
+        run: |
+          set -euo pipefail
+          trap '/tmp/kind delete cluster --name smoke' EXIT
+          /tmp/kind create cluster --name smoke --wait 180s
+          /tmp/kind get clusters | grep -Fx smoke

.github/actionlint.yaml

@@ -0,0 +1,5 @@
+---
+self-hosted-runner:
+  labels:
+    - pod
+    - vm

README.md

@@ -20,7 +20,7 @@ runs-on: [self-hosted, vm]
 - `controller`：接收 Gitea `workflow_job` webhook，将指定 label 的 queued job
   发布到 NATS JetStream。
 - `worker`：领取任务、限制并发，并通过 Pod 或 microVM backend 创建一次性环境。
-- `microvm-runner-launch`：为每个任务创建 COW disk、NoCloud seed 和 TAP，运行
+- `microvm-runner-launch`：为每个任务以 direct I/O 转换出 flat qcow2 root disk、创建 NoCloud seed 和 TAP，运行
   Cloud Hypervisor，退出后完整清理。
 - `guest-runner`：在 guest 中领取一次性 runner registration token，注册 ephemeral
   runner，执行一个 job 后关机。
@@ -51,9 +51,16 @@ pytest
 
 - NATS 密码、webhook secret 和 Gitea registration token 只从文件读取。
 - registration token 不写入 seed image；worker 通过单次 nonce endpoint 交给 guest。
+- guest 启动时从仅监听 microVM bridge 的 worker endpoint 获取固定版本 Runner 和配置
+  资产；基础镜像无需为 Runner 发布而重做。
+- `runner-vm-bootstrap.yaml` 暂时只验证 VM 调度和生命周期，不提供 SPIFFE
+  identity；VM agent attestation 完成前不得将它当作身份链路验证结果。
 - guest runner 使用 `--ephemeral`，每台 VM 只执行一个 job。
 - launcher 只接受 UUID instance ID 和 URL-safe nonce，所有临时文件都位于独立目录。
 - base image 不得包含 runner identity、registration token、SSH 密码或 host key。
+- LXC 内运行 Cloud Hypervisor 必须为 guest memory 启用 `shared=on`。默认的 private
+  memfd 映射会在 guest 写入后同时产生 shmem 与 anonymous CoW charge，使 LXC cgroup
+  对 guest RAM 接近双倍计费。
 
 homelab 的 Kubernetes、OpenBao、LXC、bridge 和容量配置保留在
 `panxiao81/homelab-infra`。

config/runner-vm-bootstrap.yaml

@@ -0,0 +1,11 @@
+runner:
+  capacity: 1
+  timeout: 3h
+  shutdown_timeout: 1m
+
+host:
+  workdir_parent: /workspace
+
+container:
+  require_docker: false
+  valid_volumes: []

container/runner.Dockerfile

@@ -1,8 +1,20 @@
 FROM ghcr.io/spiffe/spire-agent:1.15.3@sha256:41b0dcd8b258a69db9e2768292a060766fb76fd866e4bc925849981ea1b825ff AS spire
 
-FROM docker.io/gitea/runner:3.5.0@sha256:66b7da94dc7dcadb2e076bec6928221336a9a637196399281c4b766fe1288242
+FROM docker.io/gitea/runner:3.5.0@sha256:66b7da94dc7dcadb2e076bec6928221336a9a637196399281c4b766fe1288242 AS runner
+
+# The runner daemon image is intentionally minimal and does not contain the
+# Node.js runtime required by JavaScript actions such as actions/checkout.
+# Run the daemon in Gitea's Ubuntu workflow image so host-mode jobs and their
+# actions share a GitHub Actions-compatible userspace.
+FROM docker.io/gitea/runner-images:ubuntu-latest@sha256:fd911d7417bfbf0f454530e447da95b58001e1df41bbc5e1a8dd35d432575aae
 USER root
+
+COPY --from=runner /usr/local/bin/gitea-runner /usr/local/bin/gitea-runner
+COPY --from=runner /usr/local/bin/run.sh /usr/local/bin/run.sh
 COPY --from=spire /opt/spire/bin/spire-agent /opt/spire/bin/spire-agent
 COPY config/runner.yaml /etc/gitea-runner/config.yaml
-COPY scripts/gitea-job-started /usr/local/libexec/gitea-job-started
-RUN chmod 0755 /usr/local/libexec/gitea-job-started
+COPY --chmod=0755 scripts/gitea-job-started /usr/local/libexec/gitea-job-started
+
+VOLUME ["/data"]
+WORKDIR /
+ENTRYPOINT ["/usr/local/bin/run.sh"]

scripts/gitea-microvm-guest-runner

@@ -6,6 +6,8 @@ instance=${2:?Gitea instance is required}
 runner_name=${3:?runner name is required}
 runner_labels=${4:?runner labels are required}
 token_file=/run/gitea-runner-registration-token
+config_file=${GITEA_RUNNER_CONFIG_FILE:-/etc/gitea-runner/config-vm-bootstrap.yaml}
+export HOME="${HOME:-/root}"
 
 cleanup() {
     rm -f -- "$token_file"
@@ -25,4 +27,5 @@ gitea-runner register \
     --labels "$runner_labels" \
     --token-file "$token_file"
 rm -f -- "$token_file"
-gitea-runner daemon
+gitea-runner daemon --config "$config_file" --once
+echo gitea-runner-job-complete >/dev/ttyS0

scripts/microvm-runner-launch

@@ -29,8 +29,13 @@ tap="mvr${instance_id%%-*}"
 overlay="$vm_dir/root.qcow2"
 seed="$vm_dir/seed.img"
 serial="$vm_dir/serial.log"
+log_dir="$state_root/logs"
 
 cleanup() {
+    if test -f "$serial"; then
+        install -d -m 0700 "$log_dir"
+        cp "$serial" "$log_dir/$instance_id.log"
+    fi
     ip link delete "$tap" 2>/dev/null || true
     rm -rf -- "$vm_dir"
 }
@@ -39,7 +44,10 @@ trap cleanup EXIT INT TERM
 test -r "$base_image"
 test -r "$firmware"
 install -d -m 0700 "$state_root/instances" "$vm_dir"
-qemu-img create -q -f qcow2 -F qcow2 -b "$base_image" "$overlay"
+# Cloud Hypervisor cannot open qcow2 backing chains.  Keep each disposable
+# root disk flat and bypass the LXC page cache: caching both a 5 GiB copy and
+# guest RAM can otherwise trigger the container memory limit.
+qemu-img convert -q -T none -t none -f qcow2 -O qcow2 "$base_image" "$overlay"
 
 cat >"$vm_dir/meta-data" <<EOF
 instance-id: $instance_id
@@ -48,7 +56,8 @@ EOF
 cat >"$vm_dir/user-data" <<EOF
 #cloud-config
 runcmd:
-  - [ /usr/local/libexec/gitea-microvm-guest-runner, "$token_url/token/$nonce", "$gitea_instance", "gitea-${instance_id%%-*}", "$runner_labels" ]
+  - [ sh, -c, "curl --fail --silent --show-error --retry 10 --retry-all-errors $token_url/assets/guest-assets.tar.gz | tar -xz -C /" ]
+  - [ /usr/local/libexec/gitea-microvm-guest-runner, "$token_url/token/$nonce", "$gitea_instance", "gitea-vm-${instance_id%%-*}", "$runner_labels" ]
 EOF
 cloud-localds "$seed" "$vm_dir/user-data" "$vm_dir/meta-data"
 
@@ -63,12 +72,16 @@ ip tuntap add dev "$tap" mode tap
 ip link set "$tap" master "$bridge"
 ip link set "$tap" up
 
-timeout --signal=TERM --kill-after=30s "$vm_timeout" "$cloud_hypervisor" \
-    --firmware "$firmware" \
-    --cpus "boot=$cpus" \
-    --memory "size=$memory" \
-    --disk "path=$overlay" \
-    --disk "path=$seed,readonly=on" \
-    --net "tap=$tap,mac=$mac" \
-    --serial "file=$serial" \
-    --console off
+if timeout --signal=TERM --kill-after=30s "$vm_timeout" "$cloud_hypervisor" \
+        --firmware "$firmware" \
+        --cpus "boot=$cpus" \
+        --memory "size=$memory,shared=on" \
+        --disk "path=$overlay,image_type=qcow2,direct=on,sparse=off" \
+        --disk "path=$seed,readonly=on,image_type=raw,direct=on,sparse=off" \
+        --net "tap=$tap,mac=$mac" \
+        --serial "file=$serial" \
+        --console off; then
+    grep -Fq gitea-runner-job-complete "$serial"
+else
+    exit $?
+fi

src/gitea_dynamic_runner/controller.py

@@ -4,6 +4,7 @@
 import hashlib
 import hmac
 import json
+import logging
 import os
 import ssl
 from pathlib import Path
@@ -16,6 +17,7 @@ from nats.js.errors import NotFoundError
 from .models import IdentityBinding, RunnerRequest
 
 
+LOG = logging.getLogger(__name__)
 SUBJECT_PREFIX = os.environ.get("NATS_SUBJECT_PREFIX", "ci.runner")
 STREAM = os.environ.get("NATS_STREAM", "CI_RUNNER")
 NATS_URL = os.environ.get("NATS_URL", "tls://nats.ad.ddupan.top:4222")
@@ -64,27 +66,70 @@ async def webhook(request: web.Request) -> web.Response:
         payload = json.loads(body)
     except json.JSONDecodeError as error:
         raise web.HTTPBadRequest(text="invalid JSON\n") from error
+    context = _webhook_context(payload)
+    LOG.info("workflow_job webhook received %s", context)
+
     runner_request = RunnerRequest.from_webhook(payload)
     if runner_request is not None:
+        subject = f"{SUBJECT_PREFIX}.{runner_request.backend}"
         await request.app["js"].publish(
-            f"{SUBJECT_PREFIX}.{runner_request.backend}",
+            subject,
             runner_request.to_json(),
             headers={"Nats-Msg-Id": f"gitea-workflow-job-{runner_request.job_id}-queued"},
         )
+        LOG.info(
+            "runner request published subject=%s job_id=%d run_id=%d backend=%s "
+            "repository=%s job_name=%r labels=%s",
+            subject,
+            runner_request.job_id,
+            runner_request.run_id,
+            runner_request.backend,
+            runner_request.repository,
+            runner_request.job_name,
+            runner_request.labels,
+        )
         return web.Response(status=202, text="queued\n")
 
     binding = IdentityBinding.from_webhook(payload)
     if binding is not None:
+        subject = f"{SUBJECT_PREFIX}.{binding.backend}.binding"
         await request.app["js"].publish(
-            f"{SUBJECT_PREFIX}.{binding.backend}.binding",
+            subject,
             binding.to_json(),
             headers={"Nats-Msg-Id": f"gitea-workflow-job-{binding.job_id}-in-progress"},
         )
+        LOG.info(
+            "identity binding published subject=%s job_id=%d run_id=%d backend=%s "
+            "runner_name=%s repository=%s job_name=%r",
+            subject,
+            binding.job_id,
+            binding.run_id,
+            binding.backend,
+            binding.runner_name,
+            binding.repository,
+            binding.job_name,
+        )
         return web.Response(status=202, text="binding queued\n")
 
+    LOG.warning("workflow_job webhook ignored %s", context)
     return web.Response(status=204)
 
 
+def _webhook_context(payload: object) -> str:
+    if not isinstance(payload, dict):
+        return f"payload_type={type(payload).__name__}"
+    job = payload.get("workflow_job")
+    repository = payload.get("repository")
+    job = job if isinstance(job, dict) else {}
+    repository = repository if isinstance(repository, dict) else {}
+    return (
+        f"action={payload.get('action')!r} job_id={job.get('id')!r} "
+        f"run_id={job.get('run_id')!r} runner_name={job.get('runner_name')!r} "
+        f"repository={repository.get('full_name')!r} job_name={job.get('name')!r} "
+        f"labels={job.get('labels')!r}"
+    )
+
+
 async def health(request: web.Request) -> web.Response:
     connected = request.app["nc"].is_connected
     return web.Response(text="ok\n" if connected else "disconnected\n", status=200 if connected else 503)
@@ -115,6 +160,7 @@ def create_app() -> web.Application:
 
 
 def main() -> None:
+    logging.basicConfig(level=os.environ.get("LOG_LEVEL", "INFO"))
     web.run_app(create_app(), host=os.environ.get("LISTEN", "0.0.0.0"), port=int(os.environ.get("PORT", "8787")))
 
 

src/gitea_dynamic_runner/pod_worker.py

@@ -4,12 +4,13 @@
 from __future__ import annotations
 
 import asyncio
+import hashlib
 import json
 import logging
 import os
 from pathlib import Path
+import re
 import ssl
-from urllib.parse import quote
 import uuid
 
 import nats
@@ -49,16 +50,26 @@ RUNNER_TOKEN_SECRET = os.environ.get("RUNNER_TOKEN_SECRET", "gitea-dynamic-runne
 GITEA_INSTANCE = os.environ.get("GITEA_INSTANCE", "https://git.ddupan.top")
 CAPACITY = int(os.environ.get("RUNNER_CAPACITY", "4"))
 POD_TIMEOUT = int(os.environ.get("RUNNER_POD_TIMEOUT", str(4 * 60 * 60)))
+SPIFFE_PATH_SEGMENT = re.compile(r"^[A-Za-z0-9._-]+$")
 
 
 def identity_path(repository: str, job_name: str) -> str:
     parts = repository.split("/")
     if len(parts) != 2 or not all(parts):
         raise ValueError("repository must be owner/name")
-    encoded = [quote(part, safe="-._~") for part in (*parts, job_name)]
-    if not job_name.strip() or any(not part for part in encoded):
+    if not job_name.strip():
         raise ValueError("identity components must not be empty")
-    return "/".join(encoded)
+    return "/".join(safe_identity_segment(part) for part in (*parts, job_name))
+
+
+def safe_identity_segment(value: str) -> str:
+    """Map arbitrary Gitea names to stable SPIFFE Operator path segments."""
+    if SPIFFE_PATH_SEGMENT.fullmatch(value):
+        return value
+    slug = re.sub(r"[^A-Za-z0-9._-]+", "-", value).strip("-._")
+    slug = slug[:48].rstrip("-._") or "segment"
+    digest = hashlib.sha256(value.encode()).hexdigest()[:12]
+    return f"{slug}-{digest}"
 
 
 def pod_manifest(request: RunnerRequest, pod_name: str) -> dict[str, object]:
@@ -174,36 +185,90 @@ async def run_request(message: object, client: KubernetesClient) -> None:
         return
 
     pod_name = f"gitea-pod-{uuid.uuid4().hex[:12]}"
+    LOG.info(
+        "runner request received job_id=%d run_id=%d repository=%s job_name=%r "
+        "backend=%s pod=%s",
+        request.job_id,
+        request.run_id,
+        request.repository,
+        request.job_name,
+        request.backend,
+        pod_name,
+    )
     stop = asyncio.Event()
     pulse = asyncio.create_task(heartbeat(message, stop))
     try:
         await client.create_pod(pod_manifest(request, pod_name))
+        LOG.info(
+            "runner Pod created pod=%s job_id=%d run_id=%d image=%s",
+            pod_name,
+            request.job_id,
+            request.run_id,
+            RUNNER_IMAGE,
+        )
         if await wait_for_pod(client, pod_name):
             await message.ack()
+            LOG.info(
+                "runner request acknowledged pod=%s job_id=%d result=succeeded",
+                pod_name,
+                request.job_id,
+            )
         else:
-            LOG.error("runner Pod %s failed", pod_name)
+            LOG.error(
+                "runner Pod failed pod=%s job_id=%d; request will be retried",
+                pod_name,
+                request.job_id,
+            )
             await message.nak(delay=30)
     except Exception:
+        LOG.exception(
+            "runner request failed pod=%s job_id=%d; request will be retried",
+            pod_name,
+            request.job_id,
+        )
         await message.nak(delay=30)
         raise
     finally:
         stop.set()
         await pulse
+        LOG.info("deleting runner Pod pod=%s job_id=%d", pod_name, request.job_id)
         await client.delete_pod(pod_name)
 
 
 async def bind_request(message: object, client: KubernetesClient) -> None:
+    binding: IdentityBinding | None = None
     try:
         document = json.loads(message.data)
         binding = IdentityBinding(**document)
         if binding.backend != "pod" or not binding.runner_name.startswith("gitea-pod-"):
             raise ValueError("invalid Pod identity binding")
         path = identity_path(binding.repository, binding.job_name)
+        LOG.info(
+            "identity binding received job_id=%d run_id=%d runner_name=%s "
+            "repository=%s job_name=%r identity_path=%s",
+            binding.job_id,
+            binding.run_id,
+            binding.runner_name,
+            binding.repository,
+            binding.job_name,
+            path,
+        )
         await client.bind_identity(binding.runner_name, path)
     except ClientResponseError as error:
         if error.status == 404:
+            LOG.warning(
+                "identity binding Pod not found runner_name=%s job_id=%s; binding will be retried",
+                binding.runner_name if binding else None,
+                binding.job_id if binding else None,
+            )
             await message.nak(delay=2)
             return
+        LOG.exception(
+            "identity binding Kubernetes request failed runner_name=%s job_id=%s status=%d",
+            binding.runner_name if binding else None,
+            binding.job_id if binding else None,
+            error.status,
+        )
         await message.nak(delay=30)
         raise
     except (json.JSONDecodeError, TypeError, ValueError) as error:
@@ -211,6 +276,12 @@ async def bind_request(message: object, client: KubernetesClient) -> None:
         await message.ack()
         return
     await message.ack()
+    LOG.info(
+        "identity binding applied and acknowledged runner_name=%s job_id=%d identity_path=%s",
+        binding.runner_name,
+        binding.job_id,
+        path,
+    )
 
 
 async def consume_requests(subscription: object, client: KubernetesClient) -> None:

src/gitea_dynamic_runner/worker.py

@@ -31,6 +31,9 @@ REGISTRATION_TOKEN_FILE = Path(os.environ.get("REGISTRATION_TOKEN_FILE", "/etc/m
 LAUNCHER = os.environ.get("LAUNCHER", "/usr/local/libexec/microvm-runner-launch")
 TOKEN_LISTEN = os.environ.get("TOKEN_LISTEN", "172.30.0.1")
 TOKEN_PORT = int(os.environ.get("TOKEN_PORT", "8787"))
+GUEST_ASSETS = Path(
+    os.environ.get("GUEST_ASSETS", "/opt/gitea-dynamic-runner/guest-assets.tar.gz")
+)
 
 tokens: dict[str, bytes] = {}
 token_lock = asyncio.Lock()
@@ -45,6 +48,13 @@ async def token(request: web.Request) -> web.Response:
     return web.Response(body=value, headers={"Cache-Control": "no-store"})
 
 
+async def guest_assets(_: web.Request) -> web.FileResponse:
+    return web.FileResponse(
+        GUEST_ASSETS,
+        headers={"Cache-Control": "public, immutable"},
+    )
+
+
 async def heartbeat(message: object, stop: asyncio.Event) -> None:
     while True:
         try:
@@ -162,6 +172,7 @@ async def consume() -> None:
 async def main() -> None:
     app = web.Application()
     app.router.add_get("/token/{nonce}", token)
+    app.router.add_get("/assets/guest-assets.tar.gz", guest_assets)
     runner = web.AppRunner(app)
     await runner.setup()
     await web.TCPSite(runner, TOKEN_LISTEN, TOKEN_PORT).start()

tests/test_pod_worker.py

@@ -16,8 +16,22 @@ def request() -> RunnerRequest:
 
 
 def test_identity_path_is_meaningful_and_uri_safe():
-    assert pod_worker.identity_path("panxiao81/example", "publish image") == (
-        "panxiao81/example/publish%20image"
+    path = pod_worker.identity_path("panxiao81/example", "publish image")
+    assert path.startswith("panxiao81/example/publish-image-")
+    assert "%" not in path
+    assert all(pod_worker.SPIFFE_PATH_SEGMENT.fullmatch(part) for part in path.split("/"))
+    assert pod_worker.identity_path("panxiao81/example", "lint") == "panxiao81/example/lint"
+    assert path == pod_worker.identity_path("panxiao81/example", "publish image")
+    assert path != pod_worker.identity_path("panxiao81/example", "publish-image")
+    real_path = pod_worker.identity_path(
+        "panxiao81/postgresql-tenant-operator", "Run on Ubuntu"
+    )
+    assert real_path.startswith(
+        "panxiao81/postgresql-tenant-operator/Run-on-Ubuntu-"
+    )
+    assert all(
+        pod_worker.SPIFFE_PATH_SEGMENT.fullmatch(part)
+        for part in real_path.split("/")
     )
     with pytest.raises(ValueError):
         pod_worker.identity_path("invalid", "test")