归档：Kata / microVM runner 实验（2026-09-17 工作区快照）

从旧工作区 chore/recover-old-workspace 清理时保存，内容与 2026-09-17
stash@{0} 快照中的版本一致；未合并、未在 main 上使用，仅作参考，不开 PR。
被 gitignore 的 tfstate 与凭据文件不在此分支，仍留在本地工作区。

Co-Authored-By: Claude Opus 5.5 <[email protected]>

infrastructure/microvm-runner/README.md

@@ -0,0 +1,53 @@
+# Gitea kind microVM runner
+
+`kind-microvm` 是专门运行 kind / nested Kubernetes CI 的 runner label。每个 job
+独占一台 Cloud Hypervisor VM；guest 内直接运行 Docker 与 ephemeral Gitea Runner，
+不经过 Kata。VM 完成一个 job 后关机，临时 COW 磁盘随即删除。
+
+集群内的 `controller.py` 消费 Gitea `workflow_job` webhook。仅当
+`action=queued` 且 `workflow_job.labels` 包含 `kind-microvm` 时，向 NATS JetStream
+的 `ci.runner.kind-microvm` subject 发布消息。`workflow_job.id` 写入
+`Nats-Msg-Id`，重复 webhook 不会生成第二个任务。
+
+pve2 LXC 内的 `worker.py` 使用 durable pull consumer 领取消息。领取后启动一台
+microVM，运行期间每分钟发送 `InProgress`；VM/ephemeral runner 正常退出才 ACK，
+启动失败则 NAK 并延迟重试。当前 `RUNNER_CAPACITY=1` 只是 pve2 的资源配置；以后可
+提高单 worker capacity，或在其他宿主机增加共享同一 durable consumer 的 worker，
+而无需修改 webhook/controller。相同 runner 类型必须共享 durable；WorkQueue stream
+不允许多个 consumer 使用重叠的 subject filter。
+
+worker 的凭据位于：
+
+```text
+/etc/microvm-runner/registration-token
+/etc/microvm-runner/nats-password
+```
+
+二者必须为 root-only 文件，不能写入镜像、cloud-init seed 或仓库。worker 为每次
+启动生成一次性 nonce；guest 只能从 TAP 网段请求一次 registration token，请求后
+立即从 worker 内存删除。runner 注册成功后删除 guest 内的 token 文件，再启动
+daemon。这样 job 无法从 seed disk 或 Docker inspect 取回可复用 registration token。
+
+NATS stream 使用 `WorkQueuePolicy`，ACK 后立即删除消息；同时限制为 24 小时、
+10000 条和 256 MiB，异常积压也不会无限增长。NATS 平台配置见
+`../../platform/nats/`。
+
+2026-09-16 PoC 已验证：Cloud Hypervisor v52.0、Ubuntu 24.04、4 vCPU、3 GiB RAM、
+10 GiB COW disk 中，Docker 29.1.3 成功创建 kind v0.27.0 / Kubernetes v1.32.2
+集群，并运行 `busybox:1.37` smoke Pod。冷启动约 20 秒。pve2 资源只允许一台：
+VMM cgroup（guest RAM 与 page cache）峰值约 6.2 GiB。
+
+正式启用前还需要安装 launcher、制作不含凭据的基础镜像、部署 controller，并在
+Gitea 配置只发送 `workflow_job` 的 instance/organization webhook。workflow 使用：
+
+```yaml
+runs-on: kind-microvm
+```
+
+当前 pve2 配置：
+
+```ini
+RUNNER_CAPACITY=1
+NATS_DURABLE=kind-microvm
+RUNNER_MAX_INFLIGHT=64
+```

infrastructure/microvm-runner/controller.py

@@ -0,0 +1,113 @@
+#!/usr/bin/env python3
+"""Gitea workflow_job webhook to NATS JetStream producer."""
+
+import hashlib
+import hmac
+import json
+import os
+import ssl
+from pathlib import Path
+
+import nats
+from aiohttp import web
+from nats.js.api import DiscardPolicy, RetentionPolicy, StorageType, StreamConfig
+from nats.js.errors import NotFoundError
+
+LABEL = os.environ.get("RUNNER_LABEL", "kind-microvm")
+SUBJECT = os.environ.get("NATS_SUBJECT", f"ci.runner.{LABEL}")
+STREAM = os.environ.get("NATS_STREAM", "CI_RUNNER")
+NATS_URL = os.environ.get("NATS_URL", "tls://nats.nats.svc.cluster.local:4222")
+NATS_USER = os.environ.get("NATS_USER", "ci-producer")
+NATS_PASSWORD_FILE = Path(os.environ.get("NATS_PASSWORD_FILE", "/run/secrets/nats/password"))
+NATS_CA_FILE = os.environ.get("NATS_CA_FILE", "/etc/ssl/certs/ca-certificates.crt")
+WEBHOOK_SECRET_FILE = Path(os.environ.get("WEBHOOK_SECRET_FILE", "/run/secrets/gitea/webhook-secret"))
+
+
+def accepts(payload: object) -> tuple[bool, str | None]:
+    if not isinstance(payload, dict) or payload.get("action") != "queued":
+        return False, None
+    job = payload.get("workflow_job")
+    if not isinstance(job, dict) or LABEL not in job.get("labels", []):
+        return False, None
+    job_id = job.get("id")
+    if not isinstance(job_id, int):
+        return False, None
+    return True, str(job_id)
+
+
+def valid_signature(body: bytes, signature: str) -> bool:
+    expected = hmac.new(WEBHOOK_SECRET_FILE.read_bytes().strip(), body, hashlib.sha256).hexdigest()
+    return hmac.compare_digest(signature, expected)
+
+
+async def ensure_stream(js: object) -> None:
+    config = StreamConfig(
+        name=STREAM,
+        subjects=["ci.runner.*"],
+        retention=RetentionPolicy.WORK_QUEUE,
+        storage=StorageType.FILE,
+        discard=DiscardPolicy.OLD,
+        max_age=24 * 60 * 60,
+        max_msgs=10_000,
+        max_bytes=256 * 1024 * 1024,
+        duplicate_window=24 * 60 * 60,
+    )
+    try:
+        await js.stream_info(STREAM)
+    except NotFoundError:
+        await js.add_stream(config=config)
+    else:
+        await js.update_stream(config=config)
+
+
+async def webhook(request: web.Request) -> web.Response:
+    body = await request.read()
+    if not valid_signature(body, request.headers.get("X-Gitea-Signature", "")):
+        raise web.HTTPUnauthorized()
+    try:
+        payload = json.loads(body)
+    except json.JSONDecodeError as error:
+        raise web.HTTPBadRequest(text="invalid JSON\n") from error
+    accepted, job_id = accepts(payload)
+    if not accepted:
+        return web.Response(status=204)
+    await request.app["js"].publish(
+        SUBJECT,
+        body,
+        headers={"Nats-Msg-Id": f"gitea-workflow-job-{job_id}"},
+    )
+    return web.Response(status=202, text="queued\n")
+
+
+async def health(request: web.Request) -> web.Response:
+    return web.Response(text="ok\n" if request.app["nc"].is_connected else "disconnected\n",
+                        status=200 if request.app["nc"].is_connected else 503)
+
+
+async def nats_context(app: web.Application):
+    tls = ssl.create_default_context(cafile=NATS_CA_FILE)
+    nc = await nats.connect(
+        NATS_URL,
+        user=NATS_USER,
+        password=NATS_PASSWORD_FILE.read_text().strip(),
+        tls=tls,
+        name="microvm-runner-controller",
+    )
+    app["nc"] = nc
+    app["js"] = nc.jetstream()
+    await ensure_stream(app["js"])
+    yield
+    await nc.drain()
+
+
+def create_app() -> web.Application:
+    app = web.Application(client_max_size=1024 * 1024)
+    app.cleanup_ctx.append(nats_context)
+    app.router.add_post("/webhook", webhook)
+    app.router.add_get("/healthz", health)
+    return app
+
+
+if __name__ == "__main__":
+    web.run_app(create_app(), host=os.environ.get("LISTEN", "0.0.0.0"),
+                port=int(os.environ.get("PORT", "8787")))

infrastructure/microvm-runner/microvm-runner-worker.service

@@ -0,0 +1,19 @@
+[Unit]
+Description=Gitea kind microVM runner worker
+After=network-online.target
+Wants=network-online.target
+
+[Service]
+Type=simple
+EnvironmentFile=-/etc/microvm-runner/worker.env
+ExecStart=/opt/microvm-runner/venv/bin/python /opt/microvm-runner/worker.py
+Restart=on-failure
+RestartSec=5s
+SupplementaryGroups=kvm
+PrivateTmp=yes
+ProtectHome=yes
+ProtectSystem=strict
+ReadWritePaths=/var/lib/microvm-runner /run/microvm-runner
+
+[Install]
+WantedBy=multi-user.target

infrastructure/microvm-runner/requirements.txt

@@ -0,0 +1,2 @@
+aiohttp==3.12.15
+nats-py==2.11.0

infrastructure/microvm-runner/worker.py

@@ -0,0 +1,138 @@
+#!/usr/bin/env python3
+"""Capacity-bounded JetStream consumer that launches one ephemeral VM per job."""
+
+import asyncio
+import os
+import secrets
+import ssl
+import uuid
+from pathlib import Path
+
+import nats
+from aiohttp import web
+from nats.errors import TimeoutError
+from nats.js.api import AckPolicy, ConsumerConfig
+
+CAPACITY = int(os.environ.get("RUNNER_CAPACITY", "1"))
+SUBJECT = os.environ.get("NATS_SUBJECT", "ci.runner.kind-microvm")
+STREAM = os.environ.get("NATS_STREAM", "CI_RUNNER")
+DURABLE = os.environ.get("NATS_DURABLE", "kind-microvm")
+MAX_INFLIGHT = int(os.environ.get("RUNNER_MAX_INFLIGHT", "64"))
+NATS_URL = os.environ.get("NATS_URL", "tls://nats.ad.ddupan.top:4222")
+NATS_USER = os.environ.get("NATS_USER", "ci-worker")
+NATS_PASSWORD_FILE = Path(os.environ.get("NATS_PASSWORD_FILE", "/etc/microvm-runner/nats-password"))
+NATS_CA_FILE = os.environ.get("NATS_CA_FILE", "/etc/ssl/certs/ca-certificates.crt")
+REGISTRATION_TOKEN_FILE = Path(os.environ.get("REGISTRATION_TOKEN_FILE", "/etc/microvm-runner/registration-token"))
+LAUNCHER = os.environ.get("LAUNCHER", "/usr/local/libexec/microvm-runner-launch")
+TOKEN_LISTEN = os.environ.get("TOKEN_LISTEN", "172.30.0.1")
+TOKEN_PORT = int(os.environ.get("TOKEN_PORT", "8787"))
+
+tokens: dict[str, bytes] = {}
+token_lock = asyncio.Lock()
+
+
+async def token(request: web.Request) -> web.Response:
+    nonce = request.match_info["nonce"]
+    async with token_lock:
+        value = tokens.pop(nonce, None)
+    if value is None:
+        raise web.HTTPNotFound()
+    return web.Response(body=value, headers={"Cache-Control": "no-store"})
+
+
+async def heartbeat(message: object, stop: asyncio.Event) -> None:
+    while True:
+        try:
+            await asyncio.wait_for(stop.wait(), timeout=60)
+            return
+        except asyncio.TimeoutError:
+            await message.in_progress()
+
+
+async def run_one(message: object) -> None:
+    instance_id = str(uuid.uuid4())
+    nonce = secrets.token_urlsafe(32)
+    async with token_lock:
+        tokens[nonce] = REGISTRATION_TOKEN_FILE.read_bytes().strip()
+    stop = asyncio.Event()
+    pulse = asyncio.create_task(heartbeat(message, stop))
+    try:
+        process = await asyncio.create_subprocess_exec(LAUNCHER, instance_id, nonce)
+        return_code = await process.wait()
+        if return_code == 0:
+            await message.ack()
+        else:
+            await message.nak(delay=30)
+    except Exception:
+        await message.nak(delay=30)
+        raise
+    finally:
+        stop.set()
+        await pulse
+        async with token_lock:
+            tokens.pop(nonce, None)
+
+
+async def consume() -> None:
+    if CAPACITY < 1:
+        raise ValueError("RUNNER_CAPACITY must be at least 1")
+    tls = ssl.create_default_context(cafile=NATS_CA_FILE)
+    nc = await nats.connect(
+        NATS_URL,
+        user=NATS_USER,
+        password=NATS_PASSWORD_FILE.read_text().strip(),
+        tls=tls,
+        name=DURABLE,
+    )
+    js = nc.jetstream()
+    subscription = await js.pull_subscribe(
+        SUBJECT,
+        durable=DURABLE,
+        stream=STREAM,
+        config=ConsumerConfig(
+            durable_name=DURABLE,
+            filter_subject=SUBJECT,
+            ack_policy=AckPolicy.EXPLICIT,
+            ack_wait=5 * 60,
+            # This durable consumer is shared by every host of this runner type.
+            # Local CAPACITY controls each host; this is only a global safety cap.
+            max_ack_pending=MAX_INFLIGHT,
+            max_deliver=5,
+        ),
+    )
+    active: set[asyncio.Task[None]] = set()
+    try:
+        while True:
+            active = {task for task in active if not task.done()}
+            free = CAPACITY - len(active)
+            if free == 0:
+                await asyncio.wait(active, return_when=asyncio.FIRST_COMPLETED)
+                continue
+            try:
+                messages = await subscription.fetch(batch=free, timeout=5)
+            except TimeoutError:
+                continue
+            for message in messages:
+                task = asyncio.create_task(run_one(message))
+                task.add_done_callback(lambda done: done.exception())
+                active.add(task)
+    finally:
+        if active:
+            await asyncio.gather(*active, return_exceptions=True)
+        await nc.drain()
+
+
+async def main() -> None:
+    app = web.Application()
+    app.router.add_get("/token/{nonce}", token)
+    runner = web.AppRunner(app)
+    await runner.setup()
+    await web.TCPSite(runner, TOKEN_LISTEN, TOKEN_PORT).start()
+    try:
+        await consume()
+    finally:
+        await runner.cleanup()
+
+
+if __name__ == "__main__":
+    asyncio.run(main())