修复 LXC 内 microVM Runner 启动链路
This commit is contained in:
@@ -20,7 +20,7 @@ runs-on: [self-hosted, vm]
|
||||
- `controller`:接收 Gitea `workflow_job` webhook,将指定 label 的 queued job
|
||||
发布到 NATS JetStream。
|
||||
- `worker`:领取任务、限制并发,并通过 Pod 或 microVM backend 创建一次性环境。
|
||||
- `microvm-runner-launch`:为每个任务创建 COW disk、NoCloud seed 和 TAP,运行
|
||||
- `microvm-runner-launch`:为每个任务复制 flat qcow2 root disk(支持时使用 reflink)、创建 NoCloud seed 和 TAP,运行
|
||||
Cloud Hypervisor,退出后完整清理。
|
||||
- `guest-runner`:在 guest 中领取一次性 runner registration token,注册 ephemeral
|
||||
runner,执行一个 job 后关机。
|
||||
@@ -51,6 +51,10 @@ pytest
|
||||
|
||||
- NATS 密码、webhook secret 和 Gitea registration token 只从文件读取。
|
||||
- registration token 不写入 seed image;worker 通过单次 nonce endpoint 交给 guest。
|
||||
- guest 启动时从仅监听 microVM bridge 的 worker endpoint 获取固定版本 Runner 和配置
|
||||
资产;基础镜像无需为 Runner 发布而重做。
|
||||
- `runner-vm-bootstrap.yaml` 暂时只验证 VM 调度和生命周期,不提供 SPIFFE
|
||||
identity;VM agent attestation 完成前不得将它当作身份链路验证结果。
|
||||
- guest runner 使用 `--ephemeral`,每台 VM 只执行一个 job。
|
||||
- launcher 只接受 UUID instance ID 和 URL-safe nonce,所有临时文件都位于独立目录。
|
||||
- base image 不得包含 runner identity、registration token、SSH 密码或 host key。
|
||||
|
||||
@@ -0,0 +1,11 @@
|
||||
runner:
|
||||
capacity: 1
|
||||
timeout: 3h
|
||||
shutdown_timeout: 1m
|
||||
|
||||
host:
|
||||
workdir_parent: /workspace
|
||||
|
||||
container:
|
||||
require_docker: false
|
||||
valid_volumes: []
|
||||
@@ -6,6 +6,7 @@ instance=${2:?Gitea instance is required}
|
||||
runner_name=${3:?runner name is required}
|
||||
runner_labels=${4:?runner labels are required}
|
||||
token_file=/run/gitea-runner-registration-token
|
||||
config_file=${GITEA_RUNNER_CONFIG_FILE:-/etc/gitea-runner/config-vm-bootstrap.yaml}
|
||||
|
||||
cleanup() {
|
||||
rm -f -- "$token_file"
|
||||
@@ -25,4 +26,4 @@ gitea-runner register \
|
||||
--labels "$runner_labels" \
|
||||
--token-file "$token_file"
|
||||
rm -f -- "$token_file"
|
||||
gitea-runner daemon
|
||||
gitea-runner daemon --config "$config_file" --once
|
||||
|
||||
@@ -39,7 +39,9 @@ trap cleanup EXIT INT TERM
|
||||
test -r "$base_image"
|
||||
test -r "$firmware"
|
||||
install -d -m 0700 "$state_root/instances" "$vm_dir"
|
||||
qemu-img create -q -f qcow2 -F qcow2 -b "$base_image" "$overlay"
|
||||
# Cloud Hypervisor cannot open qcow2 backing chains. Keep each disposable
|
||||
# root disk flat; reflink-capable storage still makes this copy cheap.
|
||||
cp --reflink=auto --sparse=always "$base_image" "$overlay"
|
||||
|
||||
cat >"$vm_dir/meta-data" <<EOF
|
||||
instance-id: $instance_id
|
||||
@@ -48,7 +50,8 @@ EOF
|
||||
cat >"$vm_dir/user-data" <<EOF
|
||||
#cloud-config
|
||||
runcmd:
|
||||
- [ /usr/local/libexec/gitea-microvm-guest-runner, "$token_url/token/$nonce", "$gitea_instance", "gitea-${instance_id%%-*}", "$runner_labels" ]
|
||||
- [ sh, -c, "curl --fail --silent --show-error --retry 10 --retry-all-errors $token_url/assets/guest-assets.tar.gz | tar -xz -C /" ]
|
||||
- [ /usr/local/libexec/gitea-microvm-guest-runner, "$token_url/token/$nonce", "$gitea_instance", "gitea-vm-${instance_id%%-*}", "$runner_labels" ]
|
||||
EOF
|
||||
cloud-localds "$seed" "$vm_dir/user-data" "$vm_dir/meta-data"
|
||||
|
||||
@@ -67,8 +70,8 @@ timeout --signal=TERM --kill-after=30s "$vm_timeout" "$cloud_hypervisor" \
|
||||
--firmware "$firmware" \
|
||||
--cpus "boot=$cpus" \
|
||||
--memory "size=$memory" \
|
||||
--disk "path=$overlay" \
|
||||
--disk "path=$seed,readonly=on" \
|
||||
--disk "path=$overlay,image_type=qcow2" \
|
||||
--disk "path=$seed,readonly=on,image_type=raw" \
|
||||
--net "tap=$tap,mac=$mac" \
|
||||
--serial "file=$serial" \
|
||||
--console off
|
||||
|
||||
@@ -31,6 +31,9 @@ REGISTRATION_TOKEN_FILE = Path(os.environ.get("REGISTRATION_TOKEN_FILE", "/etc/m
|
||||
LAUNCHER = os.environ.get("LAUNCHER", "/usr/local/libexec/microvm-runner-launch")
|
||||
TOKEN_LISTEN = os.environ.get("TOKEN_LISTEN", "172.30.0.1")
|
||||
TOKEN_PORT = int(os.environ.get("TOKEN_PORT", "8787"))
|
||||
GUEST_ASSETS = Path(
|
||||
os.environ.get("GUEST_ASSETS", "/opt/gitea-dynamic-runner/guest-assets.tar.gz")
|
||||
)
|
||||
|
||||
tokens: dict[str, bytes] = {}
|
||||
token_lock = asyncio.Lock()
|
||||
@@ -45,6 +48,13 @@ async def token(request: web.Request) -> web.Response:
|
||||
return web.Response(body=value, headers={"Cache-Control": "no-store"})
|
||||
|
||||
|
||||
async def guest_assets(_: web.Request) -> web.FileResponse:
|
||||
return web.FileResponse(
|
||||
GUEST_ASSETS,
|
||||
headers={"Cache-Control": "public, immutable"},
|
||||
)
|
||||
|
||||
|
||||
async def heartbeat(message: object, stop: asyncio.Event) -> None:
|
||||
while True:
|
||||
try:
|
||||
@@ -162,6 +172,7 @@ async def consume() -> None:
|
||||
async def main() -> None:
|
||||
app = web.Application()
|
||||
app.router.add_get("/token/{nonce}", token)
|
||||
app.router.add_get("/assets/guest-assets.tar.gz", guest_assets)
|
||||
runner = web.AppRunner(app)
|
||||
await runner.setup()
|
||||
await web.TCPSite(runner, TOKEN_LISTEN, TOKEN_PORT).start()
|
||||
|
||||
Reference in New Issue
Block a user