修复 LXC 内 microVM Runner 启动链路
This commit is contained in:
@@ -20,7 +20,7 @@ runs-on: [self-hosted, vm]
|
|||||||
- `controller`:接收 Gitea `workflow_job` webhook,将指定 label 的 queued job
|
- `controller`:接收 Gitea `workflow_job` webhook,将指定 label 的 queued job
|
||||||
发布到 NATS JetStream。
|
发布到 NATS JetStream。
|
||||||
- `worker`:领取任务、限制并发,并通过 Pod 或 microVM backend 创建一次性环境。
|
- `worker`:领取任务、限制并发,并通过 Pod 或 microVM backend 创建一次性环境。
|
||||||
- `microvm-runner-launch`:为每个任务创建 COW disk、NoCloud seed 和 TAP,运行
|
- `microvm-runner-launch`:为每个任务复制 flat qcow2 root disk(支持时使用 reflink)、创建 NoCloud seed 和 TAP,运行
|
||||||
Cloud Hypervisor,退出后完整清理。
|
Cloud Hypervisor,退出后完整清理。
|
||||||
- `guest-runner`:在 guest 中领取一次性 runner registration token,注册 ephemeral
|
- `guest-runner`:在 guest 中领取一次性 runner registration token,注册 ephemeral
|
||||||
runner,执行一个 job 后关机。
|
runner,执行一个 job 后关机。
|
||||||
@@ -51,6 +51,10 @@ pytest
|
|||||||
|
|
||||||
- NATS 密码、webhook secret 和 Gitea registration token 只从文件读取。
|
- NATS 密码、webhook secret 和 Gitea registration token 只从文件读取。
|
||||||
- registration token 不写入 seed image;worker 通过单次 nonce endpoint 交给 guest。
|
- registration token 不写入 seed image;worker 通过单次 nonce endpoint 交给 guest。
|
||||||
|
- guest 启动时从仅监听 microVM bridge 的 worker endpoint 获取固定版本 Runner 和配置
|
||||||
|
资产;基础镜像无需为 Runner 发布而重做。
|
||||||
|
- `runner-vm-bootstrap.yaml` 暂时只验证 VM 调度和生命周期,不提供 SPIFFE
|
||||||
|
identity;VM agent attestation 完成前不得将它当作身份链路验证结果。
|
||||||
- guest runner 使用 `--ephemeral`,每台 VM 只执行一个 job。
|
- guest runner 使用 `--ephemeral`,每台 VM 只执行一个 job。
|
||||||
- launcher 只接受 UUID instance ID 和 URL-safe nonce,所有临时文件都位于独立目录。
|
- launcher 只接受 UUID instance ID 和 URL-safe nonce,所有临时文件都位于独立目录。
|
||||||
- base image 不得包含 runner identity、registration token、SSH 密码或 host key。
|
- base image 不得包含 runner identity、registration token、SSH 密码或 host key。
|
||||||
|
|||||||
@@ -0,0 +1,11 @@
|
|||||||
|
runner:
|
||||||
|
capacity: 1
|
||||||
|
timeout: 3h
|
||||||
|
shutdown_timeout: 1m
|
||||||
|
|
||||||
|
host:
|
||||||
|
workdir_parent: /workspace
|
||||||
|
|
||||||
|
container:
|
||||||
|
require_docker: false
|
||||||
|
valid_volumes: []
|
||||||
@@ -6,6 +6,7 @@ instance=${2:?Gitea instance is required}
|
|||||||
runner_name=${3:?runner name is required}
|
runner_name=${3:?runner name is required}
|
||||||
runner_labels=${4:?runner labels are required}
|
runner_labels=${4:?runner labels are required}
|
||||||
token_file=/run/gitea-runner-registration-token
|
token_file=/run/gitea-runner-registration-token
|
||||||
|
config_file=${GITEA_RUNNER_CONFIG_FILE:-/etc/gitea-runner/config-vm-bootstrap.yaml}
|
||||||
|
|
||||||
cleanup() {
|
cleanup() {
|
||||||
rm -f -- "$token_file"
|
rm -f -- "$token_file"
|
||||||
@@ -25,4 +26,4 @@ gitea-runner register \
|
|||||||
--labels "$runner_labels" \
|
--labels "$runner_labels" \
|
||||||
--token-file "$token_file"
|
--token-file "$token_file"
|
||||||
rm -f -- "$token_file"
|
rm -f -- "$token_file"
|
||||||
gitea-runner daemon
|
gitea-runner daemon --config "$config_file" --once
|
||||||
|
|||||||
@@ -39,7 +39,9 @@ trap cleanup EXIT INT TERM
|
|||||||
test -r "$base_image"
|
test -r "$base_image"
|
||||||
test -r "$firmware"
|
test -r "$firmware"
|
||||||
install -d -m 0700 "$state_root/instances" "$vm_dir"
|
install -d -m 0700 "$state_root/instances" "$vm_dir"
|
||||||
qemu-img create -q -f qcow2 -F qcow2 -b "$base_image" "$overlay"
|
# Cloud Hypervisor cannot open qcow2 backing chains. Keep each disposable
|
||||||
|
# root disk flat; reflink-capable storage still makes this copy cheap.
|
||||||
|
cp --reflink=auto --sparse=always "$base_image" "$overlay"
|
||||||
|
|
||||||
cat >"$vm_dir/meta-data" <<EOF
|
cat >"$vm_dir/meta-data" <<EOF
|
||||||
instance-id: $instance_id
|
instance-id: $instance_id
|
||||||
@@ -48,7 +50,8 @@ EOF
|
|||||||
cat >"$vm_dir/user-data" <<EOF
|
cat >"$vm_dir/user-data" <<EOF
|
||||||
#cloud-config
|
#cloud-config
|
||||||
runcmd:
|
runcmd:
|
||||||
- [ /usr/local/libexec/gitea-microvm-guest-runner, "$token_url/token/$nonce", "$gitea_instance", "gitea-${instance_id%%-*}", "$runner_labels" ]
|
- [ sh, -c, "curl --fail --silent --show-error --retry 10 --retry-all-errors $token_url/assets/guest-assets.tar.gz | tar -xz -C /" ]
|
||||||
|
- [ /usr/local/libexec/gitea-microvm-guest-runner, "$token_url/token/$nonce", "$gitea_instance", "gitea-vm-${instance_id%%-*}", "$runner_labels" ]
|
||||||
EOF
|
EOF
|
||||||
cloud-localds "$seed" "$vm_dir/user-data" "$vm_dir/meta-data"
|
cloud-localds "$seed" "$vm_dir/user-data" "$vm_dir/meta-data"
|
||||||
|
|
||||||
@@ -67,8 +70,8 @@ timeout --signal=TERM --kill-after=30s "$vm_timeout" "$cloud_hypervisor" \
|
|||||||
--firmware "$firmware" \
|
--firmware "$firmware" \
|
||||||
--cpus "boot=$cpus" \
|
--cpus "boot=$cpus" \
|
||||||
--memory "size=$memory" \
|
--memory "size=$memory" \
|
||||||
--disk "path=$overlay" \
|
--disk "path=$overlay,image_type=qcow2" \
|
||||||
--disk "path=$seed,readonly=on" \
|
--disk "path=$seed,readonly=on,image_type=raw" \
|
||||||
--net "tap=$tap,mac=$mac" \
|
--net "tap=$tap,mac=$mac" \
|
||||||
--serial "file=$serial" \
|
--serial "file=$serial" \
|
||||||
--console off
|
--console off
|
||||||
|
|||||||
@@ -31,6 +31,9 @@ REGISTRATION_TOKEN_FILE = Path(os.environ.get("REGISTRATION_TOKEN_FILE", "/etc/m
|
|||||||
LAUNCHER = os.environ.get("LAUNCHER", "/usr/local/libexec/microvm-runner-launch")
|
LAUNCHER = os.environ.get("LAUNCHER", "/usr/local/libexec/microvm-runner-launch")
|
||||||
TOKEN_LISTEN = os.environ.get("TOKEN_LISTEN", "172.30.0.1")
|
TOKEN_LISTEN = os.environ.get("TOKEN_LISTEN", "172.30.0.1")
|
||||||
TOKEN_PORT = int(os.environ.get("TOKEN_PORT", "8787"))
|
TOKEN_PORT = int(os.environ.get("TOKEN_PORT", "8787"))
|
||||||
|
GUEST_ASSETS = Path(
|
||||||
|
os.environ.get("GUEST_ASSETS", "/opt/gitea-dynamic-runner/guest-assets.tar.gz")
|
||||||
|
)
|
||||||
|
|
||||||
tokens: dict[str, bytes] = {}
|
tokens: dict[str, bytes] = {}
|
||||||
token_lock = asyncio.Lock()
|
token_lock = asyncio.Lock()
|
||||||
@@ -45,6 +48,13 @@ async def token(request: web.Request) -> web.Response:
|
|||||||
return web.Response(body=value, headers={"Cache-Control": "no-store"})
|
return web.Response(body=value, headers={"Cache-Control": "no-store"})
|
||||||
|
|
||||||
|
|
||||||
|
async def guest_assets(_: web.Request) -> web.FileResponse:
|
||||||
|
return web.FileResponse(
|
||||||
|
GUEST_ASSETS,
|
||||||
|
headers={"Cache-Control": "public, immutable"},
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
async def heartbeat(message: object, stop: asyncio.Event) -> None:
|
async def heartbeat(message: object, stop: asyncio.Event) -> None:
|
||||||
while True:
|
while True:
|
||||||
try:
|
try:
|
||||||
@@ -162,6 +172,7 @@ async def consume() -> None:
|
|||||||
async def main() -> None:
|
async def main() -> None:
|
||||||
app = web.Application()
|
app = web.Application()
|
||||||
app.router.add_get("/token/{nonce}", token)
|
app.router.add_get("/token/{nonce}", token)
|
||||||
|
app.router.add_get("/assets/guest-assets.tar.gz", guest_assets)
|
||||||
runner = web.AppRunner(app)
|
runner = web.AppRunner(app)
|
||||||
await runner.setup()
|
await runner.setup()
|
||||||
await web.TCPSite(runner, TOKEN_LISTEN, TOKEN_PORT).start()
|
await web.TCPSite(runner, TOKEN_LISTEN, TOKEN_PORT).start()
|
||||||
|
|||||||
Reference in New Issue
Block a user