接入 OpenSandbox Kata Runner 控制面

2026-09-18 00:56:10 +00:00
parent 8d70df2128
commit d7ba64d6f3
9 changed files with 680 additions and 0 deletions
@@ -0,0 +1,274 @@
+#!/usr/bin/env python3
+"""JetStream worker that provisions disposable OpenSandbox Kata runners."""
+
+from __future__ import annotations
+
+import asyncio
+import hashlib
+import json
+import logging
+import os
+from pathlib import Path
+import ssl
+import uuid
+
+import nats
+from nats.errors import TimeoutError
+from nats.js.api import AckPolicy, ConsumerConfig
+
+from .models import RunnerRequest
+from .opensandbox import OpenSandboxClient
+from .pod_worker import heartbeat, identity_path
+from .sandbox_kubernetes import SandboxKubernetesClient, allocated_pod_name
+
+
+LOG = logging.getLogger(__name__)
+STREAM = os.environ.get("NATS_STREAM", "CI_RUNNER")
+REQUEST_SUBJECT = os.environ.get("NATS_SUBJECT", "ci.runner.vm")
+NATS_URL = os.environ.get("NATS_URL", "tls://nats.ad.ddupan.top:4222")
+NATS_USER = os.environ.get("NATS_USER", "ci-worker")
+NATS_PASSWORD_FILE = Path(os.environ.get("NATS_PASSWORD_FILE", "/run/secrets/nats/password"))
+NATS_CA_FILE = os.environ.get("NATS_CA_FILE", "/etc/ssl/certs/ca-certificates.crt")
+KUBERNETES_API = os.environ.get("KUBERNETES_API", "https://kubernetes.default.svc")
+KUBERNETES_TOKEN_FILE = Path(os.environ.get("KUBERNETES_TOKEN_FILE", "/var/run/secrets/kubernetes.io/serviceaccount/token"))
+KUBERNETES_CA_FILE = Path(os.environ.get("KUBERNETES_CA_FILE", "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt"))
+NAMESPACE = os.environ.get("OPENSANDBOX_NAMESPACE", "opensandbox")
+OPENSANDBOX_API = os.environ.get("OPENSANDBOX_API", "http://opensandbox-server.opensandbox.svc:8080")
+OPENSANDBOX_API_KEY_FILE_VALUE = os.environ.get("OPENSANDBOX_API_KEY_FILE")
+OPENSANDBOX_API_KEY_FILE = Path(OPENSANDBOX_API_KEY_FILE_VALUE) if OPENSANDBOX_API_KEY_FILE_VALUE else None
+OPENSANDBOX_POOL = os.environ.get("OPENSANDBOX_POOL", "ci-vm")
+SANDBOX_TIMEOUT = int(os.environ.get("RUNNER_SANDBOX_TIMEOUT", str(4 * 60 * 60)))
+ALLOCATION_TIMEOUT = int(os.environ.get("RUNNER_ALLOCATION_TIMEOUT", "120"))
+CAPACITY = int(os.environ.get("RUNNER_CAPACITY", "2"))
+SPIFFE_TRUST_DOMAIN = os.environ.get("SPIFFE_TRUST_DOMAIN", "ddupan.top")
+SPIRE_CLUSTER_NAME = os.environ.get("SPIRE_CLUSTER_NAME", "sandbox-kata")
+SPIRE_CLASS_NAME = os.environ.get("SPIRE_CLASS_NAME", "spire-mgmt-spire")
+RUNNER_UID = int(os.environ.get("RUNNER_UID", "2000"))
+
+
+def sandbox_request(request: RunnerRequest, runner_name: str) -> dict[str, object]:
+    if request.backend != "vm":
+        raise ValueError("OpenSandbox backend only accepts vm requests")
+    path = identity_path(request.repository, request.job_name)
+    return {
+        "pool": OPENSANDBOX_POOL,
+        "timeout": SANDBOX_TIMEOUT,
+        "entrypoint": ["/usr/local/libexec/gitea-opensandbox-runner"],
+        "env": {
+            "GITEA_RUNNER_NAME": runner_name,
+            "GITEA_RUNNER_LABELS": "self-hosted:host,vm:host",
+            "CI_SPIFFE_ID": f"spiffe://{SPIFFE_TRUST_DOMAIN}/ci/{path}",
+            "SPIFFE_ENDPOINT_SOCKET": "unix:///run/spire/agent-sockets/spire-agent.sock",
+        },
+        "metadata": {
+            "ci.ddupan.top/job-id": str(request.job_id),
+            "ci.ddupan.top/run-id": str(request.run_id),
+            "ci.ddupan.top/runner-name": runner_name,
+        },
+    }
+
+
+def entry_name(sandbox_id: str) -> str:
+    suffix = hashlib.sha256(sandbox_id.encode()).hexdigest()[:12]
+    return f"gitea-ci-{suffix}"
+
+
+def identity_entry(
+    request: RunnerRequest,
+    *,
+    sandbox_id: str,
+    pod_uid: str,
+) -> dict[str, object]:
+    path = identity_path(request.repository, request.job_name)
+    return {
+        "apiVersion": "spire.spiffe.io/v1alpha1",
+        "kind": "ClusterStaticEntry",
+        "metadata": {
+            "name": entry_name(sandbox_id),
+            "labels": {
+                "app.kubernetes.io/name": "gitea-dynamic-runner",
+                "app.kubernetes.io/component": "opensandbox-identity",
+                "ci.ddupan.top/sandbox-id": sandbox_id,
+            },
+        },
+        "spec": {
+            "className": SPIRE_CLASS_NAME,
+            "parentID": (
+                f"spiffe://{SPIFFE_TRUST_DOMAIN}/spire/agent/k8s_psat/"
+                f"{SPIRE_CLUSTER_NAME}/pod/{pod_uid}"
+            ),
+            "spiffeID": f"spiffe://{SPIFFE_TRUST_DOMAIN}/ci/{path}",
+            "selectors": [f"unix:uid:{RUNNER_UID}"],
+        },
+    }
+
+
+async def wait_for_allocation(
+    client: SandboxKubernetesClient, sandbox_id: str
+) -> tuple[str, str]:
+    deadline = asyncio.get_running_loop().time() + ALLOCATION_TIMEOUT
+    while asyncio.get_running_loop().time() < deadline:
+        batchsandbox = await client.get_batchsandbox(sandbox_id)
+        if batchsandbox is not None and (pod_name := allocated_pod_name(batchsandbox)):
+            pod = await client.get_pod(pod_name)
+            if pod is not None:
+                metadata = pod.get("metadata")
+                pod_uid = metadata.get("uid") if isinstance(metadata, dict) else None
+                if isinstance(pod_uid, str) and pod_uid:
+                    return pod_name, pod_uid
+        await asyncio.sleep(1)
+    raise asyncio.TimeoutError(f"OpenSandbox {sandbox_id} allocation timed out")
+
+
+async def wait_for_sandbox(client: OpenSandboxClient, sandbox_id: str) -> bool:
+    deadline = asyncio.get_running_loop().time() + SANDBOX_TIMEOUT
+    while asyncio.get_running_loop().time() < deadline:
+        sandbox = await client.get(sandbox_id)
+        if sandbox is None:
+            raise RuntimeError(f"OpenSandbox {sandbox_id} disappeared")
+        status = sandbox.get("status")
+        state = status.get("state") if isinstance(status, dict) else None
+        if state == "Terminated":
+            return True
+        if state == "Failed":
+            return False
+        await asyncio.sleep(2)
+    raise asyncio.TimeoutError(f"OpenSandbox {sandbox_id} timed out")
+
+
+async def run_request(
+    message: object,
+    sandbox_client: OpenSandboxClient,
+    kubernetes_client: SandboxKubernetesClient,
+) -> None:
+    try:
+        request = RunnerRequest.from_json(message.data)
+    except (json.JSONDecodeError, UnicodeDecodeError, ValueError) as error:
+        LOG.error("discarding invalid OpenSandbox request: %s", error)
+        await message.ack()
+        return
+    if request.backend != "vm":
+        LOG.error("discarding %s request received by OpenSandbox worker", request.backend)
+        await message.ack()
+        return
+
+    runner_name = f"gitea-vm-{uuid.uuid4().hex[:12]}"
+    sandbox_id: str | None = None
+    static_entry: str | None = None
+    stop = asyncio.Event()
+    pulse = asyncio.create_task(heartbeat(message, stop))
+    try:
+        create = await sandbox_client.create(**sandbox_request(request, runner_name))
+        candidate = create.get("id")
+        if not isinstance(candidate, str) or not candidate:
+            raise RuntimeError("OpenSandbox create response has no id")
+        sandbox_id = candidate
+        pod_name, pod_uid = await wait_for_allocation(kubernetes_client, sandbox_id)
+        manifest = identity_entry(request, sandbox_id=sandbox_id, pod_uid=pod_uid)
+        static_entry = entry_name(sandbox_id)
+        await kubernetes_client.create_entry(manifest)
+        LOG.info(
+            "OpenSandbox identity created sandbox=%s pod=%s pod_uid=%s entry=%s runner=%s job_id=%d",
+            sandbox_id, pod_name, pod_uid, static_entry, runner_name, request.job_id,
+        )
+        if await wait_for_sandbox(sandbox_client, sandbox_id):
+            await message.ack()
+        else:
+            await message.nak(delay=30)
+    except Exception:
+        LOG.exception("OpenSandbox runner failed sandbox=%s job_id=%d", sandbox_id, request.job_id)
+        await message.nak(delay=30)
+        raise
+    finally:
+        stop.set()
+        await pulse
+        try:
+            if static_entry is not None:
+                await kubernetes_client.delete_entry(static_entry)
+        finally:
+            if sandbox_id is not None:
+                await sandbox_client.delete(sandbox_id)
+
+
+async def consume_requests(
+    subscription: object,
+    sandbox_client: OpenSandboxClient,
+    kubernetes_client: SandboxKubernetesClient,
+) -> None:
+    active: set[asyncio.Task[None]] = set()
+    while True:
+        active = {task for task in active if not task.done()}
+        free = CAPACITY - len(active)
+        if free < 1:
+            await asyncio.wait(active, return_when=asyncio.FIRST_COMPLETED)
+            continue
+        try:
+            messages = await subscription.fetch(batch=free, timeout=5)
+        except TimeoutError:
+            continue
+        for message in messages:
+            task = asyncio.create_task(
+                run_request(message, sandbox_client, kubernetes_client)
+            )
+            task.add_done_callback(_report_task)
+            active.add(task)
+
+
+def _report_task(task: asyncio.Task[None]) -> None:
+    if not task.cancelled() and (error := task.exception()) is not None:
+        LOG.error(
+            "OpenSandbox runner task failed",
+            exc_info=(type(error), error, error.__traceback__),
+        )
+
+
+async def main() -> None:
+    if CAPACITY < 1:
+        raise ValueError("RUNNER_CAPACITY must be at least 1")
+    context = ssl.create_default_context(cafile=NATS_CA_FILE)
+    nc = await nats.connect(
+        NATS_URL,
+        user=NATS_USER,
+        password=NATS_PASSWORD_FILE.read_text().strip(),
+        tls=context,
+        name="opensandbox-runner-worker",
+    )
+    js = nc.jetstream()
+    subscription = await js.pull_subscribe(
+        REQUEST_SUBJECT,
+        durable="vm",
+        stream=STREAM,
+        config=ConsumerConfig(
+            durable_name="vm",
+            filter_subject=REQUEST_SUBJECT,
+            ack_policy=AckPolicy.EXPLICIT,
+            ack_wait=5 * 60,
+            max_ack_pending=max(CAPACITY, 1),
+            max_deliver=5,
+        ),
+    )
+    async with (
+        OpenSandboxClient(
+            api_url=OPENSANDBOX_API,
+            api_key_file=OPENSANDBOX_API_KEY_FILE,
+        ) as sandbox_client,
+        SandboxKubernetesClient(
+            api_url=KUBERNETES_API,
+            token_file=KUBERNETES_TOKEN_FILE,
+            ca_file=KUBERNETES_CA_FILE,
+            namespace=NAMESPACE,
+        ) as kubernetes_client,
+    ):
+        try:
+            await consume_requests(subscription, sandbox_client, kubernetes_client)
+        finally:
+            await nc.drain()
+
+
+def cli() -> None:
+    logging.basicConfig(level=os.environ.get("LOG_LEVEL", "INFO"))
+    asyncio.run(main())
+
+
+if __name__ == "__main__":
+    cli()