重构为直接 OpenSandbox 生命周期调度
This commit is contained in:
@@ -1,68 +1,51 @@
|
||||
#!/usr/bin/env python3
|
||||
"""JetStream worker that provisions disposable OpenSandbox Kata runners."""
|
||||
"""Direct OpenSandbox lifecycle scheduler used by the webhook controller."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import hashlib
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
from pathlib import Path
|
||||
import ssl
|
||||
import secrets
|
||||
import uuid
|
||||
|
||||
import nats
|
||||
from nats.errors import TimeoutError
|
||||
from nats.js.api import AckPolicy, ConsumerConfig
|
||||
from collections.abc import Callable
|
||||
|
||||
from .models import RunnerRequest
|
||||
from .opensandbox import OpenSandboxClient
|
||||
from .pod_worker import heartbeat, identity_path
|
||||
from .sandbox_kubernetes import SandboxKubernetesClient, allocated_pod_name
|
||||
from .pod_worker import identity_path
|
||||
|
||||
|
||||
LOG = logging.getLogger(__name__)
|
||||
STREAM = os.environ.get("NATS_STREAM", "CI_RUNNER")
|
||||
REQUEST_SUBJECT = os.environ.get("NATS_SUBJECT", "ci.runner.vm")
|
||||
NATS_URL = os.environ.get("NATS_URL", "tls://nats.ad.ddupan.top:4222")
|
||||
NATS_USER = os.environ.get("NATS_USER", "ci-worker")
|
||||
NATS_PASSWORD_FILE = Path(os.environ.get("NATS_PASSWORD_FILE", "/run/secrets/nats/password"))
|
||||
NATS_CA_FILE = os.environ.get("NATS_CA_FILE", "/etc/ssl/certs/ca-certificates.crt")
|
||||
KUBERNETES_API = os.environ.get("KUBERNETES_API", "https://kubernetes.default.svc")
|
||||
KUBERNETES_TOKEN_FILE = Path(os.environ.get("KUBERNETES_TOKEN_FILE", "/var/run/secrets/kubernetes.io/serviceaccount/token"))
|
||||
KUBERNETES_CA_FILE = Path(os.environ.get("KUBERNETES_CA_FILE", "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt"))
|
||||
NAMESPACE = os.environ.get("OPENSANDBOX_NAMESPACE", "opensandbox")
|
||||
OPENSANDBOX_API = os.environ.get(
|
||||
"OPENSANDBOX_API", "http://opensandbox-server.opensandbox-system.svc"
|
||||
)
|
||||
OPENSANDBOX_API_KEY_FILE_VALUE = os.environ.get("OPENSANDBOX_API_KEY_FILE")
|
||||
OPENSANDBOX_API_KEY_FILE = Path(OPENSANDBOX_API_KEY_FILE_VALUE) if OPENSANDBOX_API_KEY_FILE_VALUE else None
|
||||
OPENSANDBOX_POOL = os.environ.get("OPENSANDBOX_POOL", "ci-vm")
|
||||
SANDBOX_TIMEOUT = int(os.environ.get("RUNNER_SANDBOX_TIMEOUT", str(4 * 60 * 60)))
|
||||
ALLOCATION_TIMEOUT = int(os.environ.get("RUNNER_ALLOCATION_TIMEOUT", "120"))
|
||||
CAPACITY = int(os.environ.get("RUNNER_CAPACITY", "2"))
|
||||
SPIFFE_TRUST_DOMAIN = os.environ.get("SPIFFE_TRUST_DOMAIN", "ddupan.top")
|
||||
SPIRE_CLUSTER_NAME = os.environ.get("SPIRE_CLUSTER_NAME", "sandbox-kata")
|
||||
SPIRE_CLASS_NAME = os.environ.get("SPIRE_CLASS_NAME", "spire-mgmt-spire")
|
||||
RUNNER_UID = int(os.environ.get("RUNNER_UID", "2000"))
|
||||
TOKEN_BASE_URL = os.environ.get(
|
||||
"RUNNER_TOKEN_BASE_URL", "http://192.168.10.127:8787/token"
|
||||
).rstrip("/")
|
||||
|
||||
|
||||
def sandbox_request(request: RunnerRequest, runner_name: str) -> dict[str, object]:
|
||||
if request.backend != "vm":
|
||||
raise ValueError("OpenSandbox backend only accepts vm requests")
|
||||
def sandbox_request(
|
||||
request: RunnerRequest,
|
||||
runner_name: str,
|
||||
registration_token_url: str,
|
||||
) -> dict[str, object]:
|
||||
path = identity_path(request.repository, request.job_name)
|
||||
return {
|
||||
"pool": OPENSANDBOX_POOL,
|
||||
"pool": f"ci-{request.backend}",
|
||||
"timeout": SANDBOX_TIMEOUT,
|
||||
"entrypoint": ["/usr/local/libexec/gitea-opensandbox-runner"],
|
||||
"env": {
|
||||
"GITEA_RUNNER_NAME": runner_name,
|
||||
"GITEA_RUNNER_LABELS": "self-hosted:host,vm:host",
|
||||
"GITEA_RUNNER_LABELS": f"self-hosted:host,{request.backend}:host",
|
||||
"GITEA_RUNNER_REGISTRATION_TOKEN_URL": registration_token_url,
|
||||
"GITEA_RUNNER_EPHEMERAL": "1",
|
||||
"GITEA_RUNNER_ONCE": "1",
|
||||
"CONFIG_FILE": "/etc/gitea-runner/config.yaml",
|
||||
"CI_SPIFFE_ID": f"spiffe://{SPIFFE_TRUST_DOMAIN}/ci/{path}",
|
||||
"SPIFFE_ENDPOINT_SOCKET": "unix:///run/spire/agent-sockets/spire-agent.sock",
|
||||
"SPIFFE_ENDPOINT_SOCKET": (
|
||||
"unix:///run/spire/agent-sockets/spire-agent.sock"
|
||||
),
|
||||
},
|
||||
"metadata": {
|
||||
"ci.ddupan.top/runner": "true",
|
||||
"ci.ddupan.top/job-id": str(request.job_id),
|
||||
"ci.ddupan.top/run-id": str(request.run_id),
|
||||
"ci.ddupan.top/runner-name": runner_name,
|
||||
@@ -70,207 +53,111 @@ def sandbox_request(request: RunnerRequest, runner_name: str) -> dict[str, objec
|
||||
}
|
||||
|
||||
|
||||
def entry_name(sandbox_id: str) -> str:
|
||||
suffix = hashlib.sha256(sandbox_id.encode()).hexdigest()[:12]
|
||||
return f"gitea-ci-{suffix}"
|
||||
class RegistrationTokens:
|
||||
"""Single-use registration-token URLs; values never enter Sandbox CRs."""
|
||||
|
||||
def __init__(self, token: bytes) -> None:
|
||||
self._token = token.strip()
|
||||
self._values: dict[str, bytes] = {}
|
||||
self._lock = asyncio.Lock()
|
||||
|
||||
async def issue(self) -> tuple[str, str]:
|
||||
nonce = secrets.token_urlsafe(32)
|
||||
async with self._lock:
|
||||
self._values[nonce] = self._token
|
||||
return nonce, f"{TOKEN_BASE_URL}/{nonce}"
|
||||
|
||||
async def consume(self, nonce: str) -> bytes | None:
|
||||
async with self._lock:
|
||||
return self._values.pop(nonce, None)
|
||||
|
||||
async def revoke(self, nonce: str) -> None:
|
||||
async with self._lock:
|
||||
self._values.pop(nonce, None)
|
||||
|
||||
|
||||
def identity_entry(
|
||||
request: RunnerRequest,
|
||||
*,
|
||||
sandbox_id: str,
|
||||
pod_uid: str,
|
||||
) -> dict[str, object]:
|
||||
path = identity_path(request.repository, request.job_name)
|
||||
return {
|
||||
"apiVersion": "spire.spiffe.io/v1alpha1",
|
||||
"kind": "ClusterStaticEntry",
|
||||
"metadata": {
|
||||
"name": entry_name(sandbox_id),
|
||||
"labels": {
|
||||
"app.kubernetes.io/name": "gitea-dynamic-runner",
|
||||
"app.kubernetes.io/component": "opensandbox-identity",
|
||||
"ci.ddupan.top/sandbox-id": sandbox_id,
|
||||
},
|
||||
},
|
||||
"spec": {
|
||||
"className": SPIRE_CLASS_NAME,
|
||||
"parentID": (
|
||||
f"spiffe://{SPIFFE_TRUST_DOMAIN}/spire/agent/k8s_psat/"
|
||||
f"{SPIRE_CLUSTER_NAME}/pod/{pod_uid}"
|
||||
),
|
||||
"spiffeID": f"spiffe://{SPIFFE_TRUST_DOMAIN}/ci/{path}",
|
||||
"selectors": [f"unix:uid:{RUNNER_UID}"],
|
||||
},
|
||||
}
|
||||
class OpenSandboxScheduler:
|
||||
def __init__(
|
||||
self,
|
||||
client: OpenSandboxClient,
|
||||
tokens: RegistrationTokens,
|
||||
*,
|
||||
on_finished: Callable[[int], None] | None = None,
|
||||
) -> None:
|
||||
self.client = client
|
||||
self.tokens = tokens
|
||||
self.on_finished = on_finished
|
||||
self.active: dict[int, asyncio.Task[None]] = {}
|
||||
|
||||
|
||||
async def wait_for_allocation(
|
||||
client: SandboxKubernetesClient, sandbox_id: str
|
||||
) -> tuple[str, str]:
|
||||
deadline = asyncio.get_running_loop().time() + ALLOCATION_TIMEOUT
|
||||
while asyncio.get_running_loop().time() < deadline:
|
||||
batchsandbox = await client.get_batchsandbox(sandbox_id)
|
||||
if batchsandbox is not None and (pod_name := allocated_pod_name(batchsandbox)):
|
||||
pod = await client.get_pod(pod_name)
|
||||
if pod is not None:
|
||||
metadata = pod.get("metadata")
|
||||
pod_uid = metadata.get("uid") if isinstance(metadata, dict) else None
|
||||
if isinstance(pod_uid, str) and pod_uid:
|
||||
return pod_name, pod_uid
|
||||
await asyncio.sleep(1)
|
||||
raise asyncio.TimeoutError(f"OpenSandbox {sandbox_id} allocation timed out")
|
||||
|
||||
|
||||
async def wait_for_sandbox(client: OpenSandboxClient, sandbox_id: str) -> bool:
|
||||
deadline = asyncio.get_running_loop().time() + SANDBOX_TIMEOUT
|
||||
while asyncio.get_running_loop().time() < deadline:
|
||||
sandbox = await client.get(sandbox_id)
|
||||
if sandbox is None:
|
||||
raise RuntimeError(f"OpenSandbox {sandbox_id} disappeared")
|
||||
status = sandbox.get("status")
|
||||
state = status.get("state") if isinstance(status, dict) else None
|
||||
if state == "Terminated":
|
||||
return True
|
||||
if state == "Failed":
|
||||
return False
|
||||
await asyncio.sleep(2)
|
||||
raise asyncio.TimeoutError(f"OpenSandbox {sandbox_id} timed out")
|
||||
|
||||
|
||||
async def run_request(
|
||||
message: object,
|
||||
sandbox_client: OpenSandboxClient,
|
||||
kubernetes_client: SandboxKubernetesClient,
|
||||
) -> None:
|
||||
try:
|
||||
request = RunnerRequest.from_json(message.data)
|
||||
except (json.JSONDecodeError, UnicodeDecodeError, ValueError) as error:
|
||||
LOG.error("discarding invalid OpenSandbox request: %s", error)
|
||||
await message.ack()
|
||||
return
|
||||
if request.backend != "vm":
|
||||
LOG.error("discarding %s request received by OpenSandbox worker", request.backend)
|
||||
await message.ack()
|
||||
return
|
||||
|
||||
runner_name = f"gitea-vm-{uuid.uuid4().hex[:12]}"
|
||||
sandbox_id: str | None = None
|
||||
static_entry: str | None = None
|
||||
stop = asyncio.Event()
|
||||
pulse = asyncio.create_task(heartbeat(message, stop))
|
||||
try:
|
||||
create = await sandbox_client.create(**sandbox_request(request, runner_name))
|
||||
candidate = create.get("id")
|
||||
if not isinstance(candidate, str) or not candidate:
|
||||
raise RuntimeError("OpenSandbox create response has no id")
|
||||
sandbox_id = candidate
|
||||
pod_name, pod_uid = await wait_for_allocation(kubernetes_client, sandbox_id)
|
||||
manifest = identity_entry(request, sandbox_id=sandbox_id, pod_uid=pod_uid)
|
||||
static_entry = entry_name(sandbox_id)
|
||||
await kubernetes_client.create_entry(manifest)
|
||||
LOG.info(
|
||||
"OpenSandbox identity created sandbox=%s pod=%s pod_uid=%s entry=%s runner=%s job_id=%d",
|
||||
sandbox_id, pod_name, pod_uid, static_entry, runner_name, request.job_id,
|
||||
)
|
||||
if await wait_for_sandbox(sandbox_client, sandbox_id):
|
||||
await message.ack()
|
||||
else:
|
||||
await message.nak(delay=30)
|
||||
except Exception:
|
||||
LOG.exception("OpenSandbox runner failed sandbox=%s job_id=%d", sandbox_id, request.job_id)
|
||||
await message.nak(delay=30)
|
||||
raise
|
||||
finally:
|
||||
stop.set()
|
||||
await pulse
|
||||
async def create(self, request: RunnerRequest) -> str:
|
||||
if request.job_id in self.active:
|
||||
raise ValueError(f"job {request.job_id} already has an active sandbox")
|
||||
runner_name = f"gitea-{request.backend}-{uuid.uuid4().hex[:12]}"
|
||||
nonce, token_url = await self.tokens.issue()
|
||||
try:
|
||||
if static_entry is not None:
|
||||
await kubernetes_client.delete_entry(static_entry)
|
||||
finally:
|
||||
if sandbox_id is not None:
|
||||
await sandbox_client.delete(sandbox_id)
|
||||
|
||||
|
||||
async def consume_requests(
|
||||
subscription: object,
|
||||
sandbox_client: OpenSandboxClient,
|
||||
kubernetes_client: SandboxKubernetesClient,
|
||||
) -> None:
|
||||
active: set[asyncio.Task[None]] = set()
|
||||
while True:
|
||||
active = {task for task in active if not task.done()}
|
||||
free = CAPACITY - len(active)
|
||||
if free < 1:
|
||||
await asyncio.wait(active, return_when=asyncio.FIRST_COMPLETED)
|
||||
continue
|
||||
try:
|
||||
messages = await subscription.fetch(batch=free, timeout=5)
|
||||
except TimeoutError:
|
||||
continue
|
||||
for message in messages:
|
||||
task = asyncio.create_task(
|
||||
run_request(message, sandbox_client, kubernetes_client)
|
||||
response = await self.client.create(
|
||||
**sandbox_request(request, runner_name, token_url)
|
||||
)
|
||||
task.add_done_callback(_report_task)
|
||||
active.add(task)
|
||||
sandbox_id = response.get("id")
|
||||
if not isinstance(sandbox_id, str) or not sandbox_id:
|
||||
raise RuntimeError("OpenSandbox create response has no id")
|
||||
except Exception:
|
||||
await self.tokens.revoke(nonce)
|
||||
raise
|
||||
|
||||
|
||||
def _report_task(task: asyncio.Task[None]) -> None:
|
||||
if not task.cancelled() and (error := task.exception()) is not None:
|
||||
LOG.error(
|
||||
"OpenSandbox runner task failed",
|
||||
exc_info=(type(error), error, error.__traceback__),
|
||||
task = asyncio.create_task(
|
||||
self._monitor(request, sandbox_id, nonce),
|
||||
name=f"opensandbox-{sandbox_id}",
|
||||
)
|
||||
task.add_done_callback(self._report)
|
||||
self.active[request.job_id] = task
|
||||
LOG.info(
|
||||
"OpenSandbox runner created sandbox=%s runner=%s job_id=%d "
|
||||
"repository=%s job_name=%r pool=ci-%s",
|
||||
sandbox_id,
|
||||
runner_name,
|
||||
request.job_id,
|
||||
request.repository,
|
||||
request.job_name,
|
||||
request.backend,
|
||||
)
|
||||
return sandbox_id
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
if CAPACITY < 1:
|
||||
raise ValueError("RUNNER_CAPACITY must be at least 1")
|
||||
context = ssl.create_default_context(cafile=NATS_CA_FILE)
|
||||
nc = await nats.connect(
|
||||
NATS_URL,
|
||||
user=NATS_USER,
|
||||
password=NATS_PASSWORD_FILE.read_text().strip(),
|
||||
tls=context,
|
||||
name="opensandbox-runner-worker",
|
||||
)
|
||||
js = nc.jetstream()
|
||||
subscription = await js.pull_subscribe(
|
||||
REQUEST_SUBJECT,
|
||||
durable="vm",
|
||||
stream=STREAM,
|
||||
config=ConsumerConfig(
|
||||
durable_name="vm",
|
||||
filter_subject=REQUEST_SUBJECT,
|
||||
ack_policy=AckPolicy.EXPLICIT,
|
||||
ack_wait=5 * 60,
|
||||
max_ack_pending=max(CAPACITY, 1),
|
||||
max_deliver=5,
|
||||
),
|
||||
)
|
||||
async with (
|
||||
OpenSandboxClient(
|
||||
api_url=OPENSANDBOX_API,
|
||||
api_key_file=OPENSANDBOX_API_KEY_FILE,
|
||||
) as sandbox_client,
|
||||
SandboxKubernetesClient(
|
||||
api_url=KUBERNETES_API,
|
||||
token_file=KUBERNETES_TOKEN_FILE,
|
||||
ca_file=KUBERNETES_CA_FILE,
|
||||
namespace=NAMESPACE,
|
||||
) as kubernetes_client,
|
||||
):
|
||||
async def _monitor(
|
||||
self, request: RunnerRequest, sandbox_id: str, nonce: str
|
||||
) -> None:
|
||||
try:
|
||||
await consume_requests(subscription, sandbox_client, kubernetes_client)
|
||||
deadline = asyncio.get_running_loop().time() + SANDBOX_TIMEOUT
|
||||
while asyncio.get_running_loop().time() < deadline:
|
||||
sandbox = await self.client.get(sandbox_id)
|
||||
if sandbox is None:
|
||||
return
|
||||
status = sandbox.get("status")
|
||||
state = status.get("state") if isinstance(status, dict) else None
|
||||
if state in {"Terminated", "Failed"}:
|
||||
return
|
||||
await asyncio.sleep(2)
|
||||
raise asyncio.TimeoutError(f"OpenSandbox {sandbox_id} timed out")
|
||||
finally:
|
||||
await nc.drain()
|
||||
await self.tokens.revoke(nonce)
|
||||
try:
|
||||
await self.client.delete(sandbox_id)
|
||||
finally:
|
||||
self.active.pop(request.job_id, None)
|
||||
if self.on_finished is not None:
|
||||
self.on_finished(request.job_id)
|
||||
|
||||
@staticmethod
|
||||
def _report(task: asyncio.Task[None]) -> None:
|
||||
if not task.cancelled() and (error := task.exception()) is not None:
|
||||
LOG.error(
|
||||
"OpenSandbox lifecycle task failed",
|
||||
exc_info=(type(error), error, error.__traceback__),
|
||||
)
|
||||
|
||||
def cli() -> None:
|
||||
logging.basicConfig(level=os.environ.get("LOG_LEVEL", "INFO"))
|
||||
asyncio.run(main())
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
cli()
|
||||
async def close(self) -> None:
|
||||
tasks = list(self.active.values())
|
||||
for task in tasks:
|
||||
task.cancel()
|
||||
if tasks:
|
||||
await asyncio.gather(*tasks, return_exceptions=True)
|
||||
|
||||
Reference in New Issue
Block a user