379 lines
13 KiB
Python
379 lines
13 KiB
Python
#!/usr/bin/env python3
|
|
"""JetStream worker that creates one disposable Kubernetes Pod per job."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import asyncio
|
|
import hashlib
|
|
import json
|
|
import logging
|
|
import os
|
|
from pathlib import Path
|
|
import re
|
|
import ssl
|
|
import uuid
|
|
|
|
import nats
|
|
from aiohttp import ClientResponseError
|
|
from nats.errors import TimeoutError
|
|
from nats.js.api import AckPolicy, ConsumerConfig
|
|
|
|
from .kubernetes import KubernetesClient
|
|
from .models import IdentityBinding, RunnerRequest
|
|
|
|
|
|
LOG = logging.getLogger(__name__)
|
|
STREAM = os.environ.get("NATS_STREAM", "CI_RUNNER")
|
|
REQUEST_SUBJECT = os.environ.get("NATS_SUBJECT", "ci.runner.pod")
|
|
BINDING_SUBJECT = os.environ.get("NATS_BINDING_SUBJECT", "ci.runner.pod.binding")
|
|
NATS_URL = os.environ.get("NATS_URL", "tls://nats.ad.ddupan.top:4222")
|
|
NATS_USER = os.environ.get("NATS_USER", "ci-worker")
|
|
NATS_PASSWORD_FILE = Path(os.environ.get("NATS_PASSWORD_FILE", "/run/secrets/nats/password"))
|
|
NATS_CA_FILE = os.environ.get("NATS_CA_FILE", "/etc/ssl/certs/ca-certificates.crt")
|
|
KUBERNETES_API = os.environ.get("KUBERNETES_API", "https://kubernetes.default.svc")
|
|
KUBERNETES_TOKEN_FILE = Path(
|
|
os.environ.get(
|
|
"KUBERNETES_TOKEN_FILE",
|
|
"/var/run/secrets/kubernetes.io/serviceaccount/token",
|
|
)
|
|
)
|
|
KUBERNETES_CA_FILE = Path(
|
|
os.environ.get(
|
|
"KUBERNETES_CA_FILE",
|
|
"/var/run/secrets/kubernetes.io/serviceaccount/ca.crt",
|
|
)
|
|
)
|
|
NAMESPACE = os.environ.get("RUNNER_NAMESPACE", "gitea-actions")
|
|
RUNNER_IMAGE = os.environ.get("RUNNER_IMAGE", "docker.io/gitea/runner:2")
|
|
RUNNER_SERVICE_ACCOUNT = os.environ.get("RUNNER_SERVICE_ACCOUNT", "gitea-dynamic-runner")
|
|
RUNNER_TOKEN_SECRET = os.environ.get("RUNNER_TOKEN_SECRET", "gitea-dynamic-runner")
|
|
GITEA_INSTANCE = os.environ.get("GITEA_INSTANCE", "https://git.ddupan.top")
|
|
CAPACITY = int(os.environ.get("RUNNER_CAPACITY", "4"))
|
|
POD_TIMEOUT = int(os.environ.get("RUNNER_POD_TIMEOUT", str(4 * 60 * 60)))
|
|
SPIFFE_PATH_SEGMENT = re.compile(r"^[A-Za-z0-9._-]+$")
|
|
|
|
|
|
def identity_path(repository: str, job_name: str) -> str:
|
|
parts = repository.split("/")
|
|
if len(parts) != 2 or not all(parts):
|
|
raise ValueError("repository must be owner/name")
|
|
if not job_name.strip():
|
|
raise ValueError("identity components must not be empty")
|
|
return "/".join(safe_identity_segment(part) for part in (*parts, job_name))
|
|
|
|
|
|
def safe_identity_segment(value: str) -> str:
|
|
"""Map arbitrary Gitea names to stable SPIFFE Operator path segments."""
|
|
if SPIFFE_PATH_SEGMENT.fullmatch(value):
|
|
return value
|
|
slug = re.sub(r"[^A-Za-z0-9._-]+", "-", value).strip("-._")
|
|
slug = slug[:48].rstrip("-._") or "segment"
|
|
digest = hashlib.sha256(value.encode()).hexdigest()[:12]
|
|
return f"{slug}-{digest}"
|
|
|
|
|
|
def pod_manifest(request: RunnerRequest, pod_name: str) -> dict[str, object]:
|
|
if request.backend != "pod":
|
|
raise ValueError("Pod backend only accepts pod requests")
|
|
return {
|
|
"apiVersion": "v1",
|
|
"kind": "Pod",
|
|
"metadata": {
|
|
"name": pod_name,
|
|
"namespace": NAMESPACE,
|
|
"labels": {
|
|
"app.kubernetes.io/name": "gitea-dynamic-runner",
|
|
"app.kubernetes.io/component": "runner",
|
|
"ci.ddupan.top/backend": "pod",
|
|
},
|
|
"annotations": {
|
|
"ci.ddupan.top/queued-job-id": str(request.job_id),
|
|
"ci.ddupan.top/queued-run-id": str(request.run_id),
|
|
},
|
|
},
|
|
"spec": {
|
|
"serviceAccountName": RUNNER_SERVICE_ACCOUNT,
|
|
"restartPolicy": "Never",
|
|
"terminationGracePeriodSeconds": 30,
|
|
"containers": [
|
|
{
|
|
"name": "runner",
|
|
"image": RUNNER_IMAGE,
|
|
"imagePullPolicy": "IfNotPresent",
|
|
"securityContext": {"privileged": True},
|
|
"env": [
|
|
{"name": "GITEA_INSTANCE_URL", "value": GITEA_INSTANCE},
|
|
{
|
|
"name": "GITEA_RUNNER_NAME",
|
|
"valueFrom": {"fieldRef": {"fieldPath": "metadata.name"}},
|
|
},
|
|
{
|
|
"name": "GITEA_RUNNER_REGISTRATION_TOKEN_FILE",
|
|
"value": "/run/secrets/gitea/token",
|
|
},
|
|
{"name": "GITEA_RUNNER_LABELS", "value": "self-hosted:host,pod:host"},
|
|
{"name": "GITEA_RUNNER_EPHEMERAL", "value": "1"},
|
|
{"name": "GITEA_RUNNER_ONCE", "value": "1"},
|
|
{"name": "CONFIG_FILE", "value": "/etc/gitea-runner/config.yaml"},
|
|
],
|
|
"volumeMounts": [
|
|
{
|
|
"name": "registration-token",
|
|
"mountPath": "/run/secrets/gitea",
|
|
"readOnly": True,
|
|
},
|
|
{
|
|
"name": "spire-agent-socket",
|
|
"mountPath": "/run/spire/agent-sockets",
|
|
"readOnly": True,
|
|
},
|
|
],
|
|
}
|
|
],
|
|
"volumes": [
|
|
{
|
|
"name": "registration-token",
|
|
"secret": {
|
|
"secretName": RUNNER_TOKEN_SECRET,
|
|
"items": [{"key": "token", "path": "token"}],
|
|
},
|
|
},
|
|
{
|
|
"name": "spire-agent-socket",
|
|
"csi": {"driver": "csi.spiffe.io", "readOnly": True},
|
|
},
|
|
],
|
|
},
|
|
}
|
|
|
|
|
|
async def heartbeat(message: object, stop: asyncio.Event) -> None:
|
|
while True:
|
|
try:
|
|
await asyncio.wait_for(stop.wait(), timeout=60)
|
|
return
|
|
except asyncio.TimeoutError:
|
|
await message.in_progress()
|
|
|
|
|
|
async def wait_for_pod(client: KubernetesClient, name: str) -> bool:
|
|
deadline = asyncio.get_running_loop().time() + POD_TIMEOUT
|
|
while asyncio.get_running_loop().time() < deadline:
|
|
pod = await client.get_pod(name)
|
|
if pod is None:
|
|
raise RuntimeError(f"runner Pod {name} disappeared")
|
|
status = pod.get("status")
|
|
phase = status.get("phase") if isinstance(status, dict) else None
|
|
if phase == "Succeeded":
|
|
return True
|
|
if phase == "Failed":
|
|
return False
|
|
await asyncio.sleep(2)
|
|
raise asyncio.TimeoutError(f"runner Pod {name} timed out")
|
|
|
|
|
|
async def run_request(message: object, client: KubernetesClient) -> None:
|
|
try:
|
|
request = RunnerRequest.from_json(message.data)
|
|
except (json.JSONDecodeError, UnicodeDecodeError, ValueError) as error:
|
|
LOG.error("discarding invalid Pod request: %s", error)
|
|
await message.ack()
|
|
return
|
|
if request.backend != "pod":
|
|
LOG.error("discarding %s request received by Pod worker", request.backend)
|
|
await message.ack()
|
|
return
|
|
|
|
pod_name = f"gitea-pod-{uuid.uuid4().hex[:12]}"
|
|
LOG.info(
|
|
"runner request received job_id=%d run_id=%d repository=%s job_name=%r "
|
|
"backend=%s pod=%s",
|
|
request.job_id,
|
|
request.run_id,
|
|
request.repository,
|
|
request.job_name,
|
|
request.backend,
|
|
pod_name,
|
|
)
|
|
stop = asyncio.Event()
|
|
pulse = asyncio.create_task(heartbeat(message, stop))
|
|
try:
|
|
await client.create_pod(pod_manifest(request, pod_name))
|
|
LOG.info(
|
|
"runner Pod created pod=%s job_id=%d run_id=%d image=%s",
|
|
pod_name,
|
|
request.job_id,
|
|
request.run_id,
|
|
RUNNER_IMAGE,
|
|
)
|
|
if await wait_for_pod(client, pod_name):
|
|
await message.ack()
|
|
LOG.info(
|
|
"runner request acknowledged pod=%s job_id=%d result=succeeded",
|
|
pod_name,
|
|
request.job_id,
|
|
)
|
|
else:
|
|
LOG.error(
|
|
"runner Pod failed pod=%s job_id=%d; request will be retried",
|
|
pod_name,
|
|
request.job_id,
|
|
)
|
|
await message.nak(delay=30)
|
|
except Exception:
|
|
LOG.exception(
|
|
"runner request failed pod=%s job_id=%d; request will be retried",
|
|
pod_name,
|
|
request.job_id,
|
|
)
|
|
await message.nak(delay=30)
|
|
raise
|
|
finally:
|
|
stop.set()
|
|
await pulse
|
|
LOG.info("deleting runner Pod pod=%s job_id=%d", pod_name, request.job_id)
|
|
await client.delete_pod(pod_name)
|
|
|
|
|
|
async def bind_request(message: object, client: KubernetesClient) -> None:
|
|
binding: IdentityBinding | None = None
|
|
try:
|
|
document = json.loads(message.data)
|
|
binding = IdentityBinding(**document)
|
|
if binding.backend != "pod" or not binding.runner_name.startswith("gitea-pod-"):
|
|
raise ValueError("invalid Pod identity binding")
|
|
path = identity_path(binding.repository, binding.job_name)
|
|
LOG.info(
|
|
"identity binding received job_id=%d run_id=%d runner_name=%s "
|
|
"repository=%s job_name=%r identity_path=%s",
|
|
binding.job_id,
|
|
binding.run_id,
|
|
binding.runner_name,
|
|
binding.repository,
|
|
binding.job_name,
|
|
path,
|
|
)
|
|
await client.bind_identity(binding.runner_name, path)
|
|
except ClientResponseError as error:
|
|
if error.status == 404:
|
|
LOG.warning(
|
|
"identity binding Pod not found runner_name=%s job_id=%s; binding will be retried",
|
|
binding.runner_name if binding else None,
|
|
binding.job_id if binding else None,
|
|
)
|
|
await message.nak(delay=2)
|
|
return
|
|
LOG.exception(
|
|
"identity binding Kubernetes request failed runner_name=%s job_id=%s status=%d",
|
|
binding.runner_name if binding else None,
|
|
binding.job_id if binding else None,
|
|
error.status,
|
|
)
|
|
await message.nak(delay=30)
|
|
raise
|
|
except (json.JSONDecodeError, TypeError, ValueError) as error:
|
|
LOG.error("discarding invalid identity binding: %s", error)
|
|
await message.ack()
|
|
return
|
|
await message.ack()
|
|
LOG.info(
|
|
"identity binding applied and acknowledged runner_name=%s job_id=%d identity_path=%s",
|
|
binding.runner_name,
|
|
binding.job_id,
|
|
path,
|
|
)
|
|
|
|
|
|
async def consume_requests(subscription: object, client: KubernetesClient) -> None:
|
|
active: set[asyncio.Task[None]] = set()
|
|
while True:
|
|
active = {task for task in active if not task.done()}
|
|
free = CAPACITY - len(active)
|
|
if free < 1:
|
|
await asyncio.wait(active, return_when=asyncio.FIRST_COMPLETED)
|
|
continue
|
|
try:
|
|
messages = await subscription.fetch(batch=free, timeout=5)
|
|
except TimeoutError:
|
|
continue
|
|
for message in messages:
|
|
task = asyncio.create_task(run_request(message, client))
|
|
task.add_done_callback(_report_task)
|
|
active.add(task)
|
|
|
|
|
|
async def consume_bindings(subscription: object, client: KubernetesClient) -> None:
|
|
while True:
|
|
try:
|
|
messages = await subscription.fetch(batch=16, timeout=5)
|
|
except TimeoutError:
|
|
continue
|
|
await asyncio.gather(*(bind_request(message, client) for message in messages))
|
|
|
|
|
|
def _report_task(task: asyncio.Task[None]) -> None:
|
|
if not task.cancelled() and (error := task.exception()) is not None:
|
|
LOG.error("Pod runner task failed", exc_info=(type(error), error, error.__traceback__))
|
|
|
|
|
|
async def main() -> None:
|
|
if CAPACITY < 1:
|
|
raise ValueError("RUNNER_CAPACITY must be at least 1")
|
|
context = ssl.create_default_context(cafile=NATS_CA_FILE)
|
|
nc = await nats.connect(
|
|
NATS_URL,
|
|
user=NATS_USER,
|
|
password=NATS_PASSWORD_FILE.read_text().strip(),
|
|
tls=context,
|
|
name="pod-runner-worker",
|
|
)
|
|
js = nc.jetstream()
|
|
requests = await js.pull_subscribe(
|
|
REQUEST_SUBJECT,
|
|
durable="pod",
|
|
stream=STREAM,
|
|
config=ConsumerConfig(
|
|
durable_name="pod",
|
|
filter_subject=REQUEST_SUBJECT,
|
|
ack_policy=AckPolicy.EXPLICIT,
|
|
ack_wait=5 * 60,
|
|
max_ack_pending=max(CAPACITY, 1),
|
|
max_deliver=5,
|
|
),
|
|
)
|
|
bindings = await js.pull_subscribe(
|
|
BINDING_SUBJECT,
|
|
durable="pod-binding",
|
|
stream=STREAM,
|
|
config=ConsumerConfig(
|
|
durable_name="pod-binding",
|
|
filter_subject=BINDING_SUBJECT,
|
|
ack_policy=AckPolicy.EXPLICIT,
|
|
ack_wait=30,
|
|
max_ack_pending=64,
|
|
max_deliver=10,
|
|
),
|
|
)
|
|
async with KubernetesClient(
|
|
api_url=KUBERNETES_API,
|
|
token_file=KUBERNETES_TOKEN_FILE,
|
|
ca_file=KUBERNETES_CA_FILE,
|
|
namespace=NAMESPACE,
|
|
) as client:
|
|
try:
|
|
await asyncio.gather(
|
|
consume_requests(requests, client),
|
|
consume_bindings(bindings, client),
|
|
)
|
|
finally:
|
|
await nc.drain()
|
|
|
|
|
|
def cli() -> None:
|
|
logging.basicConfig(level=os.environ.get("LOG_LEVEL", "INFO"))
|
|
asyncio.run(main())
|
|
|
|
|
|
if __name__ == "__main__":
|
|
cli()
|