按执行职责拆分动态 Runner 组件

clusters/homelab/apps/dynamic-runner.yaml

@@ -1,7 +1,7 @@
 apiVersion: kustomize.toolkit.fluxcd.io/v1
 kind: Kustomization
 metadata:
-  name: microvm-runner
+  name: dynamic-runner
   namespace: flux-system
 spec:
   dependsOn:
@@ -9,7 +9,7 @@ spec:
     - name: nats
     - name: spire
   interval: 10m
-  path: ./platform/microvm-runner
+  path: ./platform/dynamic-runner
   prune: false
   sourceRef:
     kind: GitRepository

clusters/homelab/kustomization.yaml

@@ -11,7 +11,7 @@ resources:
   - apps/http-echo.yaml
   - apps/openebs.yaml
   - apps/nats.yaml
-  - apps/microvm-runner.yaml
+  - apps/dynamic-runner.yaml
   - apps/spire.yaml
   - apps/observability.yaml
   - apps/zot.yaml

platform/dynamic-runner/README.md

@@ -2,22 +2,23 @@
 
 此目录只管理 homelab 中的 controller 部署。controller、worker、Cloud Hypervisor
 launcher 和 guest runner 的源码与发布位于独立仓库
-`panxiao81/gitea-microvm-runner`。
+`panxiao81/gitea-dynamic-runner`。
 
-controller 接收 Gitea `workflow_job` webhook，将 `[self-hosted, pod]` 和
+当前 bootstrap controller 接收 Gitea `workflow_job` webhook，将 `[self-hosted, pod]` 和
 `[self-hosted, vm]` 的 queued job 分别发布到 NATS。Pod worker 在本集群创建一次性
 privileged host runner；Docker、BuildKit 和 kind 由 workflow 自行 setup。内部
 endpoint：
 
 ```text
-http://microvm-runner-controller.microvm-runner.svc.cluster.local:8787/webhook
+http://dynamic-runner-controller.dynamic-runner.svc.cluster.local:8787/webhook
 ```
 
 OpenBao 路径：
 
 - `kv/k8s/nats.ci_producer_password`：已有 NATS producer 密码。
 - `kv/k8s/nats.ci_worker_password`：已有 NATS worker 密码。
-- `kv/k8s/microvm-runner.webhook_secret`：Gitea webhook HMAC secret。
+- `kv/k8s/microvm-runner.webhook_secret`：Gitea webhook HMAC secret；这是已有 Bao
+  路径，资源重命名不迁移 secret 数据。
 - `kv/k8s/gitea-runner.token`：现有 instance runner registration token。
 
 首期 controller 与 runner 镜像由 laptop 本机构建后导入 k3s containerd，作为 CI
@@ -40,8 +41,12 @@ ci.ddupan.top/spiffe-path=<owner>/<repository>/<percent-encoded-job-name>
 SVID 可用之前不会放行第一步，因此不能根据 queued 事件错配身份。
 
 每个 runner Pod 使用 `gitea-dynamic-runner` ServiceAccount。该 ServiceAccount 没有
-Kubernetes API 权限；只有 controller ServiceAccount 能在本 namespace
+Kubernetes API 权限；只有 `dynamic-runner-pod-worker` ServiceAccount 能在本 namespace
 create/get/patch/delete Pod。
 
+长期实现将由兼容 Gitea RunnerService 的 scheduler 直接领取 task，再交给 Pod/VM
+executor；届时删除 webhook、临时 runner 注册和 identity binding 消息。跟踪见
+`panxiao81/gitea-dynamic-runner` issue #7。
+
 Gitea webhook 只订阅 `workflow_job`，content type 使用 JSON，secret 与 Bao 中值
 一致。不要启用 `send_everything`，否则 controller 会收到无关仓库事件。

platform/dynamic-runner/clusterspiffeid.yaml

@@ -7,11 +7,11 @@ spec:
   spiffeIDTemplate: 'spiffe://{{ .TrustDomain }}/ci/{{ index .PodMeta.Annotations "ci.ddupan.top/spiffe-path" }}'
   namespaceSelector:
     matchLabels:
-      kubernetes.io/metadata.name: microvm-runner
+      kubernetes.io/metadata.name: dynamic-runner
   podSelector:
     matchLabels:
       app.kubernetes.io/name: gitea-dynamic-runner
       ci.ddupan.top/identity-bound: "true"
   workloadSelectorTemplates:
-    - k8s:ns:microvm-runner
+    - k8s:ns:dynamic-runner
     - k8s:sa:gitea-dynamic-runner

platform/dynamic-runner/deployment.yaml

@@ -1,19 +1,20 @@
 apiVersion: apps/v1
 kind: Deployment
 metadata:
-  name: microvm-runner-controller
+  name: dynamic-runner-controller
-  namespace: microvm-runner
+  namespace: dynamic-runner
 spec:
   replicas: 1
   selector:
     matchLabels:
-      app.kubernetes.io/name: microvm-runner-controller
+      app.kubernetes.io/name: dynamic-runner-controller
   template:
     metadata:
       labels:
-        app.kubernetes.io/name: microvm-runner-controller
+        app.kubernetes.io/name: dynamic-runner-controller
     spec:
-      serviceAccountName: microvm-runner-controller
+      serviceAccountName: dynamic-runner-controller
+      automountServiceAccountToken: false
       initContainers:
         - name: fetch-internal-ca
           image: curlimages/curl:8.16.0@sha256:463eaf6072688fe96ac64fa623fe73e1dbe25d8ad6c34404a669ad3ce1f104b6
@@ -39,7 +40,7 @@ spec:
         - name: controller
           # Bootstrap import on laptop. Replace with a zot digest after the
           # repository's image publishing workflow has a dedicated identity.
-          image: gitea-microvm-runner-controller:0.3.0-bootstrap
+          image: gitea-dynamic-runner-controller:0.3.0-bootstrap
           imagePullPolicy: Never
           env:
             - name: NATS_URL
@@ -88,55 +89,6 @@ spec:
             - name: trust
               mountPath: /run/trust
               readOnly: true
-        - name: pod-worker
-          image: gitea-microvm-runner-controller:0.3.0-bootstrap
-          imagePullPolicy: Never
-          command: [/venv/bin/gitea-pod-worker]
-          env:
-            - name: NATS_URL
-              value: tls://nats.ad.ddupan.top:4222
-            - name: NATS_CA_FILE
-              value: /run/trust/ca.pem
-            - name: NATS_PASSWORD_FILE
-              value: /run/secrets/nats-worker-password
-            - name: RUNNER_NAMESPACE
-              valueFrom:
-                fieldRef:
-                  fieldPath: metadata.namespace
-            - name: RUNNER_IMAGE
-              value: gitea-dynamic-runner:0.3.0-bootstrap
-            - name: RUNNER_SERVICE_ACCOUNT
-              value: gitea-dynamic-runner
-            - name: RUNNER_TOKEN_SECRET
-              value: microvm-runner-controller
-            - name: RUNNER_CAPACITY
-              value: "4"
-          resources:
-            requests:
-              cpu: 25m
-              memory: 32Mi
-            limits:
-              cpu: 250m
-              memory: 128Mi
-          securityContext:
-            allowPrivilegeEscalation: false
-            capabilities:
-              drop: [ALL]
-            readOnlyRootFilesystem: true
-            runAsNonRoot: true
-            runAsUser: 65532
-            runAsGroup: 65532
-            seccompProfile:
-              type: RuntimeDefault
-          volumeMounts:
-            - name: secret
-              mountPath: /run/secrets
-              readOnly: true
-            - name: trust
-              mountPath: /run/trust
-              readOnly: true
-      nodeSelector:
-        kubernetes.io/hostname: laptop
       securityContext:
         fsGroup: 65532
         fsGroupChangePolicy: OnRootMismatch
@@ -145,7 +97,7 @@ spec:
       volumes:
         - name: secret
           secret:
-            secretName: microvm-runner-controller
+            secretName: dynamic-runner
             defaultMode: 0400
         - name: trust
           emptyDir:

platform/dynamic-runner/external-secret.yaml

@@ -1,8 +1,8 @@
 apiVersion: external-secrets.io/v1
 kind: ExternalSecret
 metadata:
-  name: microvm-runner-controller
+  name: dynamic-runner
-  namespace: microvm-runner
+  namespace: dynamic-runner
 spec:
   refreshInterval: 1h
   secretStoreRef:
@@ -10,7 +10,7 @@ spec:
     name: openbao
   target:
     creationPolicy: Owner
-    name: microvm-runner-controller
+    name: dynamic-runner
   data:
     - secretKey: nats-password
       remoteRef:

platform/dynamic-runner/kustomization.yaml

@@ -6,4 +6,5 @@ resources:
   - rbac.yaml
   - clusterspiffeid.yaml
   - deployment.yaml
+  - pod-worker-deployment.yaml
   - service.yaml

platform/dynamic-runner/namespace.yaml

@@ -1,7 +1,7 @@
 apiVersion: v1
 kind: Namespace
 metadata:
-  name: microvm-runner
+  name: dynamic-runner
   labels:
     # Disposable Pod runners may start dockerd or kind from the workflow.
     pod-security.kubernetes.io/enforce: privileged

platform/dynamic-runner/pod-worker-deployment.yaml

@@ -0,0 +1,98 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: dynamic-runner-pod-worker
+  namespace: dynamic-runner
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: dynamic-runner-pod-worker
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: dynamic-runner-pod-worker
+    spec:
+      serviceAccountName: dynamic-runner-pod-worker
+      initContainers:
+        - name: fetch-internal-ca
+          image: curlimages/curl:8.16.0@sha256:463eaf6072688fe96ac64fa623fe73e1dbe25d8ad6c34404a669ad3ce1f104b6
+          args:
+            - --fail
+            - --silent
+            - --show-error
+            - --output
+            - /trust/ca.pem
+            - https://bao.ad.ddupan.top:8200/v1/pki/ca/pem
+          securityContext:
+            allowPrivilegeEscalation: false
+            capabilities:
+              drop: [ALL]
+            readOnlyRootFilesystem: true
+            runAsNonRoot: true
+            seccompProfile:
+              type: RuntimeDefault
+          volumeMounts:
+            - name: trust
+              mountPath: /trust
+      containers:
+        - name: pod-worker
+          image: gitea-dynamic-runner-controller:0.3.0-bootstrap
+          imagePullPolicy: Never
+          command: [/venv/bin/gitea-dynamic-runner-pod-worker]
+          env:
+            - name: NATS_URL
+              value: tls://nats.ad.ddupan.top:4222
+            - name: NATS_CA_FILE
+              value: /run/trust/ca.pem
+            - name: NATS_PASSWORD_FILE
+              value: /run/secrets/nats-worker-password
+            - name: RUNNER_NAMESPACE
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.namespace
+            - name: RUNNER_IMAGE
+              value: gitea-dynamic-runner:0.3.0-bootstrap
+            - name: RUNNER_SERVICE_ACCOUNT
+              value: gitea-dynamic-runner
+            - name: RUNNER_TOKEN_SECRET
+              value: dynamic-runner
+            - name: RUNNER_CAPACITY
+              value: "4"
+          resources:
+            requests:
+              cpu: 25m
+              memory: 32Mi
+            limits:
+              cpu: 250m
+              memory: 128Mi
+          securityContext:
+            allowPrivilegeEscalation: false
+            capabilities:
+              drop: [ALL]
+            readOnlyRootFilesystem: true
+            runAsNonRoot: true
+            runAsUser: 65532
+            runAsGroup: 65532
+            seccompProfile:
+              type: RuntimeDefault
+          volumeMounts:
+            - name: secret
+              mountPath: /run/secrets
+              readOnly: true
+            - name: trust
+              mountPath: /run/trust
+              readOnly: true
+      securityContext:
+        fsGroup: 65532
+        fsGroupChangePolicy: OnRootMismatch
+        seccompProfile:
+          type: RuntimeDefault
+      volumes:
+        - name: secret
+          secret:
+            secretName: dynamic-runner
+            defaultMode: 0400
+        - name: trust
+          emptyDir:
+            sizeLimit: 1Mi

platform/dynamic-runner/rbac.yaml

@@ -1,20 +1,26 @@
 apiVersion: v1
 kind: ServiceAccount
 metadata:
-  name: microvm-runner-controller
+  name: dynamic-runner-controller
-  namespace: microvm-runner
+  namespace: dynamic-runner
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: dynamic-runner-pod-worker
+  namespace: dynamic-runner
 ---
 apiVersion: v1
 kind: ServiceAccount
 metadata:
   name: gitea-dynamic-runner
-  namespace: microvm-runner
+  namespace: dynamic-runner
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: Role
 metadata:
-  name: microvm-runner-controller
+  name: dynamic-runner-pod-worker
-  namespace: microvm-runner
+  namespace: dynamic-runner
 rules:
   - apiGroups: [""]
     resources: [pods]
@@ -23,13 +29,13 @@ rules:
 apiVersion: rbac.authorization.k8s.io/v1
 kind: RoleBinding
 metadata:
-  name: microvm-runner-controller
+  name: dynamic-runner-pod-worker
-  namespace: microvm-runner
+  namespace: dynamic-runner
 roleRef:
   apiGroup: rbac.authorization.k8s.io
   kind: Role
-  name: microvm-runner-controller
+  name: dynamic-runner-pod-worker
 subjects:
   - kind: ServiceAccount
-    name: microvm-runner-controller
+    name: dynamic-runner-pod-worker
-    namespace: microvm-runner
+    namespace: dynamic-runner

platform/dynamic-runner/service.yaml

@@ -1,11 +1,11 @@
 apiVersion: v1
 kind: Service
 metadata:
-  name: microvm-runner-controller
+  name: dynamic-runner-controller
-  namespace: microvm-runner
+  namespace: dynamic-runner
 spec:
   selector:
-    app.kubernetes.io/name: microvm-runner-controller
+    app.kubernetes.io/name: dynamic-runner-controller
   ports:
     - name: http
       port: 8787