From c4f425e9eacdffb59181ca2248bb37e69aff7198 Mon Sep 17 00:00:00 2001 From: panxiao81 Date: Wed, 16 Sep 2026 14:53:30 +0000 Subject: [PATCH] =?UTF-8?q?=E6=8C=89=E6=89=A7=E8=A1=8C=E8=81=8C=E8=B4=A3?= =?UTF-8?q?=E6=8B=86=E5=88=86=E5=8A=A8=E6=80=81=20Runner=20=E7=BB=84?= =?UTF-8?q?=E4=BB=B6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ...icrovm-runner.yaml => dynamic-runner.yaml} | 4 +- clusters/homelab/kustomization.yaml | 2 +- .../README.md | 15 ++- .../clusterspiffeid.yaml | 4 +- .../deployment.yaml | 64 ++---------- .../external-secret.yaml | 6 +- .../kustomization.yaml | 1 + .../namespace.yaml | 2 +- .../dynamic-runner/pod-worker-deployment.yaml | 98 +++++++++++++++++++ .../rbac.yaml | 26 +++-- .../service.yaml | 6 +- 11 files changed, 145 insertions(+), 83 deletions(-) rename clusters/homelab/apps/{microvm-runner.yaml => dynamic-runner.yaml} (83%) rename platform/{microvm-runner => dynamic-runner}/README.md (76%) rename platform/{microvm-runner => dynamic-runner}/clusterspiffeid.yaml (85%) rename platform/{microvm-runner => dynamic-runner}/deployment.yaml (58%) rename platform/{microvm-runner => dynamic-runner}/external-secret.yaml (86%) rename platform/{microvm-runner => dynamic-runner}/kustomization.yaml (86%) rename platform/{microvm-runner => dynamic-runner}/namespace.yaml (92%) create mode 100644 platform/dynamic-runner/pod-worker-deployment.yaml rename platform/{microvm-runner => dynamic-runner}/rbac.yaml (51%) rename platform/{microvm-runner => dynamic-runner}/service.yaml (51%) diff --git a/clusters/homelab/apps/microvm-runner.yaml b/clusters/homelab/apps/dynamic-runner.yaml similarity index 83% rename from clusters/homelab/apps/microvm-runner.yaml rename to clusters/homelab/apps/dynamic-runner.yaml index 4813de3..d67375f 100644 --- a/clusters/homelab/apps/microvm-runner.yaml +++ b/clusters/homelab/apps/dynamic-runner.yaml @@ -1,7 +1,7 @@ apiVersion: kustomize.toolkit.fluxcd.io/v1 kind: Kustomization metadata: - name: microvm-runner + name: dynamic-runner namespace: flux-system spec: dependsOn: @@ -9,7 +9,7 @@ spec: - name: nats - name: spire interval: 10m - path: ./platform/microvm-runner + path: ./platform/dynamic-runner prune: false sourceRef: kind: GitRepository diff --git a/clusters/homelab/kustomization.yaml b/clusters/homelab/kustomization.yaml index 4fc4476..00b74f8 100644 --- a/clusters/homelab/kustomization.yaml +++ b/clusters/homelab/kustomization.yaml @@ -11,7 +11,7 @@ resources: - apps/http-echo.yaml - apps/openebs.yaml - apps/nats.yaml - - apps/microvm-runner.yaml + - apps/dynamic-runner.yaml - apps/spire.yaml - apps/observability.yaml - apps/zot.yaml diff --git a/platform/microvm-runner/README.md b/platform/dynamic-runner/README.md similarity index 76% rename from platform/microvm-runner/README.md rename to platform/dynamic-runner/README.md index cd8789d..cddd7d1 100644 --- a/platform/microvm-runner/README.md +++ b/platform/dynamic-runner/README.md @@ -2,22 +2,23 @@ 此目录只管理 homelab 中的 controller 部署。controller、worker、Cloud Hypervisor launcher 和 guest runner 的源码与发布位于独立仓库 -`panxiao81/gitea-microvm-runner`。 +`panxiao81/gitea-dynamic-runner`。 -controller 接收 Gitea `workflow_job` webhook，将 `[self-hosted, pod]` 和 +当前 bootstrap controller 接收 Gitea `workflow_job` webhook，将 `[self-hosted, pod]` 和 `[self-hosted, vm]` 的 queued job 分别发布到 NATS。Pod worker 在本集群创建一次性 privileged host runner；Docker、BuildKit 和 kind 由 workflow 自行 setup。内部 endpoint： ```text -http://microvm-runner-controller.microvm-runner.svc.cluster.local:8787/webhook +http://dynamic-runner-controller.dynamic-runner.svc.cluster.local:8787/webhook ``` OpenBao 路径： - `kv/k8s/nats.ci_producer_password`：已有 NATS producer 密码。 - `kv/k8s/nats.ci_worker_password`：已有 NATS worker 密码。 -- `kv/k8s/microvm-runner.webhook_secret`：Gitea webhook HMAC secret。 +- `kv/k8s/microvm-runner.webhook_secret`：Gitea webhook HMAC secret；这是已有 Bao + 路径，资源重命名不迁移 secret 数据。 - `kv/k8s/gitea-runner.token`：现有 instance runner registration token。 首期 controller 与 runner 镜像由 laptop 本机构建后导入 k3s containerd，作为 CI @@ -40,8 +41,12 @@ ci.ddupan.top/spiffe-path=// SVID 可用之前不会放行第一步，因此不能根据 queued 事件错配身份。 每个 runner Pod 使用 `gitea-dynamic-runner` ServiceAccount。该 ServiceAccount 没有 -Kubernetes API 权限；只有 controller ServiceAccount 能在本 namespace +Kubernetes API 权限；只有 `dynamic-runner-pod-worker` ServiceAccount 能在本 namespace create/get/patch/delete Pod。 +长期实现将由兼容 Gitea RunnerService 的 scheduler 直接领取 task，再交给 Pod/VM +executor；届时删除 webhook、临时 runner 注册和 identity binding 消息。跟踪见 +`panxiao81/gitea-dynamic-runner` issue #7。 + Gitea webhook 只订阅 `workflow_job`，content type 使用 JSON，secret 与 Bao 中值 一致。不要启用 `send_everything`，否则 controller 会收到无关仓库事件。 diff --git a/platform/microvm-runner/clusterspiffeid.yaml b/platform/dynamic-runner/clusterspiffeid.yaml similarity index 85% rename from platform/microvm-runner/clusterspiffeid.yaml rename to platform/dynamic-runner/clusterspiffeid.yaml index 0ae87dd..57bda53 100644 --- a/platform/microvm-runner/clusterspiffeid.yaml +++ b/platform/dynamic-runner/clusterspiffeid.yaml @@ -7,11 +7,11 @@ spec: spiffeIDTemplate: 'spiffe://{{ .TrustDomain }}/ci/{{ index .PodMeta.Annotations "ci.ddupan.top/spiffe-path" }}' namespaceSelector: matchLabels: - kubernetes.io/metadata.name: microvm-runner + kubernetes.io/metadata.name: dynamic-runner podSelector: matchLabels: app.kubernetes.io/name: gitea-dynamic-runner ci.ddupan.top/identity-bound: "true" workloadSelectorTemplates: - - k8s:ns:microvm-runner + - k8s:ns:dynamic-runner - k8s:sa:gitea-dynamic-runner diff --git a/platform/microvm-runner/deployment.yaml b/platform/dynamic-runner/deployment.yaml similarity index 58% rename from platform/microvm-runner/deployment.yaml rename to platform/dynamic-runner/deployment.yaml index 685d0db..62cac55 100644 --- a/platform/microvm-runner/deployment.yaml +++ b/platform/dynamic-runner/deployment.yaml @@ -1,19 +1,20 @@ apiVersion: apps/v1 kind: Deployment metadata: - name: microvm-runner-controller - namespace: microvm-runner + name: dynamic-runner-controller + namespace: dynamic-runner spec: replicas: 1 selector: matchLabels: - app.kubernetes.io/name: microvm-runner-controller + app.kubernetes.io/name: dynamic-runner-controller template: metadata: labels: - app.kubernetes.io/name: microvm-runner-controller + app.kubernetes.io/name: dynamic-runner-controller spec: - serviceAccountName: microvm-runner-controller + serviceAccountName: dynamic-runner-controller + automountServiceAccountToken: false initContainers: - name: fetch-internal-ca image: curlimages/curl:8.16.0@sha256:463eaf6072688fe96ac64fa623fe73e1dbe25d8ad6c34404a669ad3ce1f104b6 @@ -39,7 +40,7 @@ spec: - name: controller # Bootstrap import on laptop. Replace with a zot digest after the # repository's image publishing workflow has a dedicated identity. - image: gitea-microvm-runner-controller:0.3.0-bootstrap + image: gitea-dynamic-runner-controller:0.3.0-bootstrap imagePullPolicy: Never env: - name: NATS_URL @@ -88,55 +89,6 @@ spec: - name: trust mountPath: /run/trust readOnly: true - - name: pod-worker - image: gitea-microvm-runner-controller:0.3.0-bootstrap - imagePullPolicy: Never - command: [/venv/bin/gitea-pod-worker] - env: - - name: NATS_URL - value: tls://nats.ad.ddupan.top:4222 - - name: NATS_CA_FILE - value: /run/trust/ca.pem - - name: NATS_PASSWORD_FILE - value: /run/secrets/nats-worker-password - - name: RUNNER_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - - name: RUNNER_IMAGE - value: gitea-dynamic-runner:0.3.0-bootstrap - - name: RUNNER_SERVICE_ACCOUNT - value: gitea-dynamic-runner - - name: RUNNER_TOKEN_SECRET - value: microvm-runner-controller - - name: RUNNER_CAPACITY - value: "4" - resources: - requests: - cpu: 25m - memory: 32Mi - limits: - cpu: 250m - memory: 128Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: [ALL] - readOnlyRootFilesystem: true - runAsNonRoot: true - runAsUser: 65532 - runAsGroup: 65532 - seccompProfile: - type: RuntimeDefault - volumeMounts: - - name: secret - mountPath: /run/secrets - readOnly: true - - name: trust - mountPath: /run/trust - readOnly: true - nodeSelector: - kubernetes.io/hostname: laptop securityContext: fsGroup: 65532 fsGroupChangePolicy: OnRootMismatch @@ -145,7 +97,7 @@ spec: volumes: - name: secret secret: - secretName: microvm-runner-controller + secretName: dynamic-runner defaultMode: 0400 - name: trust emptyDir: diff --git a/platform/microvm-runner/external-secret.yaml b/platform/dynamic-runner/external-secret.yaml similarity index 86% rename from platform/microvm-runner/external-secret.yaml rename to platform/dynamic-runner/external-secret.yaml index 946071c..f406cd5 100644 --- a/platform/microvm-runner/external-secret.yaml +++ b/platform/dynamic-runner/external-secret.yaml @@ -1,8 +1,8 @@ apiVersion: external-secrets.io/v1 kind: ExternalSecret metadata: - name: microvm-runner-controller - namespace: microvm-runner + name: dynamic-runner + namespace: dynamic-runner spec: refreshInterval: 1h secretStoreRef: @@ -10,7 +10,7 @@ spec: name: openbao target: creationPolicy: Owner - name: microvm-runner-controller + name: dynamic-runner data: - secretKey: nats-password remoteRef: diff --git a/platform/microvm-runner/kustomization.yaml b/platform/dynamic-runner/kustomization.yaml similarity index 86% rename from platform/microvm-runner/kustomization.yaml rename to platform/dynamic-runner/kustomization.yaml index 00d940d..6bf792e 100644 --- a/platform/microvm-runner/kustomization.yaml +++ b/platform/dynamic-runner/kustomization.yaml @@ -6,4 +6,5 @@ resources: - rbac.yaml - clusterspiffeid.yaml - deployment.yaml + - pod-worker-deployment.yaml - service.yaml diff --git a/platform/microvm-runner/namespace.yaml b/platform/dynamic-runner/namespace.yaml similarity index 92% rename from platform/microvm-runner/namespace.yaml rename to platform/dynamic-runner/namespace.yaml index 7dfdab1..5c2cdd8 100644 --- a/platform/microvm-runner/namespace.yaml +++ b/platform/dynamic-runner/namespace.yaml @@ -1,7 +1,7 @@ apiVersion: v1 kind: Namespace metadata: - name: microvm-runner + name: dynamic-runner labels: # Disposable Pod runners may start dockerd or kind from the workflow. pod-security.kubernetes.io/enforce: privileged diff --git a/platform/dynamic-runner/pod-worker-deployment.yaml b/platform/dynamic-runner/pod-worker-deployment.yaml new file mode 100644 index 0000000..3e65309 --- /dev/null +++ b/platform/dynamic-runner/pod-worker-deployment.yaml @@ -0,0 +1,98 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: dynamic-runner-pod-worker + namespace: dynamic-runner +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: dynamic-runner-pod-worker + template: + metadata: + labels: + app.kubernetes.io/name: dynamic-runner-pod-worker + spec: + serviceAccountName: dynamic-runner-pod-worker + initContainers: + - name: fetch-internal-ca + image: curlimages/curl:8.16.0@sha256:463eaf6072688fe96ac64fa623fe73e1dbe25d8ad6c34404a669ad3ce1f104b6 + args: + - --fail + - --silent + - --show-error + - --output + - /trust/ca.pem + - https://bao.ad.ddupan.top:8200/v1/pki/ca/pem + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: [ALL] + readOnlyRootFilesystem: true + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault + volumeMounts: + - name: trust + mountPath: /trust + containers: + - name: pod-worker + image: gitea-dynamic-runner-controller:0.3.0-bootstrap + imagePullPolicy: Never + command: [/venv/bin/gitea-dynamic-runner-pod-worker] + env: + - name: NATS_URL + value: tls://nats.ad.ddupan.top:4222 + - name: NATS_CA_FILE + value: /run/trust/ca.pem + - name: NATS_PASSWORD_FILE + value: /run/secrets/nats-worker-password + - name: RUNNER_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: RUNNER_IMAGE + value: gitea-dynamic-runner:0.3.0-bootstrap + - name: RUNNER_SERVICE_ACCOUNT + value: gitea-dynamic-runner + - name: RUNNER_TOKEN_SECRET + value: dynamic-runner + - name: RUNNER_CAPACITY + value: "4" + resources: + requests: + cpu: 25m + memory: 32Mi + limits: + cpu: 250m + memory: 128Mi + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: [ALL] + readOnlyRootFilesystem: true + runAsNonRoot: true + runAsUser: 65532 + runAsGroup: 65532 + seccompProfile: + type: RuntimeDefault + volumeMounts: + - name: secret + mountPath: /run/secrets + readOnly: true + - name: trust + mountPath: /run/trust + readOnly: true + securityContext: + fsGroup: 65532 + fsGroupChangePolicy: OnRootMismatch + seccompProfile: + type: RuntimeDefault + volumes: + - name: secret + secret: + secretName: dynamic-runner + defaultMode: 0400 + - name: trust + emptyDir: + sizeLimit: 1Mi diff --git a/platform/microvm-runner/rbac.yaml b/platform/dynamic-runner/rbac.yaml similarity index 51% rename from platform/microvm-runner/rbac.yaml rename to platform/dynamic-runner/rbac.yaml index a560c8b..7d96083 100644 --- a/platform/microvm-runner/rbac.yaml +++ b/platform/dynamic-runner/rbac.yaml @@ -1,20 +1,26 @@ apiVersion: v1 kind: ServiceAccount metadata: - name: microvm-runner-controller - namespace: microvm-runner + name: dynamic-runner-controller + namespace: dynamic-runner +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: dynamic-runner-pod-worker + namespace: dynamic-runner --- apiVersion: v1 kind: ServiceAccount metadata: name: gitea-dynamic-runner - namespace: microvm-runner + namespace: dynamic-runner --- apiVersion: rbac.authorization.k8s.io/v1 kind: Role metadata: - name: microvm-runner-controller - namespace: microvm-runner + name: dynamic-runner-pod-worker + namespace: dynamic-runner rules: - apiGroups: [""] resources: [pods] @@ -23,13 +29,13 @@ rules: apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding metadata: - name: microvm-runner-controller - namespace: microvm-runner + name: dynamic-runner-pod-worker + namespace: dynamic-runner roleRef: apiGroup: rbac.authorization.k8s.io kind: Role - name: microvm-runner-controller + name: dynamic-runner-pod-worker subjects: - kind: ServiceAccount - name: microvm-runner-controller - namespace: microvm-runner + name: dynamic-runner-pod-worker + namespace: dynamic-runner diff --git a/platform/microvm-runner/service.yaml b/platform/dynamic-runner/service.yaml similarity index 51% rename from platform/microvm-runner/service.yaml rename to platform/dynamic-runner/service.yaml index 87f9f0f..e12f4b8 100644 --- a/platform/microvm-runner/service.yaml +++ b/platform/dynamic-runner/service.yaml @@ -1,11 +1,11 @@ apiVersion: v1 kind: Service metadata: - name: microvm-runner-controller - namespace: microvm-runner + name: dynamic-runner-controller + namespace: dynamic-runner spec: selector: - app.kubernetes.io/name: microvm-runner-controller + app.kubernetes.io/name: dynamic-runner-controller ports: - name: http port: 8787