diff --git a/clusters/homelab/apps/microvm-runner.yaml b/clusters/homelab/apps/microvm-runner.yaml new file mode 100644 index 0000000..4813de3 --- /dev/null +++ b/clusters/homelab/apps/microvm-runner.yaml @@ -0,0 +1,18 @@ +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: microvm-runner + namespace: flux-system +spec: + dependsOn: + - name: external-secrets + - name: nats + - name: spire + interval: 10m + path: ./platform/microvm-runner + prune: false + sourceRef: + kind: GitRepository + name: flux-system + timeout: 5m + wait: true diff --git a/clusters/homelab/kustomization.yaml b/clusters/homelab/kustomization.yaml index 4bdcde0..4fc4476 100644 --- a/clusters/homelab/kustomization.yaml +++ b/clusters/homelab/kustomization.yaml @@ -11,6 +11,7 @@ resources: - apps/http-echo.yaml - apps/openebs.yaml - apps/nats.yaml + - apps/microvm-runner.yaml - apps/spire.yaml - apps/observability.yaml - apps/zot.yaml diff --git a/platform/microvm-runner/README.md b/platform/microvm-runner/README.md new file mode 100644 index 0000000..cd8789d --- /dev/null +++ b/platform/microvm-runner/README.md @@ -0,0 +1,47 @@ +# Gitea dynamic runner controller + +此目录只管理 homelab 中的 controller 部署。controller、worker、Cloud Hypervisor +launcher 和 guest runner 的源码与发布位于独立仓库 +`panxiao81/gitea-microvm-runner`。 + +controller 接收 Gitea `workflow_job` webhook,将 `[self-hosted, pod]` 和 +`[self-hosted, vm]` 的 queued job 分别发布到 NATS。Pod worker 在本集群创建一次性 +privileged host runner;Docker、BuildKit 和 kind 由 workflow 自行 setup。内部 +endpoint: + +```text +http://microvm-runner-controller.microvm-runner.svc.cluster.local:8787/webhook +``` + +OpenBao 路径: + +- `kv/k8s/nats.ci_producer_password`:已有 NATS producer 密码。 +- `kv/k8s/nats.ci_worker_password`:已有 NATS worker 密码。 +- `kv/k8s/microvm-runner.webhook_secret`:Gitea webhook HMAC secret。 +- `kv/k8s/gitea-runner.token`:现有 instance runner registration token。 + +首期 controller 与 runner 镜像由 laptop 本机构建后导入 k3s containerd,作为 CI +发布链路建立前的 bootstrap。部署使用 `imagePullPolicy: Never`。正式发布 workflow +获得专用 SPIFFE ID 后,必须将 image 改为 zot digest 并移除本地导入步骤。 + +## 身份绑定 + +queued webhook 只负责创建没有业务身份的 Pod。runner 实际领取任务后,Gitea 的 +`in_progress` webhook 会携带实际 `runner_name`;controller 将 binding 消息发布到 +NATS,Pod worker 再给对应 Pod 添加: + +```text +ci.ddupan.top/identity-bound=true +ci.ddupan.top/spiffe-path=// +``` + +`ClusterSPIFFEID/gitea-dynamic-runner` 只匹配已经绑定的 Pod,并签发 +`spiffe://ddupan.top/ci///`。runner 的 job-start hook 在 +SVID 可用之前不会放行第一步,因此不能根据 queued 事件错配身份。 + +每个 runner Pod 使用 `gitea-dynamic-runner` ServiceAccount。该 ServiceAccount 没有 +Kubernetes API 权限;只有 controller ServiceAccount 能在本 namespace +create/get/patch/delete Pod。 + +Gitea webhook 只订阅 `workflow_job`,content type 使用 JSON,secret 与 Bao 中值 +一致。不要启用 `send_everything`,否则 controller 会收到无关仓库事件。 diff --git a/platform/microvm-runner/clusterspiffeid.yaml b/platform/microvm-runner/clusterspiffeid.yaml new file mode 100644 index 0000000..0ae87dd --- /dev/null +++ b/platform/microvm-runner/clusterspiffeid.yaml @@ -0,0 +1,17 @@ +apiVersion: spire.spiffe.io/v1alpha1 +kind: ClusterSPIFFEID +metadata: + name: gitea-dynamic-runner +spec: + className: spire-mgmt-spire + spiffeIDTemplate: 'spiffe://{{ .TrustDomain }}/ci/{{ index .PodMeta.Annotations "ci.ddupan.top/spiffe-path" }}' + namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: microvm-runner + podSelector: + matchLabels: + app.kubernetes.io/name: gitea-dynamic-runner + ci.ddupan.top/identity-bound: "true" + workloadSelectorTemplates: + - k8s:ns:microvm-runner + - k8s:sa:gitea-dynamic-runner diff --git a/platform/microvm-runner/deployment.yaml b/platform/microvm-runner/deployment.yaml new file mode 100644 index 0000000..685d0db --- /dev/null +++ b/platform/microvm-runner/deployment.yaml @@ -0,0 +1,152 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: microvm-runner-controller + namespace: microvm-runner +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: microvm-runner-controller + template: + metadata: + labels: + app.kubernetes.io/name: microvm-runner-controller + spec: + serviceAccountName: microvm-runner-controller + initContainers: + - name: fetch-internal-ca + image: curlimages/curl:8.16.0@sha256:463eaf6072688fe96ac64fa623fe73e1dbe25d8ad6c34404a669ad3ce1f104b6 + args: + - --fail + - --silent + - --show-error + - --output + - /trust/ca.pem + - https://bao.ad.ddupan.top:8200/v1/pki/ca/pem + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: [ALL] + readOnlyRootFilesystem: true + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault + volumeMounts: + - name: trust + mountPath: /trust + containers: + - name: controller + # Bootstrap import on laptop. Replace with a zot digest after the + # repository's image publishing workflow has a dedicated identity. + image: gitea-microvm-runner-controller:0.3.0-bootstrap + imagePullPolicy: Never + env: + - name: NATS_URL + value: tls://nats.ad.ddupan.top:4222 + - name: NATS_CA_FILE + value: /run/trust/ca.pem + - name: NATS_PASSWORD_FILE + value: /run/secrets/nats-password + - name: WEBHOOK_SECRET_FILE + value: /run/secrets/webhook-secret + ports: + - name: http + containerPort: 8787 + readinessProbe: + httpGet: + path: /healthz + port: http + periodSeconds: 5 + livenessProbe: + httpGet: + path: /healthz + port: http + initialDelaySeconds: 10 + periodSeconds: 10 + resources: + requests: + cpu: 25m + memory: 32Mi + limits: + cpu: 250m + memory: 128Mi + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: [ALL] + readOnlyRootFilesystem: true + runAsNonRoot: true + runAsUser: 65532 + runAsGroup: 65532 + seccompProfile: + type: RuntimeDefault + volumeMounts: + - name: secret + mountPath: /run/secrets + readOnly: true + - name: trust + mountPath: /run/trust + readOnly: true + - name: pod-worker + image: gitea-microvm-runner-controller:0.3.0-bootstrap + imagePullPolicy: Never + command: [/venv/bin/gitea-pod-worker] + env: + - name: NATS_URL + value: tls://nats.ad.ddupan.top:4222 + - name: NATS_CA_FILE + value: /run/trust/ca.pem + - name: NATS_PASSWORD_FILE + value: /run/secrets/nats-worker-password + - name: RUNNER_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: RUNNER_IMAGE + value: gitea-dynamic-runner:0.3.0-bootstrap + - name: RUNNER_SERVICE_ACCOUNT + value: gitea-dynamic-runner + - name: RUNNER_TOKEN_SECRET + value: microvm-runner-controller + - name: RUNNER_CAPACITY + value: "4" + resources: + requests: + cpu: 25m + memory: 32Mi + limits: + cpu: 250m + memory: 128Mi + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: [ALL] + readOnlyRootFilesystem: true + runAsNonRoot: true + runAsUser: 65532 + runAsGroup: 65532 + seccompProfile: + type: RuntimeDefault + volumeMounts: + - name: secret + mountPath: /run/secrets + readOnly: true + - name: trust + mountPath: /run/trust + readOnly: true + nodeSelector: + kubernetes.io/hostname: laptop + securityContext: + fsGroup: 65532 + fsGroupChangePolicy: OnRootMismatch + seccompProfile: + type: RuntimeDefault + volumes: + - name: secret + secret: + secretName: microvm-runner-controller + defaultMode: 0400 + - name: trust + emptyDir: + sizeLimit: 1Mi diff --git a/platform/microvm-runner/external-secret.yaml b/platform/microvm-runner/external-secret.yaml new file mode 100644 index 0000000..946071c --- /dev/null +++ b/platform/microvm-runner/external-secret.yaml @@ -0,0 +1,30 @@ +apiVersion: external-secrets.io/v1 +kind: ExternalSecret +metadata: + name: microvm-runner-controller + namespace: microvm-runner +spec: + refreshInterval: 1h + secretStoreRef: + kind: ClusterSecretStore + name: openbao + target: + creationPolicy: Owner + name: microvm-runner-controller + data: + - secretKey: nats-password + remoteRef: + key: k8s/nats + property: ci_producer_password + - secretKey: nats-worker-password + remoteRef: + key: k8s/nats + property: ci_worker_password + - secretKey: webhook-secret + remoteRef: + key: k8s/microvm-runner + property: webhook_secret + - secretKey: token + remoteRef: + key: k8s/gitea-runner + property: token diff --git a/platform/microvm-runner/kustomization.yaml b/platform/microvm-runner/kustomization.yaml new file mode 100644 index 0000000..00d940d --- /dev/null +++ b/platform/microvm-runner/kustomization.yaml @@ -0,0 +1,9 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - namespace.yaml + - external-secret.yaml + - rbac.yaml + - clusterspiffeid.yaml + - deployment.yaml + - service.yaml diff --git a/platform/microvm-runner/namespace.yaml b/platform/microvm-runner/namespace.yaml new file mode 100644 index 0000000..7dfdab1 --- /dev/null +++ b/platform/microvm-runner/namespace.yaml @@ -0,0 +1,9 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: microvm-runner + labels: + # Disposable Pod runners may start dockerd or kind from the workflow. + pod-security.kubernetes.io/enforce: privileged + pod-security.kubernetes.io/audit: restricted + pod-security.kubernetes.io/warn: restricted diff --git a/platform/microvm-runner/rbac.yaml b/platform/microvm-runner/rbac.yaml new file mode 100644 index 0000000..a560c8b --- /dev/null +++ b/platform/microvm-runner/rbac.yaml @@ -0,0 +1,35 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: microvm-runner-controller + namespace: microvm-runner +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: gitea-dynamic-runner + namespace: microvm-runner +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: microvm-runner-controller + namespace: microvm-runner +rules: + - apiGroups: [""] + resources: [pods] + verbs: [create, get, patch, delete] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: microvm-runner-controller + namespace: microvm-runner +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: microvm-runner-controller +subjects: + - kind: ServiceAccount + name: microvm-runner-controller + namespace: microvm-runner diff --git a/platform/microvm-runner/service.yaml b/platform/microvm-runner/service.yaml new file mode 100644 index 0000000..87f9f0f --- /dev/null +++ b/platform/microvm-runner/service.yaml @@ -0,0 +1,12 @@ +apiVersion: v1 +kind: Service +metadata: + name: microvm-runner-controller + namespace: microvm-runner +spec: + selector: + app.kubernetes.io/name: microvm-runner-controller + ports: + - name: http + port: 8787 + targetPort: http diff --git a/platform/nats/README.md b/platform/nats/README.md index caabc9d..68dee54 100644 --- a/platform/nats/README.md +++ b/platform/nats/README.md @@ -35,10 +35,11 @@ ci_worker_password ## CI stream 约定 -controller 首次启动时幂等创建 `CI_RUNNER` stream:`ci.runner.*`、 +controller 首次启动时幂等创建 `CI_RUNNER` stream:`ci.runner.>`、 `WorkQueuePolicy`、file storage、24h/10000 条/256 MiB 上限。每类 runner 使用独立 -subject 和 durable pull consumer;同类型的多个 worker 共享 durable consumer。 -ACK 后消息立即删除,不保存 CI 历史。 +subject 和 durable pull consumer;`ci.runner..binding` 传递 runner 实际 +领取任务后的身份绑定。同类型的多个 worker 共享 durable consumer。ACK 后消息立即 +删除,不保存 CI 历史。 ## 验证