diff --git a/clusters/sandbox/README.md b/clusters/sandbox/README.md index 8559e17..cecfba0 100644 --- a/clusters/sandbox/README.md +++ b/clusters/sandbox/README.md @@ -15,7 +15,8 @@ Root bootstrap 已完成。后续按依赖顺序分别引入: 1. 监控 CRD、kube-state-metrics 以及 kubelet/cAdvisor 抓取配置; 2. SPIRE Agent、SPIFFE CSI Driver 与 workload registration; 3. Kata Containers、`block-plain` RuntimeClass; -4. OpenSandbox controller/server;CI Pool 与 runner 调度器随后独立接入。 +4. 独立 External Secrets Operator 与 sandbox 专用 OpenBao auth backend; +5. OpenSandbox controller/server;CI Pool 与 runner 调度器随后独立接入。 每一阶段单独合并并等待对应 Flux Kustomization Ready,不在 bootstrap 时一次性部署。 第一阶段监控拆为 `monitoring-operator` 与依赖它的 `monitoring`,防止 VM CR 在 @@ -54,10 +55,12 @@ Kata 阶段使用官方 4.1.0 `kata-deploy` chart 的短生命周期 `job` 模 `emptyDir` 固定使用 `block-plain`,为 Docker/BuildKit overlay2 与 kind 提供 guest 内块设备文件系统。详细限制与上线验收见 `platform/sandbox-kata/README.md`。 +Sandbox 的 ESO 通过独立 `auth/kubernetes-sandbox` 向 OpenBao 证明 ServiceAccount +身份,只能读取共享的 `kv/k8s/opensandbox-api`。它不保存 reviewer JWT 或长期 Bao token;相关 +Terraform 与 Flux 边界见 `platform/sandbox-external-secrets/README.md`。 + OpenSandbox 阶段固定官方源码 commit 与 umbrella chart `0.2.2`,只部署 controller、 -ClusterIP server 和 CRD。server 当前仅能从集群内部访问;在 CI 调度器接入并建立 -API key 的 Secret 生命周期前,显式运行于无认证 bootstrap 模式。这个临时边界和切换 -步骤记录在 `platform/sandbox-opensandbox/README.md`。 +ClusterIP server 和 CRD。API key 由 ESO 从 OpenBao 投影,明文不进入 Git。 ## 监控边界 diff --git a/clusters/sandbox/apps/external-secrets-operator.yaml b/clusters/sandbox/apps/external-secrets-operator.yaml new file mode 100644 index 0000000..a16f115 --- /dev/null +++ b/clusters/sandbox/apps/external-secrets-operator.yaml @@ -0,0 +1,17 @@ +--- +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: external-secrets-operator + namespace: flux-system +spec: + dependsOn: + - name: monitoring-operator + interval: 10m + path: ./platform/sandbox-external-secrets/operator + prune: true + sourceRef: + kind: GitRepository + name: flux-system + timeout: 10m + wait: true diff --git a/clusters/sandbox/apps/external-secrets.yaml b/clusters/sandbox/apps/external-secrets.yaml new file mode 100644 index 0000000..7384ab3 --- /dev/null +++ b/clusters/sandbox/apps/external-secrets.yaml @@ -0,0 +1,17 @@ +--- +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: external-secrets + namespace: flux-system +spec: + dependsOn: + - name: external-secrets-operator + interval: 10m + path: ./platform/sandbox-external-secrets/config + prune: true + sourceRef: + kind: GitRepository + name: flux-system + timeout: 10m + wait: true diff --git a/clusters/sandbox/apps/opensandbox.yaml b/clusters/sandbox/apps/opensandbox.yaml index 59b26ae..e051506 100644 --- a/clusters/sandbox/apps/opensandbox.yaml +++ b/clusters/sandbox/apps/opensandbox.yaml @@ -6,6 +6,7 @@ metadata: namespace: flux-system spec: dependsOn: + - name: external-secrets - name: kata - name: monitoring-operator interval: 10m diff --git a/clusters/sandbox/kustomization.yaml b/clusters/sandbox/kustomization.yaml index e23a3f3..10118f1 100644 --- a/clusters/sandbox/kustomization.yaml +++ b/clusters/sandbox/kustomization.yaml @@ -7,4 +7,6 @@ resources: - apps/spire-bootstrap.yaml - apps/spire-agents.yaml - apps/kata.yaml + - apps/external-secrets-operator.yaml + - apps/external-secrets.yaml - apps/opensandbox.yaml diff --git a/infrastructure/openbao/README.md b/infrastructure/openbao/README.md index 7eeeb60..c807bab 100644 --- a/infrastructure/openbao/README.md +++ b/infrastructure/openbao/README.md @@ -51,7 +51,7 @@ few things Terraform must not own.** | secrets engine mounts (`kv`, `pki`, `ssh-client-signer`) | the daemon, Raft, TLS files, systemd | | PKI role, issuing/CRL URLs, cluster paths, **ACME** | `bao operator init` / unseal (manual, PGP-wrapped) | | SSH signing role (`ai-agent`) | **PKI root CA + SSH CA signing key** | -| OIDC auth *mount* and *role* | OIDC **client secret** (`auth/oidc/config`) | +| OIDC/Kubernetes auth mount、config 与 role | OIDC **client secret** (`auth/oidc/config`) | | all policies | snapshot token + script + systemd timer | | | host-level CA trust distribution (`openbao_ssh_ca_trust`) | @@ -92,6 +92,13 @@ then `VAULT_ADDR`/`VAULT_TOKEN`), mirroring how `smtp-relay/terraform` uses native `openbao/openbao` provider is published only to the OpenTofu registry and cannot be resolved by the HashiCorp `terraform` CLI. +Sandbox 集群使用独立的 `auth/kubernetes-sandbox`。其 API 地址、公开 Kubernetes CA、 +ESO role 与只读 `kv/k8s/opensandbox-api` policy 全部由 Terraform 管理;CA 位于 +`terraform/certs/sandbox-kubernetes-ca.crt`。集群重建并轮换 CA 后,先更新该文件并 +apply,再让 Flux 恢复 ESO reconciliation。该 backend 不保存 reviewer JWT,而是使用 +ESO 的短期登录 JWT 执行 TokenReview。该组资源已于 2026-09-18 apply,随后复验 plan +为 zero-diff。 + ## DNS `bao.ad.ddupan.top` is an **internal-only** name — not a public Cloudflare record and diff --git a/infrastructure/openbao/terraform/auth.tf b/infrastructure/openbao/terraform/auth.tf index 869a981..aae9bfc 100644 --- a/infrastructure/openbao/terraform/auth.tf +++ b/infrastructure/openbao/terraform/auth.tf @@ -59,3 +59,33 @@ resource "vault_kubernetes_auth_backend_role" "external_secrets" { # in a long TTL and every extra hour is a longer-lived credential in memory. token_ttl = 3600 } + +# A Kubernetes auth mount can validate identities from only the API server it is +# configured against. The sandbox cluster therefore cannot reuse auth/kubernetes, +# whose TokenReview endpoint belongs to homelab. +resource "vault_auth_backend" "sandbox_kubernetes" { + type = "kubernetes" + path = "kubernetes-sandbox" +} + +resource "vault_kubernetes_auth_backend_config" "sandbox" { + backend = vault_auth_backend.sandbox_kubernetes.path + kubernetes_host = "https://10.60.0.13:6443" + kubernetes_ca_cert = file("${path.module}/certs/sandbox-kubernetes-ca.crt") + disable_local_ca_jwt = true + + # Deliberately omit token_reviewer_jwt. OpenBao uses the login JWT for + # TokenReview; the sandbox external-secrets ServiceAccount is bound only to + # system:auth-delegator and all issued JWTs remain short-lived. +} + +resource "vault_kubernetes_auth_backend_role" "sandbox_external_secrets" { + backend = vault_auth_backend.sandbox_kubernetes.path + role_name = "external-secrets" + + bound_service_account_names = ["external-secrets"] + bound_service_account_namespaces = ["external-secrets"] + + token_policies = [vault_policy.sandbox_external_secrets.name] + token_ttl = 3600 +} diff --git a/infrastructure/openbao/terraform/certs/sandbox-kubernetes-ca.crt b/infrastructure/openbao/terraform/certs/sandbox-kubernetes-ca.crt new file mode 100644 index 0000000..82f619e --- /dev/null +++ b/infrastructure/openbao/terraform/certs/sandbox-kubernetes-ca.crt @@ -0,0 +1,10 @@ +-----BEGIN CERTIFICATE----- +MIIBdjCCAR2gAwIBAgIBADAKBggqhkjOPQQDAjAjMSEwHwYDVQQDDBhrM3Mtc2Vy +dmVyLWNhQDE3ODk2NTI4NTcwHhcNMjYwOTE3MTI0NzM3WhcNMzYwOTE0MTI0NzM3 +WjAjMSEwHwYDVQQDDBhrM3Mtc2VydmVyLWNhQDE3ODk2NTI4NTcwWTATBgcqhkjO +PQIBBggqhkjOPQMBBwNCAAR4SbqzTXZnlZdUPz7viN6+dYbB1Maw44Qiepn9r5XG +sOzkYkN8t1aG3Ugo8TqQ3xJaKkM89n1Rluj0vbOhiNajo0IwQDAOBgNVHQ8BAf8E +BAMCAqQwDwYDVR0TAQH/BAUwAwEB/zAdBgNVHQ4EFgQUyLSGoKAKAJuiniuRdBLG +XYaDQC8wCgYIKoZIzj0EAwIDRwAwRAIgFkVzyUZexk/ynnxBEOg+3foJv3WKqAei +hTSRjO1gL0UCIFbBKR7BMrJJAgW3DJFeeBM+b+tTg93jNx55qZACbFOL +-----END CERTIFICATE----- diff --git a/infrastructure/openbao/terraform/policies.tf b/infrastructure/openbao/terraform/policies.tf index 9010a21..75c2017 100644 --- a/infrastructure/openbao/terraform/policies.tf +++ b/infrastructure/openbao/terraform/policies.tf @@ -35,3 +35,10 @@ resource "vault_policy" "external_secrets" { name = "external-secrets" policy = file("${path.module}/policies/external-secrets.hcl") } + +# The sandbox cluster has its own Kubernetes auth backend and a deliberately +# narrower KV view than the homelab ESO instance. +resource "vault_policy" "sandbox_external_secrets" { + name = "sandbox-external-secrets" + policy = file("${path.module}/policies/sandbox-external-secrets.hcl") +} diff --git a/infrastructure/openbao/terraform/policies/sandbox-external-secrets.hcl b/infrastructure/openbao/terraform/policies/sandbox-external-secrets.hcl new file mode 100644 index 0000000..706276e --- /dev/null +++ b/infrastructure/openbao/terraform/policies/sandbox-external-secrets.hcl @@ -0,0 +1,9 @@ +# Read only the shared OpenSandbox control-plane API key. The same Bao object is +# consumed by the server in sandbox and, later, by the scheduler in homelab. +path "kv/data/k8s/opensandbox-api" { + capabilities = ["read"] +} + +path "kv/metadata/k8s/opensandbox-api" { + capabilities = ["read"] +} diff --git a/infrastructure/sandbox-cluster/README.md b/infrastructure/sandbox-cluster/README.md index 1fde90e..e0bfeaa 100644 --- a/infrastructure/sandbox-cluster/README.md +++ b/infrastructure/sandbox-cluster/README.md @@ -18,7 +18,8 @@ Flux 管理以下 Kubernetes 资源: - Kata Containers 和 CI 专用的 `block-plain` RuntimeClass; - SPIRE Agent、SPIFFE CSI Driver 与 workload identity 声明; - vmagent、kube-state-metrics、kubelet/cAdvisor scrape 配置和告警; -- OpenSandbox controller/server;CI Pool 与 runner 调度器由 runner 项目接入。 +- sandbox External Secrets Operator、OpenSandbox controller/server;CI Pool 与 runner + 调度器由 runner 项目接入。 同一个对象只能有一个 owner。Ansible 不直接部署上述集群内 workload;Flux 不管理 LXC、K3s datastore 或 K3s 本身。 diff --git a/platform/sandbox-external-secrets/README.md b/platform/sandbox-external-secrets/README.md new file mode 100644 index 0000000..027972d --- /dev/null +++ b/platform/sandbox-external-secrets/README.md @@ -0,0 +1,53 @@ +# Sandbox External Secrets Operator + +本目录在 sandbox 集群部署独立的 External Secrets Operator `2.8.0`,并通过 +`ClusterSecretStore/openbao` 读取 OpenBao KV v2 中共享的 OpenSandbox API key。它不复用 +homelab 集群的 ESO Pod、ServiceAccount 或 Kubernetes auth backend。 + +## 当前状态 + +- OpenBao `auth/kubernetes-sandbox`、backend config、`external-secrets` role 与 + `sandbox-external-secrets` policy 已于 2026-09-18 由 Terraform 创建;apply 后 plan + 为 zero-diff; +- `kv/k8s/opensandbox-api` 已由本机 `spiffe://ddupan.top/dev/panxiao81` 身份生成并写入, + 值未输出或落盘; +- sandbox ESO operator、`ClusterSecretStore/openbao` 与 OpenSandbox `ExternalSecret` + 由 Flux 管理; +- homelab runner 对同一 key 的投影不在本目录,留给 runner 项目管理。 + +OpenBao 的 `auth/kubernetes-sandbox`、对应 role、policy、sandbox API 地址与公开 CA +完全由 `infrastructure/openbao/terraform/` 管理。CA 文件提交到 Git 是刻意设计:CA +是公开信任材料,版本化后集群重建造成的 trust root 变化会产生可审计的 Terraform diff。 + +ESO 使用 TokenRequest 生成短期 ServiceAccount JWT。OpenBao 未配置长期 +`token_reviewer_jwt`,而是使用登录 JWT 调用 sandbox TokenReview;因此 +`external-secrets` ServiceAccount 仅额外绑定内建 `system:auth-delegator`。 + +## 重建顺序 + +1. 在 OpenBao 写入 OpenSandbox API key: + + ```bash + openssl rand -hex 32 | bao kv put kv/k8s/opensandbox-api api_key=- + ``` + +2. 在 `infrastructure/openbao/terraform` 执行 `terraform plan` 和 `terraform apply`, + 创建 `kubernetes-sandbox` auth mount、backend config、role 与只允许读取 + `kv/k8s/opensandbox-api` 的最小权限 policy。 +3. 合并 Flux 变更。依次等待 `flux-system/external-secrets-operator`、 + `flux-system/external-secrets` Ready,再等待 `flux-system/opensandbox` 滚动完成。 + +operator 与配置拆成两个 Flux Kustomization,确保全新集群先安装 CRD,再声明 +`ClusterSecretStore`;不要为了减少目录而把两层重新合并。 + +## 验收 + +```bash +kubectl get clustersecretstore openbao +kubectl -n opensandbox-system get externalsecret opensandbox-api-key +kubectl -n opensandbox-system get secret opensandbox-api-key +``` + +只检查 Secret 是否存在及 key 名,不输出 `data`。`ClusterSecretStore` 或 +`ExternalSecret` 不 Ready 时,先检查 `auth/kubernetes-sandbox`,不要临时创建静态 +Bao token Secret。 diff --git a/platform/sandbox-external-secrets/config/auth-delegator.yaml b/platform/sandbox-external-secrets/config/auth-delegator.yaml new file mode 100644 index 0000000..3e1cb43 --- /dev/null +++ b/platform/sandbox-external-secrets/config/auth-delegator.yaml @@ -0,0 +1,13 @@ +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: sandbox-external-secrets-token-review +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:auth-delegator +subjects: + - kind: ServiceAccount + name: external-secrets + namespace: external-secrets diff --git a/platform/sandbox-external-secrets/config/clustersecretstore.yaml b/platform/sandbox-external-secrets/config/clustersecretstore.yaml new file mode 100644 index 0000000..ec764bd --- /dev/null +++ b/platform/sandbox-external-secrets/config/clustersecretstore.yaml @@ -0,0 +1,18 @@ +--- +apiVersion: external-secrets.io/v1 +kind: ClusterSecretStore +metadata: + name: openbao +spec: + provider: + vault: + server: https://bao.ad.ddupan.top:8200 + path: kv + version: v2 + auth: + kubernetes: + mountPath: kubernetes-sandbox + role: external-secrets + serviceAccountRef: + name: external-secrets + namespace: external-secrets diff --git a/platform/sandbox-external-secrets/config/kustomization.yaml b/platform/sandbox-external-secrets/config/kustomization.yaml new file mode 100644 index 0000000..370afb5 --- /dev/null +++ b/platform/sandbox-external-secrets/config/kustomization.yaml @@ -0,0 +1,6 @@ +--- +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - auth-delegator.yaml + - clustersecretstore.yaml diff --git a/platform/sandbox-external-secrets/operator/helmrelease.yaml b/platform/sandbox-external-secrets/operator/helmrelease.yaml new file mode 100644 index 0000000..f209acf --- /dev/null +++ b/platform/sandbox-external-secrets/operator/helmrelease.yaml @@ -0,0 +1,35 @@ +--- +apiVersion: helm.toolkit.fluxcd.io/v2 +kind: HelmRelease +metadata: + name: external-secrets + namespace: external-secrets +spec: + chart: + spec: + chart: external-secrets + interval: 1h + sourceRef: + kind: HelmRepository + name: external-secrets + version: 2.8.0 + driftDetection: + mode: enabled + install: + crds: CreateReplace + strategy: + name: RetryOnFailure + retryInterval: 5m + interval: 30m + releaseName: external-secrets + targetNamespace: external-secrets + timeout: 10m + upgrade: + crds: CreateReplace + strategy: + name: RetryOnFailure + retryInterval: 5m + valuesFrom: + - kind: ConfigMap + name: external-secrets-values + valuesKey: values.yaml diff --git a/platform/sandbox-external-secrets/operator/kustomization.yaml b/platform/sandbox-external-secrets/operator/kustomization.yaml new file mode 100644 index 0000000..75df7a4 --- /dev/null +++ b/platform/sandbox-external-secrets/operator/kustomization.yaml @@ -0,0 +1,8 @@ +--- +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - namespace.yaml + - repository.yaml + - values.yaml + - helmrelease.yaml diff --git a/platform/sandbox-external-secrets/operator/namespace.yaml b/platform/sandbox-external-secrets/operator/namespace.yaml new file mode 100644 index 0000000..591aac5 --- /dev/null +++ b/platform/sandbox-external-secrets/operator/namespace.yaml @@ -0,0 +1,5 @@ +--- +apiVersion: v1 +kind: Namespace +metadata: + name: external-secrets diff --git a/platform/sandbox-external-secrets/operator/repository.yaml b/platform/sandbox-external-secrets/operator/repository.yaml new file mode 100644 index 0000000..2452d75 --- /dev/null +++ b/platform/sandbox-external-secrets/operator/repository.yaml @@ -0,0 +1,9 @@ +--- +apiVersion: source.toolkit.fluxcd.io/v1 +kind: HelmRepository +metadata: + name: external-secrets + namespace: external-secrets +spec: + interval: 1h + url: https://charts.external-secrets.io diff --git a/platform/sandbox-external-secrets/operator/values.yaml b/platform/sandbox-external-secrets/operator/values.yaml new file mode 100644 index 0000000..e674dc5 --- /dev/null +++ b/platform/sandbox-external-secrets/operator/values.yaml @@ -0,0 +1,31 @@ +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: external-secrets-values + namespace: external-secrets +data: + values.yaml: | + replicaCount: 1 + + webhook: + replicaCount: 1 + resources: + requests: {cpu: 10m, memory: 32Mi} + limits: {memory: 128Mi} + + certController: + replicaCount: 1 + resources: + requests: {cpu: 10m, memory: 32Mi} + limits: {memory: 128Mi} + + resources: + requests: {cpu: 10m, memory: 64Mi} + limits: {memory: 256Mi} + + serviceAccount: + create: true + name: external-secrets + + installCRDs: true diff --git a/platform/sandbox-opensandbox/README.md b/platform/sandbox-opensandbox/README.md index 2aaa5fc..d04e982 100644 --- a/platform/sandbox-opensandbox/README.md +++ b/platform/sandbox-opensandbox/README.md @@ -10,14 +10,12 @@ Gateway、Ingress 或 LoadBalancer。sandbox workload 位于 `opensandbox` names 默认使用 `kata-clh-runtime-rs`;CI Pool、runner 镜像、动态 SPIFFE registration 均由 runner 项目后续声明,本目录不预制。 -## 临时认证边界 +## API 认证 -当前尚无 sandbox 集群内的 Secret 分发链路。为避免把长期凭据提交到 Git,server 暂时 -通过 `OPENSANDBOX_INSECURE_SERVER=YES` 显式确认无认证模式;其网络边界严格保持为 -ClusterIP。这不是最终认证方案。 - -runner 接入前必须先创建 `opensandbox-api-key` Secret,并把 Helm values 中的环境变量 -改为: +`ExternalSecret/opensandbox-api-key` 从 OpenBao +`kv/k8s/opensandbox-api:api_key` 投影同名 Secret。这个路径不归属于某个 Kubernetes +集群:sandbox server 与 homelab runner 调度器分别通过自己的 Bao 身份读取。server 只通过 +`secretKeyRef` 读取: ```yaml - name: OPENSANDBOX_SERVER_API_KEY @@ -27,8 +25,8 @@ runner 接入前必须先创建 `opensandbox-api-key` Secret,并把 Helm value key: api-key ``` -随后删除 `OPENSANDBOX_INSECURE_SERVER`。Secret 必须由 OpenBao/SPIFFE 派生的自动化 -链路或 Ansible 注入,不得把明文写入仓库。 +仓库与 Helm values 均不保存 API key。OpenSandbox 不支持更丰富的原生 workload +authentication;runner 后续读取同一 Bao 路径并在请求头中使用 API key。 ## 验收 diff --git a/platform/sandbox-opensandbox/external-secret.yaml b/platform/sandbox-opensandbox/external-secret.yaml new file mode 100644 index 0000000..f32cef0 --- /dev/null +++ b/platform/sandbox-opensandbox/external-secret.yaml @@ -0,0 +1,19 @@ +--- +apiVersion: external-secrets.io/v1 +kind: ExternalSecret +metadata: + name: opensandbox-api-key + namespace: opensandbox-system +spec: + refreshInterval: 1h + secretStoreRef: + kind: ClusterSecretStore + name: openbao + target: + name: opensandbox-api-key + creationPolicy: Owner + data: + - secretKey: api-key + remoteRef: + key: k8s/opensandbox-api + property: api_key diff --git a/platform/sandbox-opensandbox/kustomization.yaml b/platform/sandbox-opensandbox/kustomization.yaml index 886577f..0f2df4b 100644 --- a/platform/sandbox-opensandbox/kustomization.yaml +++ b/platform/sandbox-opensandbox/kustomization.yaml @@ -3,6 +3,7 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: - namespace.yaml + - external-secret.yaml - repository.yaml - template.yaml - values.yaml diff --git a/platform/sandbox-opensandbox/values.yaml b/platform/sandbox-opensandbox/values.yaml index ab70075..4653436 100644 --- a/platform/sandbox-opensandbox/values.yaml +++ b/platform/sandbox-opensandbox/values.yaml @@ -17,10 +17,11 @@ data: server: replicaCount: 1 env: - # 临时 bootstrap 边界:Service 仅为 ClusterIP。runner 接入时改为 - # secretKeyRef(OPENSANDBOX_SERVER_API_KEY) 并删除本项。 - - name: OPENSANDBOX_INSECURE_SERVER - value: "YES" + - name: OPENSANDBOX_SERVER_API_KEY + valueFrom: + secretKeyRef: + name: opensandbox-api-key + key: api-key resources: limits: cpu: "1"