ci: 输出 kind 节点准备调试日志

Reviewed-on: #55

.gitea/workflows/ansible.yml

@@ -6,11 +6,13 @@ on:
     branches: [main]
     paths:
       - 'infrastructure/**/ansible/**'
+      - 'infrastructure/dns/**'
       - '.ansible-lint'
       - '.gitea/workflows/ansible.yml'
   pull_request:
     paths:
       - 'infrastructure/**/ansible/**'
+      - 'infrastructure/dns/**'
       - '.ansible-lint'
       - '.gitea/workflows/ansible.yml'
 
@@ -48,6 +50,7 @@ jobs:
       - name: ansible-lint
         run: |
           export PATH="$HOME/.local/bin:$PATH"
+          export ANSIBLE_COLLECTIONS_PATH="$PWD/infrastructure/samba-ad/ansible/collections:/root/.ansible/collections"
           rc=0
           for p in infrastructure/openbao infrastructure/samba-ad infrastructure/proxmox; do
             echo "::group::$p"
@@ -55,3 +58,22 @@ jobs:
             echo "::endgroup::"
           done
           exit $rc
+
+  collection-test:
+    runs-on: self-hosted
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Install ansible-core
+        run: |
+          python3 -m pip install --user --break-system-packages \
+            --index-url https://pypi.org/simple --quiet uv==0.11.7
+          export PATH="$HOME/.local/bin:$PATH"
+          uv tool install ansible-core --quiet
+
+      - name: Run ansible-test
+        working-directory: infrastructure/samba-ad/ansible/collections/ansible_collections/ddupan/homelab
+        run: |
+          export PATH="$HOME/.local/bin:$PATH"
+          ansible-test sanity --venv --requirements --python 3.12 --color no
+          ansible-test units --venv --requirements --python 3.12 --color no

.gitea/workflows/kind-on-kata-smoke.yml

@@ -0,0 +1,63 @@
+---
+name: kind-on-kata-smoke
+
+on:
+  push:
+    branches:
+      - poc/kind-on-kata
+    paths:
+      - .gitea/workflows/kind-on-kata-smoke.yml
+  workflow_dispatch:
+
+jobs:
+  smoke:
+    runs-on: kata-poc
+    steps:
+      - name: Create nested kind cluster
+        shell: sh
+        env:
+          KIND_VERSION: v0.27.0
+          KIND_NODE_IMAGE: kindest/node:v1.32.2@sha256:f226345927d7e348497136874b6d207e0b32cc52154ad8323129352923a3142f
+        run: |
+          set -eu
+          apk add --no-cache ca-certificates curl docker-cli
+          curl --retry 5 --retry-all-errors --connect-timeout 15 -fsSLo /tmp/kind \
+            "https://kind.sigs.k8s.io/dl/${KIND_VERSION}/kind-linux-amd64"
+          curl --retry 5 --retry-all-errors --connect-timeout 15 -fsSLo /tmp/kind.sha256sum \
+            "https://kind.sigs.k8s.io/dl/${KIND_VERSION}/kind-linux-amd64.sha256sum"
+          expected="$(awk '{print $1}' /tmp/kind.sha256sum)"
+          printf '%s  %s\n' "$expected" /tmp/kind | sha256sum -c -
+          install -m 0755 /tmp/kind /usr/local/bin/kind
+          docker info --format 'kernel={{.KernelVersion}} driver={{.Driver}}'
+          test "$(docker info --format '{{.Driver}}')" = overlay2
+
+          cleanup() {
+            kind delete cluster --name nested >/dev/null 2>&1 || true
+          }
+          trap cleanup EXIT
+
+          cat >/tmp/kind-config.yaml <<'EOF'
+          kind: Cluster
+          apiVersion: kind.x-k8s.io/v1alpha4
+          name: nested
+          nodes:
+            - role: control-plane
+              extraMounts:
+                - hostPath: /dev/kmsg
+                  containerPath: /dev/kmsg
+          EOF
+          kind create cluster -v 9 --retain --config /tmp/kind-config.yaml --image "$KIND_NODE_IMAGE" --wait 5m
+          docker exec nested-control-plane kubectl \
+            --kubeconfig=/etc/kubernetes/admin.conf wait \
+            --for=condition=Ready node/nested-control-plane --timeout=2m
+          docker exec nested-control-plane kubectl \
+            --kubeconfig=/etc/kubernetes/admin.conf run smoke \
+            --image=docker.io/library/busybox:1.37 --restart=Never \
+            --command -- sh -c 'echo kind-on-kata-ok'
+          docker exec nested-control-plane kubectl \
+            --kubeconfig=/etc/kubernetes/admin.conf wait \
+            --for=jsonpath='{.status.phase}'=Succeeded pod/smoke --timeout=2m
+          test "$(docker exec nested-control-plane kubectl \
+            --kubeconfig=/etc/kubernetes/admin.conf logs smoke)" = kind-on-kata-ok
+          kind delete cluster --name nested
+          trap - EXIT

CLAUDE.md

@@ -150,6 +150,11 @@ recovered, so `.vault_pass.gpg` is the authoritative recovery path.
   all-clear. Use `git check-ignore --no-index` and `git rm --cached` to actually remove it.
 - **A `.tfplan` is a zip containing a full `tfstate`.** It walks straight past `*.tfstate`
   ignore rules. Ignore `*.tfplan` everywhere.
+- **SPIRE CLI JSON can be an array of response blocks.** `spire-agent api fetch jwt
+  -output json` in 1.15.3 returns blocks containing `svids` and `bundles`. Capture stdout
+  privately and type-check before extracting fields; `list(response)` prints full tokens
+  when the response is already an array. Never inspect credential payloads by printing
+  their containers, and never put fetched JWTs in command arguments or Pod logs.
 - **Quoting does not survive two ssh hops.** `ssh pve1 "ssh pve3 'cmd | qm monitor 103'"`
   loses the inner quotes — ssh re-joins argv with spaces, so the pipeline splits and the
   tail runs on the **jump host**. It fails silently if you discard stderr: a `screendump`

apps/seaweedfs/README.md

@@ -11,7 +11,9 @@
 | `helm.sh` | Installs or upgrades the SeaweedFS release. |
 
 **Install**
-1. Set real S3 access and secret keys in `values.yaml`.
+1. 在 OpenBao `kv/k8s/seaweedfs-s3` 维护基础 S3 配置；zot 凭据单独以
+   `kv/k8s/zot-s3` 为唯一来源。ESO 合成为 `seaweedfs-s3-config`，详见下文。
+   不要把真实 AK/SK 放进 `values.yaml`。
 2. Apply the manifests:
    ```bash
    bash ~/services/apps/seaweedfs/helm.sh
@@ -31,4 +33,22 @@
 
 **Notes**
 - The chart manages master, volume, filer, S3, and admin components.
-- The chart-managed S3 secret uses the current AK/SK pair for the admin user.
+- The filer uses the ESO-managed `seaweedfs-s3-config` Secret for static S3 identities.
+
+## zot 制品存储
+
+`zot` bucket 专用于 [zot Registry](../zot/README.md)，OCI 数据位于 `registry/`
+前缀。静态身份 `zot` 只有该 bucket 的 Read/Write/List/Tagging 权限，凭据唯一来源为
+Bao `kv/k8s/zot-s3` 的 `access_key` / `secret_key`，同时供 zot consumer 和
+SeaweedFS 服务端使用。
+
+[ExternalSecret 模板](../../platform/external-secrets/externalsecrets.yaml) 保留
+`kv/k8s/seaweedfs-s3` 的原有身份及其他配置，再追加 zot 身份与限定 bucket 的权限。
+基础配置当前版本不保存 zot AK/SK；旧 KV 版本历史仍保留。新增其他身份时使用
+KV compare-and-set 保留已有内容，不覆盖 Terraform 或其他应用的 AK/SK。
+不要直接编辑生成的 Kubernetes Secret。该 ExternalSecret 已单独应用到集群，
+目前仍未加入 ESO 的 Flux Kustomization，遵循该组件现有 ownership 边界。
+
+运行版本 `4.22` 可在 Secret volume 更新后向 filer/内嵌 S3 的 `weed` 进程发送
+SIGHUP，重新加载静态配置，无需重启共享 S3 服务。本次接入没有启用 SeaweedFS
+OIDC/STS；SPIRE 认证发生在 zot 的客户端入口。

apps/zot/README.md

@@ -0,0 +1,123 @@
+# zot OCI Registry
+
+内网入口为 `https://zot.ad.ddupan.top`。使用官方 Helm chart `0.1.124`，运行
+zot `v2.1.21`，镜像固定到官方 linux/amd64 digest。
+
+## 存储与凭据
+
+制品、manifest 和 OCI layout 保存在现有 SeaweedFS 的 `zot` bucket，前缀为
+`registry/`，S3 endpoint 为 `https://s3.ad.ddupan.top`。**不创建 PVC**；chart 的
+`/var/lib/registry` 是 `emptyDir`，仅用于运行时本地工作数据。
+
+首期单副本，关闭跨仓库 dedupe，不额外部署 Redis/DynamoDB 缓存。保留 zot GC，
+暂不配置自动删除已发布版本的 retention policy。增加副本、启用 dedupe 或搜索等
+扩展前，需要重新检查共享元数据与缓存的持久化要求。
+
+凭据链路：
+
+```text
+OpenBao kv/k8s/seaweedfs-s3
+  → 原有 S3 身份及基础配置 ─┐
+                         ├→ ESO 模板 → seaweedfs/seaweedfs-s3-config
+OpenBao kv/k8s/zot-s3 ────┘                 → 完整 s3.config
+  └→ ESO → zot/zot-s3 → zot 的 AWS_ACCESS_KEY_ID / AWS_SECRET_ACCESS_KEY
+
+```
+
+`kv/k8s/zot-s3` 是 zot AK/SK 的唯一维护来源。基础配置保留原有身份及其他字段，
+不再保存 zot 凭据副本；[SeaweedFS ExternalSecret](../../platform/external-secrets/externalsecrets.yaml)
+使用 ESO v2 模板追加 zot 身份。两个 Kubernetes Secret 都是自动生成的消费副本，
+不手工编辑。Bao 的旧版本历史保留，回滚基础配置时模板也会替换其中的旧 zot 身份。
+
+专用 S3 身份只有 `Read:zot`、`Write:zot`、`List:zot`、`Tagging:zot`，不能读取
+Terraform 的 `tfstate` bucket。`kv/k8s/zot-s3` 的字段是 `access_key` 和
+`secret_key`。AK/SK 不进入 Git、Helm values 或 CI；这里仍是静态 S3 凭据，尚未
+接入 SPIRE/STS。
+
+本次归一没有轮换密钥，生成的完整配置与归一前语义一致。当前模板只有一组 zot
+凭据，尚未实现新旧密钥重叠轮换。后续轮换只修改 `kv/k8s/zot-s3`，但仍需协调
+两个 ExternalSecret 同步：确认 SeaweedFS Secret volume 更新后向 filer 的
+`weed` 进程发送 SIGHUP，再确认 zot Secret 更新并重启 zot（环境变量不会热更新）。
+两端异步更新期间可能短暂认证失败；需要无中断轮换时先扩展模板支持新旧凭据重叠。
+
+## SPIRE 认证和授权
+
+| 参数 | 值 |
+|---|---|
+| issuer | `https://spire-oidc.ad.ddupan.top` |
+| JWT audience | `zot` |
+| subject | `spiffe://ddupan.top/` 下的 workload SPIFFE ID |
+| token endpoint | `https://zot.ad.ddupan.top/zot/auth/token` |
+| 当前权限 | 受信身份可以读取所有仓库；没有常驻写入或删除授权 |
+
+zot 通过已配置的 issuer discovery/JWKS 验证 JWT-SVID，再以 `sub` 作为授权身份。
+不接受任意 issuer，不关闭 TLS/issuer 验证。新的 Kata CI 负责取得并更新自己的
+JWT-SVID；确认其身份命名后，再添加针对具体 repository 的 `create`/`update`
+授权，不能把整个 trust domain 都授予写权限。
+
+现阶段拉取也需要 JWT-SVID。原定内网匿名拉取尚未启用：zot `v2.1.21` 的
+OIDC Bearer middleware 会在授权阶段之前拒绝无 token 请求，单独增加
+`anonymousPolicy` 无法解决。匿名读取与 SPIRE 写入共存需后续单独验证方案。
+
+已有 SPIRE 身份的进程可以通过 Workload API 获取 `aud=zot` 的 JWT-SVID，然后
+通过 `docker login` 或 `crane auth login` 的 `--password-stdin` 交给 Registry。
+用户名可以使用 `zot`，实际权限取自已验证 JWT 的身份。使用独立、权限为 `0700`
+的临时 `DOCKER_CONFIG`，结束后删除；不要开启 shell tracing，不要打印 token，
+不要把 token 放进命令参数。token 接口不会延长 SVID 有效期。
+
+## 部署与网络
+
+- 官方 chart 管理 Deployment、Service、ConfigMap 和 HTTPRoute。
+- `persistence: false`，Service 为 ClusterIP，TLS 由已有 Envoy Gateway 的
+  `https` listener 与内网通配符证书终止。
+- 仅配置 Samba AD 内网 DNS；不创建公网 DNS 或 Cloudflare Tunnel route。
+- NetworkPolicy 只允许现有 Envoy Gateway 数据面访问 zot 的 5000 端口。
+- HTTPRoute 只暴露 `/v2/` 和 `/zot/auth/token`，不暴露内部健康检查或管理端点。
+- namespace 使用 restricted PodSecurity，容器非 root、只读根文件系统。
+
+首次已按用户授权从本地执行 `kubectl apply -k apps/zot`，由集群 Helm controller
+安装。`clusters/homelab/apps/zot.yaml` 是 GitOps composition；对应文件合并进入
+Flux 跟踪分支后，才由根 Kustomization 持续管理，不能把未提交的本地部署写成
+已完成 Git 接管。
+
+检查与渲染：
+
+```bash
+helm template zot --repo https://zotregistry.dev/helm-charts \
+  --version 0.1.124 --namespace zot -f apps/zot/values.yaml --skip-tests
+sudo k3s kubectl -n zot get helmrelease,pods,externalsecret,httproute
+sudo k3s kubectl -n zot get pvc
+```
+
+上游 chart 的 Helm test Pod 不满足本 namespace 的 restricted 策略，也没有
+SPIRE 凭据，因此不运行默认 `helm test`；使用下述真实身份验收。
+
+## 验收与恢复
+
+验收使用独立临时 Pod，通过 SPIFFE CSI socket 和真实 Workload API 取得 JWT-SVID，
+没有修改现有 runner。仅在初始化 `verification/smoke:spire-s3` 测试镜像时临时
+授予该测试身份针对该仓库的写权限；完成后必须撤回 HelmRelease override，并删除
+临时 Pod、ServiceAccount 与 ClusterSPIFFEID。
+
+验收项目：有效 SVID + crane pull、manifest digest 一致、错误 audience、错误
+signature、过期 token、无凭据写入、跨仓库写入、只读身份写入和删除拒绝；另外检查
+Pod 重建后镜像仍可拉取，以及 S3 身份不能访问 `tfstate`。
+
+2026-09-14 已完成上述验收：HelmRelease Ready、HTTPRoute Accepted/ResolvedRefs，
+DNS 第二次 Ansible check 为 `changed=0`；一分钟真实 JWT-SVID 到期后返回 401。
+临时写权限已移除。测试镜像可供后续 CI 验证拉取：
+
+```text
+zot.ad.ddupan.top/verification/smoke:spire-s3
+sha256:b8d3b977a1235022759470903dab4a46b7cf8107958624f1f76a323eabe37c5e
+```
+
+它是仅含验证文本的 OCI 测试镜像，没有可执行入口，不用于运行服务。
+
+Registry 恢复需要完整的 SeaweedFS bucket 数据、Bao 专用凭据和此目录配置。
+zot 的临时目录不是制品备份。独立异机/离线备份尚未在本次部署中建立；不能把同一
+SeaweedFS 内的数据副本当作独立灾备。重装 zot 不得删除 `zot` bucket。
+
+参考：[官方 Kubernetes 安装](https://zotregistry.dev/v2.1.21/install-guides/install-guide-k8s/)、
+[S3 存储](https://zotregistry.dev/v2.1.21/articles/storage/)、
+[OIDC workload identity](https://github.com/project-zot/zot/blob/v2.1.21/examples/README-OIDC-WORKLOAD-IDENTITY.md)。

apps/zot/external-secret.yaml

@@ -0,0 +1,22 @@
+apiVersion: external-secrets.io/v1
+kind: ExternalSecret
+metadata:
+  name: zot-s3
+  namespace: zot
+spec:
+  refreshInterval: 1h
+  secretStoreRef:
+    kind: ClusterSecretStore
+    name: openbao
+  target:
+    name: zot-s3
+    creationPolicy: Owner
+  data:
+    - secretKey: access_key
+      remoteRef:
+        key: k8s/zot-s3
+        property: access_key
+    - secretKey: secret_key
+      remoteRef:
+        key: k8s/zot-s3
+        property: secret_key

apps/zot/helmrelease.yaml

@@ -0,0 +1,30 @@
+apiVersion: helm.toolkit.fluxcd.io/v2
+kind: HelmRelease
+metadata:
+  name: zot
+  namespace: zot
+spec:
+  chart:
+    spec:
+      chart: zot
+      version: 0.1.124
+      interval: 1h
+      sourceRef:
+        kind: HelmRepository
+        name: zot
+  releaseName: zot
+  interval: 30m
+  timeout: 5m
+  driftDetection:
+    mode: enabled
+  install:
+    strategy:
+      name: RetryOnFailure
+      retryInterval: 5m
+  upgrade:
+    strategy:
+      name: RetryOnFailure
+      retryInterval: 5m
+  valuesFrom:
+    - kind: ConfigMap
+      name: zot-values

apps/zot/helmrepository.yaml

@@ -0,0 +1,8 @@
+apiVersion: source.toolkit.fluxcd.io/v1
+kind: HelmRepository
+metadata:
+  name: zot
+  namespace: zot
+spec:
+  interval: 1h
+  url: https://zotregistry.dev/helm-charts

apps/zot/kustomization.yaml

@@ -0,0 +1,18 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - namespace.yaml
+  - serviceaccount.yaml
+  - external-secret.yaml
+  - helmrepository.yaml
+  - helmrelease.yaml
+  - networkpolicy.yaml
+generatorOptions:
+  disableNameSuffixHash: true
+  labels:
+    reconcile.fluxcd.io/watch: Enabled
+configMapGenerator:
+  - name: zot-values
+    namespace: zot
+    files:
+      - values.yaml=values.yaml

apps/zot/namespace.yaml

@@ -0,0 +1,7 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: zot
+  labels:
+    pod-security.kubernetes.io/enforce: restricted
+    pod-security.kubernetes.io/enforce-version: v1.36

apps/zot/networkpolicy.yaml

@@ -0,0 +1,22 @@
+apiVersion: networking.k8s.io/v1
+kind: NetworkPolicy
+metadata:
+  name: zot-ingress
+  namespace: zot
+spec:
+  podSelector:
+    matchLabels:
+      app.kubernetes.io/name: zot
+  policyTypes: [Ingress]
+  ingress:
+    - from:
+        - namespaceSelector:
+            matchLabels:
+              kubernetes.io/metadata.name: envoy-gateway-system
+          podSelector:
+            matchLabels:
+              gateway.envoyproxy.io/owning-gateway-name: eg
+              gateway.envoyproxy.io/owning-gateway-namespace: envoy-gateway-system
+      ports:
+        - protocol: TCP
+          port: 5000

apps/zot/serviceaccount.yaml

@@ -0,0 +1,6 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: zot
+  namespace: zot
+automountServiceAccountToken: false

apps/zot/values.yaml

@@ -0,0 +1,147 @@
+# 官方 chart 0.1.124 / zot v2.1.21；制品与 manifests 保存在 SeaweedFS S3。
+# persistence=false 仅保留 chart 的 emptyDir，不创建 PVC。
+# 首期关闭跨仓库 dedupe，不额外引入 Redis/DynamoDB 持久缓存。
+replicaCount: 1
+image:
+  repository: ghcr.io/project-zot/zot
+  tag: v2.1.21@sha256:8258443838e95989c13c891f78a02bc1c391b5a00591ffef24cb8c17cde28038
+persistence: false
+strategy:
+  type: Recreate
+serviceAccount:
+  create: false
+  name: zot
+service:
+  type: ClusterIP
+  port: 5000
+mountConfig: true
+mountSecret: false
+secretFiles: {}
+configFiles:
+  config.json: |
+    {
+      "distSpecVersion": "1.1.1",
+      "storage": {
+        "rootDirectory": "/var/lib/registry",
+        "dedupe": false,
+        "gc": true,
+        "gcDelay": "24h",
+        "gcInterval": "24h",
+        "storageDriver": {
+          "name": "s3",
+          "region": "us-east-1",
+          "regionendpoint": "https://s3.ad.ddupan.top",
+          "bucket": "zot",
+          "rootdirectory": "/registry",
+          "secure": true,
+          "skipverify": false,
+          "forcepathstyle": true
+        }
+      },
+      "http": {
+        "address": "0.0.0.0",
+        "port": "5000",
+        "externalUrl": "https://zot.ad.ddupan.top",
+        "compat": [
+          "docker2s2"
+        ],
+        "auth": {
+          "bearer": {
+            "realm": "https://zot.ad.ddupan.top/zot/auth/token",
+            "service": "zot.ad.ddupan.top",
+            "oidc": [
+              {
+                "issuer": "https://spire-oidc.ad.ddupan.top",
+                "audiences": [
+                  "zot"
+                ],
+                "claimMapping": {
+                  "username": "claims.sub",
+                  "validations": [
+                    {
+                      "expression": "claims.sub.startsWith('spiffe://ddupan.top/')",
+                      "message": "SPIFFE trust domain mismatch"
+                    }
+                  ]
+                }
+              }
+            ]
+          }
+        },
+        "accessControl": {
+          "repositories": {
+            "**": {
+              "defaultPolicy": [
+                "read"
+              ]
+            }
+          }
+        }
+      },
+      "log": {
+        "level": "info"
+      }
+    }
+env:
+- name: AWS_ACCESS_KEY_ID
+  valueFrom:
+    secretKeyRef:
+      name: zot-s3
+      key: access_key
+- name: AWS_SECRET_ACCESS_KEY
+  valueFrom:
+    secretKeyRef:
+      name: zot-s3
+      key: secret_key
+- name: AWS_EC2_METADATA_DISABLED
+  value: 'true'
+podSecurityContext:
+  runAsNonRoot: true
+  runAsUser: 10001
+  runAsGroup: 10001
+  fsGroup: 10001
+  seccompProfile:
+    type: RuntimeDefault
+securityContext:
+  allowPrivilegeEscalation: false
+  readOnlyRootFilesystem: true
+  capabilities:
+    drop:
+    - ALL
+resources:
+  requests:
+    cpu: 100m
+    memory: 128Mi
+  limits:
+    cpu: '1'
+    memory: 512Mi
+extraVolumes:
+- name: tmp
+  emptyDir:
+    sizeLimit: 128Mi
+extraVolumeMounts:
+- name: tmp
+  mountPath: /tmp
+startupProbe:
+  initialDelaySeconds: 5
+  periodSeconds: 5
+  failureThreshold: 60
+httproute:
+  enabled: true
+  parentRefs:
+  - name: eg
+    namespace: envoy-gateway-system
+    sectionName: https
+  hostnames:
+  - zot.ad.ddupan.top
+  rules:
+  - matches:
+    - path:
+        type: PathPrefix
+        value: /v2/
+    - path:
+        type: Exact
+        value: /zot/auth/token
+    timeouts:
+      request: 900s
+      backendRequest: 900s

clusters/homelab/README.md

@@ -46,6 +46,10 @@ sudo k3s kubectl -n flux-system get gitrepositories,kustomizations
 - `gitea-actions`、`gitea` 与 External Secrets 已由 Flux HelmRelease 接管，Gitea 已升级到 `1.27.3`；
 - cert-manager 已固定现有 `v1.21.0` 并完成分阶段 Flux HelmRelease 接管；
 - Envoy Gateway 已固定现有 `v1.5.6` 并完成分阶段 Flux HelmRelease 接管；
-- OpenEBS 已进入分阶段接管：固定现有 `4.4.0`，HelmRelease 暂停等待存储基线验证；
+- OpenEBS 已固定现有 `4.4.0` 并完成分阶段 Flux HelmRelease 接管；
+- VictoriaMetrics Operator 已固定现有 chart `0.66.2` 并完成分阶段 Flux HelmRelease
+  接管；Metrics、Logs、Traces 与 Grafana 也已统一完成 Flux 接管；
 - External Secrets Operator 已固定 chart `2.8.0` 并完成分阶段接管；
+- SPIRE 已按官方 hardened chart `0.30.2`（SPIRE `1.15.3`）声明，使用共享
+  PostgreSQL 与独立 signing-key PVC；首次上线和 OpenBao JWT-SVID PoC 尚待合并后验证；
 - root Kustomization 与所有 brownfield 子 Kustomization 继续保持 `prune: false`。

clusters/homelab/apps/observability.yaml

@@ -0,0 +1,14 @@
+apiVersion: kustomize.toolkit.fluxcd.io/v1
+kind: Kustomization
+metadata:
+  name: observability
+  namespace: flux-system
+spec:
+  interval: 10m
+  path: ./platform/observability
+  prune: false
+  sourceRef:
+    kind: GitRepository
+    name: flux-system
+  timeout: 3m
+  wait: false

clusters/homelab/apps/spire.yaml

@@ -0,0 +1,14 @@
+apiVersion: kustomize.toolkit.fluxcd.io/v1
+kind: Kustomization
+metadata:
+  name: spire
+  namespace: flux-system
+spec:
+  interval: 10m
+  path: ./platform/spire
+  prune: false
+  sourceRef:
+    kind: GitRepository
+    name: flux-system
+  timeout: 15m
+  wait: false

clusters/homelab/apps/zot.yaml

@@ -0,0 +1,18 @@
+apiVersion: kustomize.toolkit.fluxcd.io/v1
+kind: Kustomization
+metadata:
+  name: zot
+  namespace: flux-system
+spec:
+  dependsOn:
+    - name: envoy-gateway
+    - name: external-secrets
+    - name: spire
+  interval: 10m
+  path: ./apps/zot
+  prune: false
+  sourceRef:
+    kind: GitRepository
+    name: flux-system
+  timeout: 5m
+  wait: true

clusters/homelab/kustomization.yaml

@@ -10,3 +10,6 @@ resources:
   - apps/gitea-actions.yaml
   - apps/http-echo.yaml
   - apps/openebs.yaml
+  - apps/spire.yaml
+  - apps/observability.yaml
+  - apps/zot.yaml

infrastructure/dns/README.md

@@ -0,0 +1,29 @@
+# DNS 声明与权威边界
+
+`records.yml` 是 homelab DNS 的唯一声明清单，但不是 DNS 服务本身。不同视图仍由最适合
+它们的后端提供：
+
+| 视图 | 权威或递归服务 | 配置方式 |
+|---|---|---|
+| 公网 `ddupan.top` | Cloudflare | Terraform；尚待完整导入已有记录 |
+| AD `ad.ddupan.top` | Samba internal DNS | `samba_dns_record` Ansible module |
+| LAN split horizon | Blocky | 尚待从 inventory 渲染或校验 |
+| Kubernetes Pod split horizon | CoreDNS | 尚待从 inventory 渲染或校验 |
+
+## 安全边界
+
+- Samba module 只管理 `homelab_dns.samba.records` 明确列出的 RRset，不遍历或清理 zone。
+- `_ldap`、`_kerberos`、域控制器 locator 等由 Samba 自动维护的记录不进入 inventory。
+- 一个受管 RRset 默认使用 `exact: true`：同名同类型的额外值会被删除，但其他名称和类型
+  不受影响。
+- DHCP 切换不属于本阶段。Blocky 仍未成为 LAN 客户端的正式 resolver。
+
+## 分阶段接管
+
+1. 用 Samba module 接管现有静态 A RRset，首次 check mode 应为零变更。
+2. 将 Cloudflare 已有 tunnel DNS 记录导入 Terraform state。
+3. 让 Blocky 与 CoreDNS 从 `split_horizon.records` 生成配置或执行 CI 一致性检查。
+4. 验证公网、LAN、Pod、AD 四个视图后，再单独修改 DHCP。
+
+当前 inventory 已明确暴露一个既有差异：`obj.ddupan.top` 在 Blocky 中存在，但 CoreDNS
+尚无对应覆盖。本阶段不会偷偷修复它；后续在两个 resolver 同时接管时统一修复。

infrastructure/dns/records.yml

@@ -0,0 +1,46 @@
+---
+# Homelab DNS desired state. This file is the canonical inventory; individual
+# backends consume only the views they own.
+homelab_dns:
+  samba:
+    # Samba remains authoritative for the AD zone. Only these explicitly listed
+    # RRsets are reconciled; Samba-generated AD/Kerberos records are untouched.
+    records:
+      - { zone: ad.ddupan.top, name: bao, type: A, values: [192.168.10.8] }
+      - { zone: ad.ddupan.top, name: pve1, type: A, values: [192.168.10.4] }
+      - { zone: ad.ddupan.top, name: pve2, type: A, values: [192.168.10.7] }
+      - { zone: ad.ddupan.top, name: pve3, type: A, values: [192.168.10.9] }
+      - { zone: ad.ddupan.top, name: retrolab, type: A, values: [10.60.0.10] }
+      - { zone: ad.ddupan.top, name: netbox, type: A, values: [192.168.10.127] }
+      - { zone: ad.ddupan.top, name: s3, type: A, values: [192.168.10.127] }
+      - { zone: ad.ddupan.top, name: spire-oidc, type: A, values: [192.168.10.127] }
+      - { zone: ad.ddupan.top, name: zot, type: A, values: [192.168.10.127] }
+
+  split_horizon:
+    # LAN and pod resolvers should eventually render the same set from here.
+    # Adoption of Blocky/CoreDNS is deliberately a separate change.
+    records:
+      - { name: git.ddupan.top, type: A, values: [192.168.10.127] }
+      - { name: auth.ddupan.top, type: A, values: [192.168.10.127] }
+      - { name: obj.ddupan.top, type: A, values: [192.168.10.127] }
+
+  public:
+    # Names expected at Cloudflare. Terraform adoption is a separate change;
+    # complete RRsets here make the current ownership gap explicit.
+    records:
+      - name: auth.ddupan.top
+        type: CNAME
+        values: [ff392451-b0b1-45bb-964e-6d9372c3a9e3.cfargotunnel.com]
+        proxied: true
+      - name: git.ddupan.top
+        type: CNAME
+        values: [ff392451-b0b1-45bb-964e-6d9372c3a9e3.cfargotunnel.com]
+        proxied: true
+      - name: obj.ddupan.top
+        type: CNAME
+        values: [ff392451-b0b1-45bb-964e-6d9372c3a9e3.cfargotunnel.com]
+        proxied: true
+      - name: e5renew.ddupan.top
+        type: CNAME
+        values: [ff392451-b0b1-45bb-964e-6d9372c3a9e3.cfargotunnel.com]
+        proxied: true

infrastructure/openbao/README.md

@@ -36,6 +36,7 @@ configure any secrets engines/auth methods — that is a separate bootstrap play
 ```
 terraform/                     # OpenBao's API-level CONFIGURATION (see below)
   mounts.tf  pki.tf  ssh.tf  auth.tf  policies.tf
+  auth-spire.tf                # SPIFFE JWT-SVID -> short-lived Bao tokens
   imports.tf                   # adopts the already-running instance into state
   policies/*.hcl               # policy bodies, kept diffable
 ```
@@ -205,6 +206,11 @@ after this, use `BAO_ADDR=https://bao.ad.ddupan.top:8200` (no skip-verify), not
 
 ## Using it
 
+Kubernetes workload 不接收长期 `BAO_TOKEN`：它通过 SPIRE Workload API 获取
+JWT-SVID，再经 `auth/jwt-spire/login` 换取短期、最小权限 token。完整接入流程、
+manifest、exchange 脚本、安全要求和排障方法见
+[`../../platform/spire/RUNBOOK.md`](../../platform/spire/RUNBOOK.md)。
+
 ```bash
 # human: log in via Authelia (2FA)
 bao login -method=oidc            # browser → auth.ddupan.top

infrastructure/openbao/terraform/auth-spire.tf

@@ -0,0 +1,27 @@
+# Workload authentication via SPIFFE JWT-SVIDs. The discovery document and
+# JWKS contain public verification material, so this backend carries no secret.
+resource "vault_jwt_auth_backend" "spire" {
+  path               = "jwt-spire"
+  description        = "SPIFFE JWT-SVID workload authentication"
+  oidc_discovery_url = "https://spire-oidc.ad.ddupan.top"
+  bound_issuer       = "https://spire-oidc.ad.ddupan.top"
+}
+
+# First end-to-end identity. Keep the subject exact: this role is deliberately
+# not a wildcard escape hatch for every workload in the trust domain.
+resource "vault_jwt_auth_backend_role" "spire_poc" {
+  backend   = vault_jwt_auth_backend.spire.path
+  role_name = "spire-poc"
+  role_type = "jwt"
+
+  user_claim      = "sub"
+  bound_audiences = ["openbao"]
+  bound_claims = {
+    sub = "spiffe://ddupan.top/ns/spire-poc/sa/spire-jwt-poc"
+  }
+
+  token_policies          = [vault_policy.spire_poc.name]
+  token_no_default_policy = true
+  token_ttl               = 300
+  token_max_ttl           = 900
+}

infrastructure/openbao/terraform/policies.tf

@@ -14,6 +14,11 @@ resource "vault_policy" "ai_agent_ssh" {
   policy = file("${path.module}/policies/ai-agent-ssh.hcl")
 }
 
+resource "vault_policy" "spire_poc" {
+  name   = "spire-poc"
+  policy = file("${path.module}/policies/spire-poc.hcl")
+}
+
 resource "vault_policy" "snapshot" {
   name   = "snapshot"
   policy = file("${path.module}/policies/snapshot.hcl")

infrastructure/openbao/terraform/policies/spire-poc.hcl

@@ -0,0 +1,10 @@
+# Intentionally grants no secret access. This policy proves that an exact
+# SPIFFE ID can exchange a JWT-SVID for a bounded OpenBao token and inspect or
+# revoke only that token.
+path "auth/token/lookup-self" {
+  capabilities = ["read"]
+}
+
+path "auth/token/revoke-self" {
+  capabilities = ["update"]
+}

infrastructure/samba-ad/README.md

@@ -54,6 +54,7 @@ second run is a no-op there and only reconciles config drift.
 |---|---|
 | `roles/dc_vm/` | creates the DC VM locally via libvirt + cloud-init |
 | `roles/samba_ad_dc/` | provisions the DC (§1) — idempotent |
+| `ansible/collections/ansible_collections/ddupan/homelab/` | repository-local collection containing the Samba DNS module and its `ansible-test` tests |
 | `roles/samba_ad_dc/tasks/verify.yml` | smoke tests (`--tags verify`) |
 | `roles/samba_ad_dc/tasks/legacy.yml` | opt-in retro-client protocols (§ Retro clients) |
 | `roles/windows_vm/` | unattended-installs the Windows Server 2025 admin box (§2.0) |
@@ -62,6 +63,9 @@ second run is a no-op there and only reconciles config drift.
 | `group_vars/all/vault.yml` | admin passwords (gitignored; encrypt with ansible-vault) |
 | `inventory/hosts.yml` | DC + Windows hosts |
 
+Static DNS records are declared once in `../../dns/records.yml`. The role manages
+only those RRsets and deliberately leaves Samba-generated AD locator records alone.
+
 ---
 
 ## 0. Decisions — fill these in before touching anything

infrastructure/samba-ad/ansible/ansible.cfg

@@ -1,6 +1,7 @@
 [defaults]
 inventory = inventory/hosts.yml
 roles_path = roles
+collections_paths = collections
 host_key_checking = False
 callback_result_format = yaml
 nocows = True

infrastructure/samba-ad/ansible/collections/ansible_collections/ddupan/homelab/.gitignore

@@ -0,0 +1 @@
+tests/output/

infrastructure/samba-ad/ansible/collections/ansible_collections/ddupan/homelab/README.md

@@ -0,0 +1,4 @@
+# ddupan.homelab
+
+Repository-local Ansible collection for homelab infrastructure modules. It is consumed
+directly through the `collections_paths` setting and is not published.

infrastructure/samba-ad/ansible/collections/ansible_collections/ddupan/homelab/galaxy.yml

@@ -0,0 +1,16 @@
+---
+namespace: ddupan
+name: homelab
+version: 0.1.0
+readme: README.md
+authors:
+  - panxiao81
+description: Ansible plugins used to manage the ddupan.top homelab.
+license:
+  - GPL-3.0-or-later
+tags:
+  - infrastructure
+  - dns
+repository: https://git.ddupan.top/panxiao81/homelab-infra
+build_ignore:
+  - .git

infrastructure/samba-ad/ansible/collections/ansible_collections/ddupan/homelab/plugins/modules/samba_dns_record.py

@@ -0,0 +1,159 @@
+#!/usr/bin/python
+
+# Copyright: (c) 2026, homelab-infra contributors
+# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
+
+DOCUMENTATION = r"""
+---
+module: samba_dns_record
+short_description: Reconcile one Samba internal DNS RRset
+author:
+  - Pan Xiao (@panxiao81)
+description:
+  - Reconciles only the named RRset and never deletes an undeclared zone or name.
+  - Runs C(samba-tool dns) on a Samba AD domain controller using its machine account.
+options:
+  server:
+    description: DNS server accepted by C(samba-tool dns).
+    type: str
+    required: true
+  zone:
+    description: Authoritative DNS zone.
+    type: str
+    required: true
+  name:
+    description: Record owner relative to the zone.
+    type: str
+    required: true
+  type:
+    description: DNS record type.
+    type: str
+    choices: [A, AAAA, CNAME, PTR, TXT]
+    required: true
+  values:
+    description: Complete desired value set for this owner and type.
+    type: list
+    elements: str
+    default: []
+  state:
+    description: Whether the desired RRset is present or absent.
+    type: str
+    choices: [present, absent]
+    default: present
+  exact:
+    description: Remove live values not present in C(values).
+    type: bool
+    default: true
+"""
+
+EXAMPLES = r"""
+- name: Reconcile an A RRset
+  samba_dns_record:
+    server: dc1.ad.ddupan.top
+    zone: ad.ddupan.top
+    name: pve1
+    type: A
+    values: [192.168.10.4]
+    exact: true
+"""
+
+RETURN = r"""
+before:
+  description: Values observed before reconciliation.
+  type: list
+  returned: always
+after:
+  description: Values expected after reconciliation.
+  type: list
+  returned: always
+"""
+
+import re
+
+from ansible.module_utils.basic import AnsibleModule
+
+
+ABSENT_ERRORS = (
+    "WERR_DNS_ERROR_NAME_DOES_NOT_EXIST",
+    "WERR_DNS_ERROR_RECORD_DOES_NOT_EXIST",
+    "WERR_DNS_ERROR_NXDOMAIN",
+)
+
+
+def normalize_value(record_type, value):
+    value = value.strip()
+    if record_type in ("CNAME", "PTR"):
+        return value.rstrip(".").lower()
+    if record_type == "TXT" and len(value) >= 2 and value[0] == value[-1] == '"':
+        return value[1:-1]
+    return value.lower() if record_type == "AAAA" else value
+
+
+def parse_query(stdout, record_type):
+    values = []
+    prefix = re.compile(r"^\s*%s:\s*(.*?)\s*(?:\([^)]*\))?\s*$" % re.escape(record_type))
+    for line in stdout.splitlines():
+        match = prefix.match(line)
+        if match:
+            values.append(normalize_value(record_type, match.group(1)))
+    return sorted(set(values))
+
+
+def run(module, args, check_rc=True):
+    command = [module.get_bin_path("samba-tool", required=True), "dns"] + args + ["-P"]
+    return module.run_command(command, check_rc=check_rc)
+
+
+def main():
+    module = AnsibleModule(
+        argument_spec=dict(
+            server=dict(type="str", required=True),
+            zone=dict(type="str", required=True),
+            name=dict(type="str", required=True),
+            type=dict(type="str", required=True, choices=["A", "AAAA", "CNAME", "PTR", "TXT"]),
+            values=dict(type="list", elements="str", default=[]),
+            state=dict(type="str", choices=["present", "absent"], default="present"),
+            exact=dict(type="bool", default=True),
+        ),
+        supports_check_mode=True,
+    )
+
+    p = module.params
+    desired = sorted(set(normalize_value(p["type"], value) for value in p["values"]))
+    if p["state"] == "present" and not desired:
+        module.fail_json(msg="values must not be empty when state=present")
+
+    rc, stdout, stderr = run(
+        module,
+        ["query", p["server"], p["zone"], p["name"], p["type"]],
+        check_rc=False,
+    )
+    if rc == 0:
+        current = parse_query(stdout, p["type"])
+    elif any(marker in stdout + stderr for marker in ABSENT_ERRORS):
+        current = []
+    else:
+        module.fail_json(msg="samba-tool dns query failed", rc=rc, stdout=stdout, stderr=stderr)
+
+    target = desired if p["state"] == "present" else []
+    additions = sorted(set(target) - set(current))
+    removals = sorted(set(current) - set(target)) if p["exact"] or p["state"] == "absent" else []
+    changed = bool(additions or removals)
+
+    if changed and not module.check_mode:
+        for value in removals:
+            run(module, ["delete", p["server"], p["zone"], p["name"], p["type"], value])
+        for value in additions:
+            run(module, ["add", p["server"], p["zone"], p["name"], p["type"], value])
+
+    after = sorted((set(current) - set(removals)) | set(additions))
+    module.exit_json(
+        changed=changed,
+        before=current,
+        after=after,
+        diff={"before": {p["type"]: current}, "after": {p["type"]: after}},
+    )
+
+
+if __name__ == "__main__":
+    main()

infrastructure/samba-ad/ansible/collections/ansible_collections/ddupan/homelab/tests/unit/plugins/modules/test_samba_dns_record.py

@@ -0,0 +1,23 @@
+import unittest
+
+from ansible_collections.ddupan.homelab.plugins.modules import samba_dns_record
+
+
+class ParseQueryTests(unittest.TestCase):
+    def test_parses_a_records_and_ignores_other_types(self):
+        output = """  Name=host, Records=3, Children=0
+    A: 192.168.10.4 (flags=f0, serial=1, ttl=900)
+    A: 192.168.10.7 (flags=f0, serial=2, ttl=900)
+    TXT: ignored (flags=f0, serial=3, ttl=900)
+"""
+        self.assertEqual(
+            samba_dns_record.parse_query(output, "A"),
+            ["192.168.10.4", "192.168.10.7"],
+        )
+
+    def test_normalizes_dns_targets(self):
+        output = "    CNAME: Gateway.AD.DDUPAN.TOP. (flags=f0, serial=1, ttl=900)\n"
+        self.assertEqual(
+            samba_dns_record.parse_query(output, "CNAME"),
+            ["gateway.ad.ddupan.top"],
+        )

infrastructure/samba-ad/ansible/group_vars/all/vars.yml

@@ -10,29 +10,8 @@ samba_ad_dc_ip: "192.168.10.5"
 samba_ad_dns_forwarder: "192.168.10.1"
 samba_ad_reverse_zone: "10.168.192.in-addr.arpa"   # reverse of 192.168.10.0/24
 
-# Extra A records for non-domain hosts published in the AD DNS zone.
+# Static records now live in ../../dns/records.yml and are reconciled as complete
-samba_ad_extra_a_records:
+# RRsets by the local samba_dns_record module.
-  - { name: "bao", ip: "192.168.10.8" }            # OpenBao (../openbao), not domain-joined
-  # Proxmox cluster nodes (../proxmox). Not domain-joined; they authenticate
-  # USERS against this DC rather than being members themselves.
-  - { name: "pve1", ip: "192.168.10.4" }
-  - { name: "pve2", ip: "192.168.10.7" }
-  - { name: "pve3", ip: "192.168.10.9" }
-  # Lab VMs on the SDN VNets (routed via the VyOS router, see ../../proxmox).
-  # These are NOT on 192.168.10.0/24, so they have no PTR in the existing
-  # reverse zone — forward resolution only unless a 0.60.10.in-addr.arpa zone
-  # is added later.
-  - { name: "retrolab", ip: "10.60.0.10" }
-  # k3s services exposed on the LAN through the Envoy gateway (../../../platform/envoy-gateway;
-  # Contour was retired 2026-07-25). They all point at the k3s node, which is where
-  # Envoy's LoadBalancer lands; the gateway routes by Host header and serves the
-  # *.ad.ddupan.top wildcard cert.
-  # Adding another such service = one more line here + an HTTPRoute, nothing else.
-  - { name: "netbox", ip: "192.168.10.127" }        # NetBox (../../../apps/netbox)
-  # SeaweedFS S3. Exists so Terraform state does NOT ride the Cloudflare tunnel:
-  # obj.ddupan.top works, but it hairpins through the WAN, and on 2026-07-28 that
-  # path was blackholed for hours by a dead VPN tunnel. State must stay on the LAN.
-  - { name: "s3", ip: "192.168.10.127" }            # SeaweedFS S3 (../../../apps/seaweedfs)
 
 # Support legacy clients (Win9x/NT4/2000/XP)? INSECURE — see README "Retro clients".
 samba_ad_legacy_clients: false

infrastructure/samba-ad/ansible/provision-dc.yml

@@ -6,6 +6,8 @@
   hosts: samba_dc
   become: true
   gather_facts: true
+  vars_files:
+    - ../../dns/records.yml
   roles:
     - role: samba_ad_dc
   post_tasks:

infrastructure/samba-ad/ansible/roles/samba_ad_dc/tasks/dns_records.yml

@@ -0,0 +1,14 @@
+---
+- name: Reconcile explicitly managed AD DNS RRsets
+  ddupan.homelab.samba_dns_record:
+    server: "{{ samba_ad_dc_ip }}"
+    zone: "{{ item['zone'] }}"
+    name: "{{ item['name'] }}"
+    type: "{{ item['type'] }}"
+    values: "{{ item['values'] }}"
+    state: present
+    exact: true
+  loop: "{{ homelab_dns.samba.records }}"
+  loop_control:
+    label: "{{ item['name'] }}.{{ item['zone'] }} {{ item['type'] }}"
+  tags: [dns]

infrastructure/samba-ad/ansible/roles/samba_ad_dc/tasks/main.yml

@@ -198,20 +198,11 @@
     - "'already exists' not in (kms_srv.stderr | default('')) + (kms_srv.stdout | default(''))"
   no_log: true
 
-# --- Extra A records for non-domain hosts (e.g. OpenBao) -----------------------
+# --- Directory objects and explicitly managed DNS records ----------------------
 - name: Service accounts and RBAC groups
   ansible.builtin.import_tasks: directory_objects.yml
   tags: [directory, accounts]
 
-- name: Register extra A records in the AD DNS zone
+- name: Reconcile static AD DNS records
-  ansible.builtin.command:
+  ansible.builtin.import_tasks: dns_records.yml
-    cmd: >-
-      samba-tool dns add {{ samba_ad_dc_ip }} {{ samba_ad_realm | lower }}
-      {{ item.name }} A {{ item.ip }} -P
-  loop: "{{ samba_ad_extra_a_records }}"
-  register: extra_a
-  changed_when: "'Record added successfully' in (extra_a.stdout | default(''))"
-  failed_when:
-    - extra_a.rc != 0
-    - "'already exists' not in (extra_a.stderr | default('')) + (extra_a.stdout | default(''))"
   tags: [dns]

platform/external-secrets/externalsecrets.yaml

@@ -101,6 +101,7 @@ spec:
 # The chart normally GENERATES seaweedfs-s3-secret from s3.credentials. We point
 # filer.s3.existingConfigSecret at this one instead, so the chart stops rendering
 # credentials from values entirely.
+# zot 的 AK/SK 只保存在 k8s/zot-s3；在此组装服务端配置，不在基础配置中维护副本。
 apiVersion: external-secrets.io/v1
 kind: ExternalSecret
 metadata:
@@ -114,6 +115,38 @@ spec:
   target:
     name: seaweedfs-s3-config
     creationPolicy: Owner
-  dataFrom:
+    template:
-    - extract:
+      engineVersion: v2
+      mergePolicy: Replace
+      data:
+        seaweedfs_s3_config: |-
+          {{- $config := mustFromJson .baseConfig -}}
+          {{- if not (kindIs "slice" $config.identities) -}}
+            {{- fail "base S3 configuration must contain an identities array" -}}
+          {{- end -}}
+          {{- if or (eq .zotAccessKey "") (eq .zotSecretKey "") -}}
+            {{- fail "zot S3 credentials must not be empty" -}}
+          {{- end -}}
+          {{- $identities := list -}}
+          {{- range $config.identities -}}
+            {{- if ne .name "zot" -}}
+              {{- $identities = append $identities . -}}
+            {{- end -}}
+          {{- end -}}
+          {{- $credential := dict "accessKey" .zotAccessKey "secretKey" .zotSecretKey -}}
+          {{- $zot := dict "name" "zot" "credentials" (list $credential) "actions" (list "Read:zot" "Write:zot" "List:zot" "Tagging:zot") -}}
+          {{- $_ := set $config "identities" (append $identities $zot) -}}
+          {{- mustToJson $config -}}
+  data:
+    - secretKey: baseConfig
+      remoteRef:
         key: k8s/seaweedfs-s3
+        property: seaweedfs_s3_config
+    - secretKey: zotAccessKey
+      remoteRef:
+        key: k8s/zot-s3
+        property: access_key
+    - secretKey: zotSecretKey
+      remoteRef:
+        key: k8s/zot-s3
+        property: secret_key

platform/observability/README.md

@@ -13,6 +13,34 @@ Jaeger API), Grafana (3 datasources healthy, VM dashboard loaded, Tailscale ingr
 `localpv-zfs-ceph`. Note the vmagent hot-reload RBAC fix (`metrics/reload-rbac.yaml`,
 see `metrics/README-reload.md`) and that VL/VT CRDs are served under `.../v1`.
 
+### Flux 接管状态
+
+可观测性栈使用一个 `observability` Flux Kustomization，按依赖顺序逐步加入 Operator、
+Metrics、Logs、Traces 和 Grafana，避免多个 reconciler 争夺共享 namespace、Helm source
+或 VictoriaMetrics CRD。brownfield 迁移期间保持 `prune: false`。
+
+VictoriaMetrics Operator 已固定 chart `0.66.2` / app `v0.73.1` 完成分阶段 Flux
+HelmRelease 接管，保存 values 与 `operator/values.yaml` 一致。阶段一确认 Helm revision
+保持为 2、Operator Pod UID 未变、24 个 CRD 及 6 个核心 VM/VL/VT CR 未变化；随后
+通过独立 PR 解除暂停。
+
+其余已部署资源也已统一完成接管：Metrics CR、rules、scrapes、node-exporter，Logs 的
+VLSingle 与固定 `0.3.6` 的 Collector，以及 Traces 的 VTSingle、OTel Collector，最后是
+固定 `10.5.15` 的 Grafana 与 dashboard ConfigMap。阶段一确认 11 个 Pod UID、4 个 PVC、
+6 个核心 CR、17 个 rule/scrape 对象及 dashboard 哈希均未变化，Collector 和 Grafana
+Helm revision 均保持为 1；随后在同一独立 PR 中解除两个 HelmRelease 的暂停。
+
+首次 Grafana Helm reconcile 暴露了本地 ZFS RWO 卷限制：chart 默认 `RollingUpdate`
+会先创建新 Pod，但 CSI 拒绝在旧 Pod 仍挂载 PVC 时再次 mount，使 upgrade 卡在
+`pending-upgrade`。`grafana/values.yaml` 因此显式使用 `deploymentStrategy.type: Recreate`。
+Grafana 升级会有一次短暂停机，但旧 Pod 会先退出，新 Pod 才挂载同一 PVC；不要改回
+RollingUpdate，除非存储改为真正支持并发挂载的 RWX。
+
+以下内容明确不属于本批接管：尚未部署的 kube-state-metrics、`vlogs-ingress.yaml`，以及
+仅作 standalone chart 参考的 `traces-values.yaml`。`grafana/oidc-secret.yaml` 是不含真实
+值的占位模板；live `grafana-oidc` Secret 继续只被 HelmRelease 引用，尚未由 OpenBao/ESO
+纳管。
+
 Still host-side (not yet done): deploy `docker-hosts/compose.yaml` on the Docker
 hosts and `logs/vlogs-ingress.yaml` to push their logs.
 

platform/observability/grafana-helmrepository.yaml

@@ -0,0 +1,8 @@
+apiVersion: source.toolkit.fluxcd.io/v1
+kind: HelmRepository
+metadata:
+  name: grafana
+  namespace: monitoring
+spec:
+  interval: 1h
+  url: https://grafana.github.io/helm-charts

platform/observability/grafana/helmrelease.yaml

@@ -0,0 +1,31 @@
+apiVersion: helm.toolkit.fluxcd.io/v2
+kind: HelmRelease
+metadata:
+  name: grafana
+  namespace: monitoring
+spec:
+  chart:
+    spec:
+      chart: grafana
+      interval: 1h
+      sourceRef:
+        kind: HelmRepository
+        name: grafana
+      version: 10.5.15
+  driftDetection:
+    mode: enabled
+  install:
+    strategy:
+      name: RetryOnFailure
+      retryInterval: 5m
+  interval: 30m
+  releaseName: grafana
+  targetNamespace: monitoring
+  timeout: 10m
+  upgrade:
+    strategy:
+      name: RetryOnFailure
+      retryInterval: 5m
+  valuesFrom:
+    - kind: ConfigMap
+      name: grafana-values

platform/observability/grafana/kustomization.yaml

@@ -0,0 +1,21 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+generatorOptions:
+  disableNameSuffixHash: true
+configMapGenerator:
+  - name: grafana-values
+    namespace: monitoring
+    files:
+      - values.yaml=values.yaml
+    options:
+      labels:
+        reconcile.fluxcd.io/watch: Enabled
+  - name: grafana-dashboard-victoriametrics
+    namespace: monitoring
+    files:
+      - victoriametrics.json=dashboards/victoriametrics.json
+    options:
+      labels:
+        grafana_dashboard: "1"
+resources:
+  - helmrelease.yaml

platform/observability/grafana/values.yaml

@@ -48,6 +48,12 @@ persistence:
   storageClassName: localpv-zfs-ceph
   size: 5Gi
 
+# localpv-zfs-ceph cannot mount the same RWO volume into the old and new Grafana
+# Pods concurrently. RollingUpdate leaves the old Pod serving while the new Pod
+# blocks forever in verifyMount, so upgrades must stop the old Pod first.
+deploymentStrategy:
+  type: Recreate
+
 # --- Private exposure via the Tailscale ingress (like seaweedfs-admin) ---
 # The tailscale operator provisions grafana.<tailnet>.ts.net and a TLS cert.
 ingress:

platform/observability/helmrepository.yaml

@@ -0,0 +1,8 @@
+apiVersion: source.toolkit.fluxcd.io/v1
+kind: HelmRepository
+metadata:
+  name: vm
+  namespace: monitoring
+spec:
+  interval: 1h
+  url: https://victoriametrics.github.io/helm-charts/

platform/observability/kustomization.yaml

@@ -0,0 +1,11 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - namespace.yaml
+  - helmrepository.yaml
+  - grafana-helmrepository.yaml
+  - operator
+  - metrics
+  - logs
+  - traces
+  - grafana

platform/observability/logs/helmrelease.yaml

@@ -0,0 +1,31 @@
+apiVersion: helm.toolkit.fluxcd.io/v2
+kind: HelmRelease
+metadata:
+  name: victoria-logs-collector
+  namespace: monitoring
+spec:
+  chart:
+    spec:
+      chart: victoria-logs-collector
+      interval: 1h
+      sourceRef:
+        kind: HelmRepository
+        name: vm
+      version: 0.3.6
+  driftDetection:
+    mode: enabled
+  install:
+    strategy:
+      name: RetryOnFailure
+      retryInterval: 5m
+  interval: 30m
+  releaseName: victoria-logs-collector
+  targetNamespace: monitoring
+  timeout: 10m
+  upgrade:
+    strategy:
+      name: RetryOnFailure
+      retryInterval: 5m
+  valuesFrom:
+    - kind: ConfigMap
+      name: victoria-logs-collector-values

platform/observability/logs/kustomization.yaml

@@ -0,0 +1,14 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+generatorOptions:
+  disableNameSuffixHash: true
+  labels:
+    reconcile.fluxcd.io/watch: Enabled
+configMapGenerator:
+  - name: victoria-logs-collector-values
+    namespace: monitoring
+    files:
+      - values.yaml=collector-values.yaml
+resources:
+  - vlsingle.yaml
+  - helmrelease.yaml

platform/observability/metrics/kustomization.yaml

@@ -0,0 +1,16 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - vmsingle.yaml
+  - vmagent.yaml
+  - vmalert.yaml
+  - vmalertmanager.yaml
+  - reload-rbac.yaml
+  - rules/vm-health.yaml
+  - rules/vmagent.yaml
+  - rules/vmalert.yaml
+  - rules/vmsingle.yaml
+  - scrapes/blocky.yaml
+  - scrapes/docker-hosts.yaml
+  - scrapes/kubelet.yaml
+  - exporters/node-exporter.yaml

platform/observability/operator/helmrelease.yaml

@@ -0,0 +1,33 @@
+apiVersion: helm.toolkit.fluxcd.io/v2
+kind: HelmRelease
+metadata:
+  name: vm-operator
+  namespace: monitoring
+spec:
+  chart:
+    spec:
+      chart: victoria-metrics-operator
+      interval: 1h
+      sourceRef:
+        kind: HelmRepository
+        name: vm
+      version: 0.66.2
+  driftDetection:
+    mode: enabled
+  install:
+    crds: CreateReplace
+    strategy:
+      name: RetryOnFailure
+      retryInterval: 5m
+  interval: 30m
+  releaseName: vm-operator
+  targetNamespace: monitoring
+  timeout: 10m
+  upgrade:
+    crds: CreateReplace
+    strategy:
+      name: RetryOnFailure
+      retryInterval: 5m
+  valuesFrom:
+    - kind: ConfigMap
+      name: vm-operator-values

platform/observability/operator/kustomization.yaml

@@ -0,0 +1,13 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+generatorOptions:
+  disableNameSuffixHash: true
+  labels:
+    reconcile.fluxcd.io/watch: Enabled
+configMapGenerator:
+  - name: vm-operator-values
+    namespace: monitoring
+    files:
+      - values.yaml=values.yaml
+resources:
+  - helmrelease.yaml

platform/observability/traces/kustomization.yaml

@@ -0,0 +1,5 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - vtsingle.yaml
+  - otel-collector.yaml

platform/openebs/README.md

@@ -19,11 +19,10 @@ provisioner，因此 brownfield 接管期间保持 `prune: false`。
 
 ## Flux 接管状态
 
-现有 Helm release revision 为 3，保存的 user-supplied values 与本目录 `values.yaml`
+现有 Helm release 保存的 user-supplied values 与本目录 `values.yaml` 一致，并已固定
-一致。Git 中先登记固定 `4.4.0` 且 `suspend: true` 的 HelmRelease；阶段一仅让 Flux
+`4.4.0` 完成分阶段 Flux HelmRelease 接管。阶段一确认 source Ready、Helm revision
-管理 Helm source、release 声明、values ConfigMap 和现有 StorageClass，不运行 Helm
+保持为 3、8 个 OpenEBS Pod UID 未变、10/10 ZFS PV 和集群 20/20 PVC Bound，且
-upgrade。确认 source Ready、Helm revision、OpenEBS Pod UID、PV/PVC 与 ZFS volume
+`data/ceph` 下 10 个 volume 未变；随后通过独立 PR 解除暂停。
-健康后，再通过独立 PR 解除暂停。
 
 CRD 在 install/upgrade 时使用 `CreateReplace`，但绝不通过删除 CRD 迁移。接管和升级
 是两个独立动作；首次激活不得改变 chart、values、StorageClass 或存储引擎。

platform/openebs/helmrelease.yaml

@@ -4,8 +4,6 @@ metadata:
   name: openebs
   namespace: openebs
 spec:
-  # 存储控制面先只登记；解除暂停必须使用独立 PR 并保留 live PV/PVC 基线。
-  suspend: true
   chart:
     spec:
       chart: openebs

platform/spire/README.md

@@ -0,0 +1,94 @@
+# SPIFFE/SPIRE workload identity
+
+SPIRE 是 homelab 的机器与 workload identity 根。人类身份继续由 Samba AD 与
+Authelia 提供；SPIRE 不替代人类 OIDC，也不承担目标服务的资源授权。
+
+部署状态、workload 接入、JWT-SVID → OpenBao exchange、安全规则和故障恢复详见
+[RUNBOOK.md](RUNBOOK.md)。本文只保留部署声明与关键恢复边界。
+
+## 部署范围
+
+Flux 安装 SPIFFE hardened charts：
+
+- `spire-crds` `0.6.1`；
+- `spire` `0.30.2`（SPIRE `1.15.3`）；
+- SPIRE Server、Agent、Controller Manager、SPIFFE CSI Driver；
+- OIDC Discovery Provider。
+
+未启用 Tornjak、SPIRE Identity Exchange、SPIKE、federation、Delegated Identity
+API 或 Broker API。Trust domain 是 `ddupan.top`，Kubernetes cluster name 是
+`homelab`。
+
+## PostgreSQL bootstrap
+
+SPIRE registration datastore 使用共享 CloudNativePG：
+
+```text
+host:     shared-postgresql-rw.shared-db.svc.cluster.local:5432
+database: spire
+role:     spire
+```
+
+数据库与 role 当前是手工创建的临时 bootstrap。密码只存在于
+`spire-server/spire-postgresql` Secret 的 `password` key 中，不提交到 Git。
+在 PostgreSQL tenant operator/DBaaS 接管前，不得删除该 Secret 或重置数据库
+role 密码。
+
+后续声明式管理必须保持这一 Secret 接口，或者在同一个变更中更新
+`spire-server.dataStore.sql.externalSecret`，避免数据库凭据出现两个写入方。
+
+PostgreSQL保存 registration state；SPIRE Server 的 disk KeyManager 仍使用一个
+`1Gi`、`localpv-zfs-ceph` PVC 保存 trust-domain signing keys。数据库备份不能替代
+该 PVC/密钥的备份。
+
+## 身份签发策略
+
+默认的全 Pod fallback `ClusterSPIFFEID` 已关闭。新增 workload 必须显式创建
+`ClusterSPIFFEID`，并以 namespace、ServiceAccount、Pod label 等 selector 收窄。
+不得仅因 Pod 能挂载 CSI socket 就给它签发身份。
+
+稳定的 JWT issuer 预留为：
+
+```text
+https://spire-oidc.ad.ddupan.top
+```
+
+OIDC Discovery Provider 在 Pod 内部使用明文 HTTP，由现有 Envoy Gateway 的
+`https` listener 使用 `*.ad.ddupan.top` wildcard certificate 终止 TLS。对应的
+`HTTPRoute` 将 `spire-oidc.ad.ddupan.top` 转发到 ClusterIP Service；AD DNS 记录
+声明在 `../../infrastructure/dns/records.yml`，由 Samba DNS Ansible 流程应用。
+
+接入 OpenBao 前必须从集群内和 LAN 分别验证 discovery document 的 `issuer` 与
+上述 URL 完全一致。该 endpoint 只发布公开的 discovery metadata 和 JWKS，不能
+在其 HTTPRoute 上添加 Authelia forward-auth。
+
+## 首次部署与验证
+
+合并后观察：
+
+```bash
+sudo k3s kubectl -n flux-system get kustomization spire
+sudo k3s kubectl -n spire-mgmt get helmrelease
+sudo k3s kubectl -n spire-server get pods,pvc
+sudo k3s kubectl -n spire-system get daemonset,pods
+```
+
+必须先确认 `spire-crds` Ready，随后 `spire` Ready。SPIRE Server 应连接 PostgreSQL，
+Agent 应通过 PSAT attestation 注册，CSI Driver 应在节点 Ready。
+
+2026-09-14 已使用临时测试 Pod 与 `ClusterSPIFFEID` 完成
+`aud=openbao` JWT-SVID → OpenBao 登录、token 自省和主动吊销的端到端验收；临时
+Kubernetes 资源与 registration entries 已清理。
+
+OpenBao 中对应的 Terraform 资源位于
+`../../infrastructure/openbao/terraform/auth-spire.tf`。PoC role 只接受精确 subject
+`spiffe://ddupan.top/ns/spire-poc/sa/spire-jwt-poc`，token 不包含 default policy，
+且 `spire-poc` policy 不允许读取任何业务 secret。
+
+## 恢复边界
+
+- 恢复顺序：共享 PostgreSQL、SPIRE Server signing-key PVC、SPIRE Server、Agent；
+- issuer URL 与 trust domain 初始化后不得随意修改；
+- 丢失 signing keys 会使既有 SVID 和下游 JWKS 信任失效；
+- PostgreSQL或 SPIRE 不可用时，不得用新的空数据库覆盖现有状态；
+- 当前 Flux root 与本 Kustomization 均保持 `prune: false`，删除资源需单独审计。

platform/spire/RUNBOOK.md

@@ -0,0 +1,436 @@
+# SPIRE 与 OpenBao workload identity runbook
+
+本文记录 homelab 中 workload 如何取得 SPIFFE 身份、如何把 JWT-SVID 交换成
+OpenBao 短期 token，以及相关的部署、接入、验证和恢复操作。这里的命令默认在
+`laptop` 上执行。
+
+## 1. 当前架构
+
+```text
+Kubernetes Pod
+  │  Pod ServiceAccount + Pod UID
+  ▼
+SPIRE Agent（每节点 DaemonSet，k8s workload attestor）
+  │  Unix socket: /spiffe-workload-api/spire-agent.sock
+  │  Agent 自身通过 k8s_psat 向 Server 证明节点身份
+  ▼
+SPIRE Server（trust domain: ddupan.top）
+  ├─ registration state → shared PostgreSQL
+  ├─ signing keys       → localpv-zfs-ceph PVC
+  └─ JWT public keys    → OIDC Discovery Provider
+                            │
+                            ▼
+                 https://spire-oidc.ad.ddupan.top
+                            │ discovery + JWKS
+                            ▼
+                 OpenBao auth/jwt-spire/login
+                            │ exact sub + aud + role
+                            ▼
+                    短期、最小权限 Bao token
+```
+
+各层职责必须保持分离：
+
+- Kubernetes ServiceAccount 是 Pod 的初始身份证明，不是跨平台 IAM token；
+- SPIRE 负责 workload 身份、证明和 SVID 签发，不保存业务 secret；
+- OpenBao 验证 JWT-SVID，并把身份映射为本地 policy；
+- 目标服务最终仍负责自己的资源授权，SPIFFE ID 本身不等于权限；
+- 人类身份继续使用 Samba AD + Authelia OIDC，不经过 `jwt-spire`。
+
+## 2. 线上对象与稳定标识
+
+| 项目 | 当前值 |
+|---|---|
+| SPIRE chart | `0.30.2` |
+| SPIRE | `1.15.3` |
+| SPIRE CRDs chart | `0.6.1` |
+| trust domain | `ddupan.top` |
+| Kubernetes cluster name | `homelab` |
+| Controller Manager class | `spire-mgmt-spire` |
+| JWT issuer | `https://spire-oidc.ad.ddupan.top` |
+| OpenBao auth mount | `jwt-spire` |
+| OpenBao login endpoint | `auth/jwt-spire/login` |
+| SPIRE Server namespace | `spire-server` |
+| Agent/CSI namespace | `spire-system` |
+| Helm management namespace | `spire-mgmt` |
+
+`trustDomain`、`clusterName`、issuer URL 与 Controller Manager class 都进入身份或
+下游信任配置。修改它们不是普通 rename，必须按 trust-domain migration 处理。
+
+## 3. 身份与授权模型
+
+Kubernetes workload 的默认 SPIFFE ID 约定为：
+
+```text
+spiffe://ddupan.top/ns/<namespace>/sa/<service-account>
+```
+
+身份必须同时在两侧声明：
+
+1. SPIRE `ClusterSPIFFEID` 决定哪些 Pod 可以取得该身份；
+2. OpenBao JWT role 决定该 `sub`、`aud` 能换取哪些 policy。
+
+这两个声明是有意的双重门：只有 SPIRE entry 而没有 Bao role 时，workload 能取得
+SVID，但不能登录 Bao；只有 Bao role 而没有 SPIRE entry 时，没有 workload 能铸造
+满足条件的 JWT。
+
+禁止使用以下宽泛规则：
+
+- 给所有 Pod 启用 fallback `ClusterSPIFFEID`；
+- OpenBao role 接受整个 `spiffe://ddupan.top/*`；
+- 仅按 namespace 匹配高权限身份，却不限制 ServiceAccount 和 Pod labels；
+- 多个安全边界不同的 workload 共用同一个 ServiceAccount；
+- 给 workload token 附带 `default` 或 `admin` policy。
+
+## 4. 新 workload 接入流程
+
+以下示例为 namespace `example` 中的 ServiceAccount `example-worker`。
+
+### 4.1 创建专用 ServiceAccount
+
+```yaml
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: example-worker
+  namespace: example
+```
+
+不要使用 namespace 的 `default` ServiceAccount。
+
+### 4.2 声明 ClusterSPIFFEID
+
+```yaml
+apiVersion: spire.spiffe.io/v1alpha1
+kind: ClusterSPIFFEID
+metadata:
+  name: example-worker
+  labels:
+    spire.spiffe.io/class-name: spire-mgmt-spire
+spec:
+  className: spire-mgmt-spire
+  namespaceSelector:
+    matchLabels:
+      kubernetes.io/metadata.name: example
+  podSelector:
+    matchLabels:
+      app.kubernetes.io/name: example-worker
+  spiffeIDTemplate: "spiffe://{{ .TrustDomain }}/ns/{{ .PodMeta.Namespace }}/sa/{{ .PodSpec.ServiceAccountName }}"
+```
+
+Controller Manager 最终会为具体 Pod UID 创建 registration entry。检查状态：
+
+```bash
+sudo k3s kubectl get clusterspiffeid example-worker -o yaml
+sudo k3s kubectl -n spire-server exec statefulset/spire-server -c spire-server -- \
+  /opt/spire/bin/spire-server entry show \
+  -spiffeID spiffe://ddupan.top/ns/example/sa/example-worker
+```
+
+`status.stats.entryFailures` 必须是 `0`。Pod 重建后 UID 会变化，短暂看到旧 entry
+属于正常收敛过程。
+
+### 4.3 挂载 Workload API
+
+```yaml
+spec:
+  serviceAccountName: example-worker
+  containers:
+    - name: worker
+      volumeMounts:
+        - name: spiffe-workload-api
+          mountPath: /spiffe-workload-api
+          readOnly: true
+      env:
+        - name: SPIFFE_ENDPOINT_SOCKET
+          value: unix:///spiffe-workload-api/spire-agent.sock
+  volumes:
+    - name: spiffe-workload-api
+      csi:
+        driver: csi.spiffe.io
+        readOnly: true
+```
+
+挂载 socket 不会自动获得身份。Agent 会对调用进程执行 workload attestation，只有
+selector 命中 registration entry 才签发 SVID。
+
+应用应优先使用 SPIFFE SDK，通过 Workload API 按需取得并自动轮换 SVID。不要把
+JWT-SVID 写入 Kubernetes Secret、镜像、持久卷、CI artifact 或日志。
+
+### 4.4 声明 OpenBao policy
+
+在 `infrastructure/openbao/terraform/policies/` 中为 workload 建独立 policy。例如：
+
+```hcl
+path "kv/data/apps/example/*" {
+  capabilities = ["read"]
+}
+
+path "auth/token/lookup-self" {
+  capabilities = ["read"]
+}
+
+path "auth/token/revoke-self" {
+  capabilities = ["update"]
+}
+```
+
+不要直接复用 `admin`。动态数据库凭据、SSH 签名和 KV 应分别授权到精确 path。
+
+### 4.5 声明 OpenBao JWT role
+
+在 `infrastructure/openbao/terraform/auth-spire.tf` 增加 role：
+
+```hcl
+resource "vault_jwt_auth_backend_role" "example_worker" {
+  backend   = vault_jwt_auth_backend.spire.path
+  role_name = "example-worker"
+  role_type = "jwt"
+
+  user_claim      = "sub"
+  bound_audiences = ["openbao"]
+  bound_claims = {
+    sub = "spiffe://ddupan.top/ns/example/sa/example-worker"
+  }
+
+  token_policies          = [vault_policy.example_worker.name]
+  token_no_default_policy = true
+  token_ttl               = 300
+  token_max_ttl           = 900
+}
+```
+
+同一个 JWT 可以请求多个 audience，但 Bao role 只接受 `openbao`。未来接入其他服务时
+应给对应服务使用独立 audience，不能把 `openbao` 当作通用 audience。
+
+Terraform apply 必须遵循 `infrastructure/openbao/README.md` 的 remote-state 和认证
+流程。任何包含 destroy/replace 的 plan 都应停止审查；JWT role/policy 的正常新增应为
+纯 `add`。
+
+## 5. JWT-SVID 交换流程
+
+逻辑请求如下：
+
+```http
+POST /v1/auth/jwt-spire/login
+Content-Type: application/json
+
+{
+  "role": "example-worker",
+  "jwt": "<aud=openbao 的 JWT-SVID>"
+}
+```
+
+成功响应中的 `auth.client_token` 是短期 Bao token。它只应存在于进程内存或
+job-scoped `tmpfs`；通常无需主动续期，过期前重新用 Workload API 获取 JWT-SVID 并
+登录即可。
+
+如果使用 SPIRE CLI 调试，`1.15.3` 的 JSON 输出顶层是数组，JWT 位于：
+
+```text
+.[0].svids[0].svid
+```
+
+调试脚本必须把 JSON 捕获到变量中，禁止直接输出：
+
+```bash
+set -euo pipefail
+
+JWT_RESPONSE="$(spire-agent api fetch jwt \
+  -audience openbao \
+  -socketPath /spiffe-workload-api/spire-agent.sock \
+  -output json)"
+JWT_SVID="$(printf '%s' "$JWT_RESPONSE" | jq -er '.[0].svids[0].svid')"
+
+LOGIN_PAYLOAD="$(jq -nc \
+  --arg role example-worker \
+  --arg jwt "$JWT_SVID" \
+  '{role:$role,jwt:$jwt}')"
+LOGIN_RESPONSE="$(curl --fail-with-body --silent --show-error \
+  -H 'Content-Type: application/json' \
+  --data "$LOGIN_PAYLOAD" \
+  https://bao.ad.ddupan.top:8200/v1/auth/jwt-spire/login)"
+
+export BAO_ADDR=https://bao.ad.ddupan.top:8200
+export BAO_TOKEN="$(printf '%s' "$LOGIN_RESPONSE" | jq -er '.auth.client_token')"
+```
+
+不要在 shell 中启用 `set -x`，不要 `echo "$JWT_SVID"` 或输出完整 login response。
+cleanup 阶段可以尽力主动吊销；短 TTL 仍是主要安全边界：
+
+```bash
+bao token revoke -self
+unset BAO_TOKEN JWT_SVID JWT_RESPONSE LOGIN_RESPONSE LOGIN_PAYLOAD
+```
+
+## 6. CI 与 AI Agent 使用方式
+
+CI job/Agent 不应接收长期 `BAO_TOKEN`。标准启动顺序是：
+
+1. 调度到带 SPIRE Agent 与 CSI Driver 的节点；
+2. 以专用 ServiceAccount 启动，挂载 Workload API socket；
+3. 获取目标 audience 的 JWT-SVID；
+4. 用对应 Bao role 换取短期 token；
+5. 在同一进程树中以环境变量调用 `tofu`、Ansible 或其他工具；
+6. cleanup 尝试 `revoke-self`，随后销毁 job/VM/容器。
+
+通用 credential-exec 包装器未来应负责步骤 3–6。它必须满足：
+
+- 不把 JWT-SVID 或 Bao token 写到 stdout/stderr；
+- 不把凭据传入命令行参数，避免出现在进程列表；
+- 子进程退出后清除环境和临时文件；
+- 不尝试把短期 token 上传到 Actions Secret 或 artifact；
+- role、audience 和目标命令由受审查的 pipeline 配置决定。
+
+Kubernetes 以外的执行环境不能伪造 ServiceAccount。未来应分别使用 host SPIRE
+Agent、TPM/DevID、cloud instance identity、GitHub OIDC 等初始证明接入同一信任模型。
+
+## 7. 日常检查
+
+### Flux 与 Helm
+
+```bash
+sudo k3s kubectl -n flux-system get kustomization spire
+sudo k3s kubectl -n spire-mgmt get helmrepository,helmrelease
+```
+
+### Server、Agent 与 CSI
+
+```bash
+sudo k3s kubectl -n spire-server get pods,pvc
+sudo k3s kubectl -n spire-system get daemonset,pods
+sudo k3s kubectl -n spire-server exec statefulset/spire-server -c spire-server -- \
+  /opt/spire/bin/spire-server agent list
+```
+
+Agent 应显示 `Attestation type: k8s_psat` 与 `Can re-attest: true`。
+
+### OIDC discovery 与 JWKS
+
+```bash
+dig @192.168.10.5 spire-oidc.ad.ddupan.top A +short
+dig @192.168.10.127 spire-oidc.ad.ddupan.top A +short
+curl --fail --silent \
+  https://spire-oidc.ad.ddupan.top/.well-known/openid-configuration | jq
+curl --fail --silent https://spire-oidc.ad.ddupan.top/keys | jq '.keys | length'
+```
+
+discovery 的 `issuer` 必须严格等于
+`https://spire-oidc.ad.ddupan.top`。路径、scheme、hostname 或尾部 `/` 的差异都会
+导致 JWT 验证失败。JWKS 是公开验证材料，不是 secret。
+
+### OpenBao
+
+管理员只检查非敏感配置：
+
+```bash
+export BAO_ADDR=https://bao.ad.ddupan.top:8200
+bao auth list
+bao read auth/jwt-spire/role/<role-name>
+bao policy read <policy-name>
+```
+
+## 8. 故障排查
+
+### `no identity issued`
+
+依次检查：
+
+```bash
+sudo k3s kubectl get clusterspiffeid <name> -o yaml
+sudo k3s kubectl -n <namespace> get pod <pod> \
+  -o custom-columns=NAME:.metadata.name,UID:.metadata.uid,SA:.spec.serviceAccountName,LABELS:.metadata.labels
+sudo k3s kubectl -n spire-server exec statefulset/spire-server -c spire-server -- \
+  /opt/spire/bin/spire-server entry show -spiffeID <expected-spiffe-id>
+```
+
+确认 namespace、ServiceAccount、Pod labels 与 entry 中的 `k8s:pod-uid`。刚启动的
+一次性 Job 可能在 Controller Manager 建 entry 前请求身份并失败；生产客户端应重试
+Workload API，而不是假设 Pod 一启动身份就已可用。
+
+### `auth/jwt-spire/login` 返回 400
+
+常见原因：
+
+- JWT audience 不是 role 的 `bound_audiences`；
+- JWT `sub` 与 role 的 `bound_claims.sub` 不完全一致；
+- issuer 与 `bound_issuer` 不一致；
+- Bao 无法解析或验证 `spire-oidc.ad.ddupan.top`；
+- issuer/JWKS route 被 Authelia forward-auth 拦截；
+- SVID 已过期，或者节点与 Bao 时钟偏差过大；
+- 脚本错误解析 CLI JSON，向 Bao 发送了空 JWT。
+
+排查时只解码 JWT header/claims，禁止记录原始 token。公开 endpoint 可单独验证：
+
+```bash
+curl --fail https://spire-oidc.ad.ddupan.top/.well-known/openid-configuration
+curl --fail https://spire-oidc.ad.ddupan.top/keys
+```
+
+### CSI mount 失败
+
+如果事件包含 `driver name csi.spiffe.io not found`：
+
+```bash
+sudo k3s kubectl get csidriver csi.spiffe.io
+sudo k3s kubectl -n spire-system get daemonset spire-spiffe-csi-driver
+sudo k3s kubectl -n spire-system logs daemonset/spire-spiffe-csi-driver \
+  -c spiffe-csi-driver --tail=100
+```
+
+首次安装时 OIDC Provider/业务 Pod 可能早于 CSI registration，短暂 mount retry 正常；
+持续失败才需要处理。
+
+### Server 无法连接 PostgreSQL
+
+```bash
+sudo k3s kubectl -n shared-db get cluster shared-postgresql
+sudo k3s kubectl -n spire-server get secret spire-postgresql \
+  -o go-template='{{range $k, $_ := .data}}{{$k}}{{"\n"}}{{end}}'
+sudo k3s kubectl -n spire-server logs statefulset/spire-server \
+  -c spire-server --tail=100
+```
+
+Secret 必须有 `password` key。禁止为排障直接输出或提交其值。
+
+## 9. 轮换、备份与恢复
+
+SPIRE 会自动轮换 X.509 CA 与 JWT signing key；OIDC Provider 从 Workload API 获取
+当前 JWKS，下游按 `kid` 验证。正常轮换不应要求更新 Bao role。
+
+必须备份两类不同状态：
+
+- PostgreSQL：registration entries、agent state 和 SPIRE metadata；
+- `spire-data-spire-server-0` PVC：disk KeyManager 的 trust-domain signing keys。
+
+仅恢复 PostgreSQL 而丢失 PVC，不等于恢复 SPIRE。签名密钥丢失会使既有 SVID/JWKS
+信任链失效。恢复顺序：
+
+1. 恢复共享 PostgreSQL；
+2. 恢复 signing-key PVC；
+3. 启动 SPIRE Server；
+4. 确认 bundle/JWKS 后启动或恢复 Agent；
+5. 最后恢复依赖 SPIRE 登录 Bao 的 workload。
+
+不要以空数据库或空 PVC“修复”启动失败。若确实需要重建 trust domain，应把它作为
+全体下游重新建立信任的灾难恢复事件处理。
+
+## 10. 当前 PoC 结论与后续工作
+
+2026-09-14 已完成并清理一次临时 PoC：
+
+- workload 取得 `aud=openbao` JWT-SVID；
+- 精确 subject 成功登录 `jwt-spire/spire-poc`；
+- 返回 token 仅含 `spire-poc` policy，TTL 为 300 秒，无 default policy；
+- `lookup-self` 成功，随后 `revoke-self` 并验证 token 已失效；
+- 临时 Namespace、Pod/Job、ServiceAccount、`ClusterSPIFFEID` 与 registration entries
+  均已删除；
+- Terraform 完整 plan 最终为 `No changes`。
+
+下一步不是重复 PoC，而是为真实 CI/AI Agent 定义：
+
+- 独立 ServiceAccount 和稳定 SPIFFE ID；
+- 按能力拆分的 OpenBao policy（例如 SSH CA、S3 state、数据库动态凭据）；
+- 通用 credential-exec 包装器；
+- token 获取失败、过期与 cleanup 的客户端重试语义；
+- Kubernetes 外 host/VM/microVM 的 SPIRE Agent attestation 方案。

platform/spire/helmrelease-crds.yaml

@@ -0,0 +1,30 @@
+apiVersion: helm.toolkit.fluxcd.io/v2
+kind: HelmRelease
+metadata:
+  name: spire-crds
+  namespace: spire-mgmt
+spec:
+  chart:
+    spec:
+      chart: spire-crds
+      interval: 1h
+      sourceRef:
+        kind: HelmRepository
+        name: spiffe-hardened
+      version: 0.6.1
+  driftDetection:
+    mode: enabled
+  install:
+    crds: CreateReplace
+    strategy:
+      name: RetryOnFailure
+      retryInterval: 5m
+  interval: 30m
+  releaseName: spire-crds
+  targetNamespace: spire-mgmt
+  timeout: 10m
+  upgrade:
+    crds: CreateReplace
+    strategy:
+      name: RetryOnFailure
+      retryInterval: 5m

platform/spire/helmrelease.yaml

@@ -0,0 +1,34 @@
+apiVersion: helm.toolkit.fluxcd.io/v2
+kind: HelmRelease
+metadata:
+  name: spire
+  namespace: spire-mgmt
+spec:
+  chart:
+    spec:
+      chart: spire
+      interval: 1h
+      sourceRef:
+        kind: HelmRepository
+        name: spiffe-hardened
+      version: 0.30.2
+  dependsOn:
+    - name: spire-crds
+      namespace: spire-mgmt
+  driftDetection:
+    mode: enabled
+  install:
+    strategy:
+      name: RetryOnFailure
+      retryInterval: 5m
+  interval: 30m
+  releaseName: spire
+  targetNamespace: spire-mgmt
+  timeout: 15m
+  upgrade:
+    strategy:
+      name: RetryOnFailure
+      retryInterval: 5m
+  valuesFrom:
+    - kind: ConfigMap
+      name: spire-values

platform/spire/helmrepository.yaml

@@ -0,0 +1,8 @@
+apiVersion: source.toolkit.fluxcd.io/v1
+kind: HelmRepository
+metadata:
+  name: spiffe-hardened
+  namespace: spire-mgmt
+spec:
+  interval: 1h
+  url: https://spiffe.github.io/helm-charts-hardened/

platform/spire/httproute.yaml

@@ -0,0 +1,16 @@
+apiVersion: gateway.networking.k8s.io/v1
+kind: HTTPRoute
+metadata:
+  name: spire-oidc-discovery-provider
+  namespace: spire-server
+spec:
+  parentRefs:
+    - name: eg
+      namespace: envoy-gateway-system
+      sectionName: https
+  hostnames:
+    - spire-oidc.ad.ddupan.top
+  rules:
+    - backendRefs:
+        - name: spire-spiffe-oidc-discovery-provider
+          port: 80

platform/spire/kustomization.yaml

@@ -0,0 +1,17 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+generatorOptions:
+  disableNameSuffixHash: true
+  labels:
+    reconcile.fluxcd.io/watch: Enabled
+configMapGenerator:
+  - name: spire-values
+    namespace: spire-mgmt
+    files:
+      - values.yaml=values.yaml
+resources:
+  - namespaces.yaml
+  - helmrepository.yaml
+  - helmrelease-crds.yaml
+  - helmrelease.yaml
+  - httproute.yaml

platform/spire/namespaces.yaml

@@ -0,0 +1,14 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: spire-mgmt
+---
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: spire-server
+---
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: spire-system

platform/spire/values.yaml

@@ -0,0 +1,103 @@
+global:
+  k8s:
+    clusterDomain: cluster.local
+  spire:
+    clusterName: homelab
+    trustDomain: ddupan.top
+    # This is a stable external identity even before its Gateway route is added.
+    # Changing the issuer after consumers trust it would invalidate their config.
+    jwtIssuer: https://spire-oidc.ad.ddupan.top
+    caSubject:
+      country: CN
+      organization: ddupan homelab
+      commonName: ddupan.top
+    namespaces:
+      create: false
+      system:
+        name: spire-system
+      server:
+        name: spire-server
+    recommendations:
+      enabled: true
+      namespaceLayout: true
+      namespacePSS: true
+      priorityClassName: true
+      strictMode: true
+      securityContexts: true
+      prometheus: false
+
+spire-server:
+  kind: statefulset
+  replicaCount: 1
+  auditLogEnabled: true
+  persistence:
+    # PostgreSQL stores registrations, but the disk KeyManager still needs durable
+    # storage for the trust-domain signing keys.
+    type: pvc
+    size: 1Gi
+    accessMode: ReadWriteOnce
+    storageClass: localpv-zfs-ceph
+  dataStore:
+    sql:
+      databaseType: postgres
+      databaseName: spire
+      host: shared-postgresql-rw.shared-db.svc.cluster.local
+      port: 5432
+      username: spire
+      options:
+        - sslmode: require
+      externalSecret:
+        enabled: true
+        name: spire-postgresql
+        key: password
+      maxOpenConns: 20
+      maxIdleConns: 2
+      connMaxLifetime: 30m
+  controllerManager:
+    enabled: true
+    # Identities are opt-in. Do not issue a fallback SVID to every pod in the
+    # cluster merely because it can mount the Workload API socket.
+    identities:
+      clusterSPIFFEIDs:
+        default:
+          enabled: false
+        test-keys:
+          enabled: false
+
+spire-agent:
+  nodeAttestor:
+    k8sPSAT:
+      enabled: true
+  workloadAttestors:
+    k8s:
+      enabled: true
+    unix:
+      enabled: false
+
+spiffe-csi-driver:
+  enabled: true
+
+spiffe-oidc-discovery-provider:
+  enabled: true
+  # TLS will terminate at the existing Envoy Gateway when the stable issuer is
+  # exposed. The service remains ClusterIP-only during the first deployment.
+  tls:
+    spire:
+      enabled: false
+    externalSecret:
+      enabled: false
+    certManager:
+      enabled: false
+
+upstream:
+  enabled: false
+tornjak-frontend:
+  enabled: false
+spire-identity-exchange:
+  enabled: false
+spike-keeper:
+  enabled: false
+spike-nexus:
+  enabled: false
+spike-pilot:
+  enabled: false