修复 kubelet 超限与旧 Docker 采集目标 (#148)

Co-authored-by: panxiao81 <[email protected]>

.gitea/workflows/ansible.yml

@@ -16,9 +16,6 @@ on:
       - '.ansible-lint'
       - '.gitea/workflows/ansible.yml'
 
-env:
-  ANSIBLE_COLLECTIONS_PATH: /root/.ansible/collections
-
 jobs:
   lint:
     runs-on: [self-hosted, pod]
@@ -30,6 +27,7 @@ jobs:
           python3 -m pip install --user --break-system-packages \
             --index-url https://pypi.org/simple --quiet uv==0.11.7
           echo "$HOME/.local/bin" >> "$GITHUB_PATH"
+          echo "ANSIBLE_COLLECTIONS_PATH=$HOME/.ansible/collections" >> "$GITHUB_ENV"
 
       - name: Install ansible-lint and collections
         run: |
@@ -50,7 +48,7 @@ jobs:
       - name: ansible-lint
         run: |
           export PATH="$HOME/.local/bin:$PATH"
-          export ANSIBLE_COLLECTIONS_PATH="$PWD/infrastructure/samba-ad/ansible/collections:/root/.ansible/collections"
+          export ANSIBLE_COLLECTIONS_PATH="$PWD/infrastructure/samba-ad/ansible/collections:$HOME/.ansible/collections"
           # 静态检查不应依赖生产 vault 凭据。一次性 checkout 可以去掉加密变量文件；
           # syntax-check 只验证结构，不需要解析变量的运行时值。
           rm -f \

.gitea/workflows/hydra.yml

@@ -0,0 +1,22 @@
+name: hydra-login
+on:
+  pull_request:
+    paths:
+      - 'apps/hydra/login-consent/**'
+      - '.gitea/workflows/hydra.yml'
+  workflow_dispatch:
+jobs:
+  verify:
+    runs-on: [self-hosted, pod]
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-go@v5
+        with:
+          go-version-file: apps/hydra/login-consent/go.mod
+          cache-dependency-path: apps/hydra/login-consent/go.sum
+      - name: Test authentication boundaries
+        working-directory: apps/hydra/login-consent
+        run: |
+          go test -race ./...
+          go vet ./...
+          CGO_ENABLED=0 go build -trimpath .

apps/backstage/README.md

@@ -0,0 +1,20 @@
+# Backstage
+
+Backstage 作为 homelab 的只读开发者门户运行。应用源码、插件、测试和镜像构建归
+`panxiao81/backstage` 仓库管理；本目录只保存 Kubernetes/Flux 部署声明，并通过
+OCI digest 固定镜像。
+
+## 外部前置
+
+- OpenBao `kv/k8s/backstage` 必须包含以下与容器环境变量同名的字段：
+  `BACKEND_SECRET`、`AUTH_SESSION_SECRET`、`AUTH_OIDC_CLIENT_ID`、
+  `AUTH_OIDC_CLIENT_SECRET`、`POSTGRES_PASSWORD`、`GITEA_TOKEN`。
+- PostgreSQL 需要在共享集群中预先创建由 `backstage` 角色拥有的 `backstage`
+  数据库；密码必须与 OpenBao 中的 `POSTGRES_PASSWORD` 一致。
+- Authelia OIDC 客户端回调地址为
+  `https://backstage.ad.ddupan.top/api/auth/oidc/handler/frame`。
+- AD DNS 需要将 `backstage.ad.ddupan.top` 指向 Envoy Gateway
+  `192.168.10.127`。
+
+Flux 等待 ExternalSecret 和 Deployment 就绪；任何前置缺失都会使该
+Kustomization 保持 NotReady，而不会回退到明文 Secret。

apps/backstage/deployment.yaml

@@ -0,0 +1,85 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: backstage
+  namespace: backstage
+  labels:
+    app.kubernetes.io/name: backstage
+spec:
+  replicas: 1
+  strategy:
+    type: Recreate
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: backstage
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: backstage
+    spec:
+      serviceAccountName: backstage
+      securityContext:
+        fsGroup: 1000
+        runAsNonRoot: true
+        seccompProfile:
+          type: RuntimeDefault
+      containers:
+        - name: backstage
+          image: zot.ad.ddupan.top/panxiao81/backstage@sha256:6d6a11f8611a85237e3b0ffd30a2a6e119b1e5258b2ca6c193d60a8c3ec52553
+          imagePullPolicy: IfNotPresent
+          env:
+            - name: BACKSTAGE_BASE_URL
+              value: https://backstage.ad.ddupan.top
+            - name: POSTGRES_HOST
+              value: shared-postgresql-rw.shared-db.svc.cluster.local
+            - name: POSTGRES_PORT
+              value: "5432"
+            - name: POSTGRES_USER
+              value: backstage
+            - name: POSTGRES_DATABASE
+              value: backstage
+            - name: GITEA_HOST
+              value: git.ddupan.top
+            - name: GITEA_ORG
+              value: panxiao81
+          envFrom:
+            - secretRef:
+                name: backstage
+          ports:
+            - containerPort: 7007
+              name: http
+              protocol: TCP
+          readinessProbe:
+            httpGet:
+              path: /healthcheck
+              port: http
+            initialDelaySeconds: 10
+            periodSeconds: 10
+            timeoutSeconds: 3
+          livenessProbe:
+            httpGet:
+              path: /healthcheck
+              port: http
+            initialDelaySeconds: 30
+            periodSeconds: 20
+            timeoutSeconds: 3
+          resources:
+            requests:
+              cpu: 100m
+              memory: 512Mi
+            limits:
+              cpu: "1"
+              memory: 1Gi
+          securityContext:
+            allowPrivilegeEscalation: false
+            capabilities:
+              drop: [ALL]
+            readOnlyRootFilesystem: true
+            runAsNonRoot: true
+            runAsUser: 1000
+          volumeMounts:
+            - mountPath: /tmp
+              name: tmp
+      volumes:
+        - emptyDir: {}
+          name: tmp

apps/backstage/external-secret.yaml

@@ -0,0 +1,16 @@
+apiVersion: external-secrets.io/v1
+kind: ExternalSecret
+metadata:
+  name: backstage
+  namespace: backstage
+spec:
+  refreshInterval: 1h
+  secretStoreRef:
+    kind: ClusterSecretStore
+    name: openbao
+  target:
+    creationPolicy: Owner
+    name: backstage
+  dataFrom:
+    - extract:
+        key: k8s/backstage

apps/backstage/httproute.yaml

@@ -0,0 +1,20 @@
+apiVersion: gateway.networking.k8s.io/v1
+kind: HTTPRoute
+metadata:
+  name: backstage
+  namespace: backstage
+spec:
+  parentRefs:
+    - name: eg
+      namespace: envoy-gateway-system
+      sectionName: https
+  hostnames:
+    - backstage.ad.ddupan.top
+  rules:
+    - matches:
+        - path:
+            type: PathPrefix
+            value: /
+      backendRefs:
+        - name: backstage
+          port: 7007

apps/backstage/kustomization.yaml

@@ -0,0 +1,10 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - namespace.yaml
+  - serviceaccount.yaml
+  - external-secret.yaml
+  - deployment.yaml
+  - service.yaml
+  - networkpolicy.yaml
+  - httproute.yaml

apps/backstage/namespace.yaml

@@ -0,0 +1,7 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: backstage
+  labels:
+    pod-security.kubernetes.io/enforce: restricted
+    pod-security.kubernetes.io/enforce-version: v1.36

apps/backstage/networkpolicy.yaml

@@ -0,0 +1,24 @@
+apiVersion: networking.k8s.io/v1
+kind: NetworkPolicy
+metadata:
+  name: backstage-ingress
+  namespace: backstage
+spec:
+  podSelector:
+    matchLabels:
+      app.kubernetes.io/name: backstage
+  policyTypes: [Ingress]
+  ingress:
+    - from:
+        - namespaceSelector:
+            matchLabels:
+              kubernetes.io/metadata.name: envoy-gateway-system
+      ports:
+        - port: 7007
+          protocol: TCP
+    - from:
+        - ipBlock:
+            cidr: 192.168.10.127/32
+      ports:
+        - port: 7007
+          protocol: TCP

apps/backstage/service.yaml

@@ -0,0 +1,13 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: backstage
+  namespace: backstage
+spec:
+  selector:
+    app.kubernetes.io/name: backstage
+  ports:
+    - name: http
+      port: 7007
+      protocol: TCP
+      targetPort: http

apps/backstage/serviceaccount.yaml

@@ -1,6 +1,6 @@
 apiVersion: v1
 kind: ServiceAccount
 metadata:
-  name: gitea-actions
+  name: backstage
-  namespace: gitea-actions
+  namespace: backstage
 automountServiceAccountToken: false

apps/blocky/compose.yaml

@@ -58,4 +58,4 @@ networks:
   default:
     ipam:
       config:
-      - subnet: 172.28.0.0/24
+        - subnet: 172.28.0.0/24

apps/gitea/README.md

@@ -44,3 +44,15 @@ API、OIDC、Git/Flux 和 runner 均已验证。第二跳按明确决定跳过
 结构，或者 release notes 指出相关 breaking migration 时，才恢复停机一致备份、分阶段
 suspend、详细日志审计和扩展验收。出现启动失败或 migration error 时也立即升级为完整
 故障流程。
+
+## Hydra 人类登录 PoC
+
+新增 `hydra` OIDC 登录源，旧 `authelia` 入口保留。Hydra 通过通用 OIDC Login/Consent
+适配器转到现有 Authelia 完成人类认证；不是 Gitea 直接验证 LDAP 或 SPIFFE。
+入口为 `https://git.ddupan.top/user/oauth2/hydra`，需 LAN/Tailscale 可达 Hydra 内网域名。
+
+新 client secret 通过 `ExternalSecret/gitea-hydra-oidc` 从 OpenBao 投射。沿用
+preferred_username、已验证邮箱与 groups；当前仍映射 gitea-admins，不在本轮切换组模型。
+先部署并验证 Hydra discovery 后再接入本配置，避免 Gitea init 因上游不可达而失败。
+
+实际登录验收与部署状态见 wiki；依赖和回退见 [Hydra README](../hydra/README.md)。

apps/gitea/gitea-values.yaml

@@ -116,6 +116,13 @@ gitea:
       scopes: openid profile email groups
       groupClaimName: groups
       adminGroup: gitea-admins
+    - name: hydra
+      provider: openidConnect
+      existingSecret: gitea-hydra-oidc
+      autoDiscoverUrl: https://hydra.ad.ddupan.top/.well-known/openid-configuration
+      scopes: openid profile email groups
+      groupClaimName: groups
+      adminGroup: gitea-admins
 
 persistence:
   size: 20Gi

apps/gitea/hydra-external-secret.yaml

@@ -0,0 +1,22 @@
+apiVersion: external-secrets.io/v1
+kind: ExternalSecret
+metadata:
+  name: gitea-hydra-oidc
+  namespace: gitea
+spec:
+  refreshInterval: 1h
+  secretStoreRef:
+    kind: ClusterSecretStore
+    name: openbao
+  target:
+    name: gitea-hydra-oidc
+    creationPolicy: Owner
+    template:
+      data:
+        key: gitea
+        secret: "{{ .client_secret }}"
+  data:
+    - secretKey: client_secret
+      remoteRef:
+        key: k8s/hydra
+        property: gitea_client_secret

apps/gitea/kustomization.yaml

@@ -16,3 +16,4 @@ resources:
   - helmrepository.yaml
   - helmrelease.yaml
   - httproute.yaml
+  - hydra-external-secret.yaml

apps/hydra/README.md

@@ -0,0 +1,102 @@
+# Hydra 与 OIDC Login/Consent PoC
+
+本目录提供独立 Hydra 签发服务，以及一个薄的 **OIDC 上游适配器**。当前上游配置为
+Authelia；适配器不连接 LDAP，也不管理用户目录。Samba AD、密码和 MFA 继续由现有
+Authelia 链路负责。第一轮只接入人类和 Gitea，不实现 agent 动态授权。
+
+```text
+Gitea → Hydra → OIDC Login/Consent → Authelia → Samba AD
+      ← OIDC ← 经验证的上游身份 ← OIDC callback
+```
+
+目标入口：
+
+- `https://hydra.ad.ddupan.top`：Hydra 公共 OAuth2/OIDC endpoint。
+- `https://hydra-login.ad.ddupan.top`：上游 OIDC 登录及 consent 适配器。
+- `hydra-admin.hydra.svc.cluster.local:4445`：仅集群内管理接口，无 HTTPRoute。
+
+均为 LAN/Tailscale 入口，复用 Envoy `eg/https` wildcard TLS。没有增加公网 tunnel。
+部署及实际验收状态以 wiki 和对应 PR 为准，文件存在不表示登录已验收。
+
+## 首次使用与边界
+
+在 Gitea 登录页选择 `hydra`，跳转到 Authelia 完成现有人类认证，再返回原有 Gitea
+账号。旧 `authelia` 登录源保留。Gitea 的账号关联和资源权限仍由 Gitea 维护。
+
+适配器要求验证上游 issuer、audience、签名、过期时间和 nonce，使用 PKCE S256，
+并把单次 state 绑定到 Secure/HttpOnly/SameSite=Lax cookie。短期登录事务只存内存，
+最多 1024 个、10 分钟过期；单副本重启后正在登录的用户需重试，不存人类密码或 token。
+
+Hydra subject 为上游 `(issuer, sub)` 的 SHA-256 加 `human:` 前缀，与可变邮箱/用户名
+分离。第一轮要求上游返回经过验证的 email 及 preferred_username；这些 claims 必须
+明确配置进 ID token。更换 issuer 会改变本 PoC 的 subject，正式迁移前需要身份绑定设计。
+
+仅为显式 `ALLOWED_CLIENTS=gitea` 自动 consent，scope 限于 openid/profile/email/groups；
+拒绝额外 access-token audience，不发 refresh token。只按实际请求 scope 释放 claims。
+这不是通用的无人确认授权服务。组当前透传，沿用 Gitea 的 gitea-admins 映射；统一组
+模型和 agent 认证均在后续阶段。不存在对 Authelia 专有协议的调用。
+
+NetworkPolicy 限制公共端口只接收 Envoy 流量，Hydra admin 只允许适配器访问。
+Hydra 使用正式模式，TLS 由 Envoy 终止；不使用 `--dev`。管理操作使用受控
+`kubectl port-forward`，不要将 admin 接口暴露到 Gateway。
+
+## 依赖、秘密与初始化
+
+依赖共享 CloudNativePG、OpenBao/ESO、Authelia OIDC、Envoy、Samba DNS、zot 镜像仓库。
+Hydra 使用独立 `hydra` database/role，不与其他应用共享数据库角色。
+
+`kv/k8s/hydra` 保存 dsn、system_secret、upstream_client_secret、upstream_client_digest、
+gitea_client_secret；通过 ExternalSecret 投射，值不写入 Git。Bootstrap 创建角色及数据库
+后才启动 Hydra migration。system_secret 必须持久保存，不得在重启时随机重建。
+
+Authelia 中新增 confidential client `hydra-login`：
+
+- redirect URI：`https://hydra-login.ad.ddupan.top/callback`；
+- authorization policy：two_factor；grant：authorization_code；PKCE：S256；
+- token endpoint auth：client_secret_basic；scope：openid/profile/email/groups；
+- claims policy：把 preferred_username、name、email、email_verified、groups 放入 ID token；
+- client secret 的 PBKDF2 digest 存入 Authelia，原值仅供适配器使用。
+
+Authelia 尚非 Flux 管理。修改 Helm values 时保留所有已有 clients 与 secret 引用，
+通过 `--reuse-values` 和最小 overlay 增加客户端，不能以本目录配置覆盖其完整 values。
+
+Hydra 中注册 confidential client `gitea`，redirect URI 为
+`https://git.ddupan.top/user/oauth2/hydra/callback`，grant/response 为 authorization_code/code，
+scope 为 openid/profile/email/groups，token endpoint auth 为 client_secret_basic。
+Gitea 启动时读取 OIDC discovery，所以应先确认 Hydra 健康和 discovery 可达，再接入 Gitea。
+
+## 构建与检查
+
+```bash
+cd apps/hydra/login-consent
+go test -race ./...
+go vet ./...
+CGO_ENABLED=0 go build -trimpath -ldflags='-s -w' -o login-consent .
+docker build -t hydra-login-consent:VERSION .
+```
+
+Go module 独立，依赖由 go.sum 锁定；Dockerfile 固定基础镜像 digest。
+使用已授权的短期 SPIFFE zot 凭据发布镜像，部署使用匿名拉取入口与不可变 digest。
+不把 registry 凭据写入源码或 build args。
+
+```bash
+kubectl kustomize apps/hydra
+sudo k3s kubectl -n hydra get deployment,pod,externalsecret,httproute
+sudo k3s kubectl -n hydra logs deployment/hydra -c migrate
+sudo k3s kubectl -n hydra logs deployment/hydra-login
+```
+
+日志不输出上游 token、授权 code、challenge 或秘密。登录失败先查两端 Pod 状态、
+DNS/discovery 连通性、client redirect URI 和 scope，再由用户重新发起登录。
+不要在故障排查中关闭签名验证、MFA 或 state/nonce 校验。
+
+## 恢复与回退
+
+保留共享 PostgreSQL 中 Hydra 数据及 OpenBao 秘密；数据库持有 clients、会话及签名密钥，
+单独重建 Deployment 不能替代恢复数据库。先恢复依赖，再启动 Hydra 和适配器。
+当前恢复仍依赖 homelab 共享基础设施，不能声称已完成独立灾备。
+
+第一轮不切换 Authelia 的主入口。撤回 Gitea 的新增 Hydra 登录源即可回到旧入口；
+先撤消费者，再考虑停用 Hydra。不要删除旧 Authelia 登录源、用户或数据库作为回退手段。
+
+跨服务设计见 [独立 IAM 草案](https://git.ddupan.top/panxiao81/homelab-wiki/src/branch/main/architecture/independent-iam-draft.md)。

apps/hydra/deployment.yaml

@@ -0,0 +1,95 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: hydra
+  namespace: hydra
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: hydra
+  template:
+    metadata:
+      labels:
+        app: hydra
+    spec:
+      automountServiceAccountToken: false
+      securityContext:
+        runAsNonRoot: true
+        runAsUser: 1000
+        runAsGroup: 1000
+        seccompProfile:
+          type: RuntimeDefault
+      initContainers:
+      - name: migrate
+        image: docker.io/oryd/hydra:v26.2.0@sha256:ff67c7fb5f95074fa53374d41151713554960504b340cd3f95b09e65deaea2a9
+        args:
+        - migrate
+        - sql
+        - -e
+        - --yes
+        env:
+        - name: DSN
+          valueFrom:
+            secretKeyRef:
+              name: hydra
+              key: dsn
+        securityContext: &id002
+          allowPrivilegeEscalation: false
+          readOnlyRootFilesystem: true
+          capabilities:
+            drop:
+            - ALL
+        resources: &id001
+          requests:
+            cpu: 50m
+            memory: 64Mi
+          limits:
+            memory: 256Mi
+      containers:
+      - name: hydra
+        image: docker.io/oryd/hydra:v26.2.0@sha256:ff67c7fb5f95074fa53374d41151713554960504b340cd3f95b09e65deaea2a9
+        args:
+        - serve
+        - all
+        - --config
+        - /etc/hydra/hydra.yaml
+        - --sqa-opt-out
+        env:
+        - name: DSN
+          valueFrom:
+            secretKeyRef:
+              name: hydra
+              key: dsn
+        - name: SECRETS_SYSTEM
+          valueFrom:
+            secretKeyRef:
+              name: hydra
+              key: system_secret
+        ports:
+        - name: public
+          containerPort: 4444
+        - name: admin
+          containerPort: 4445
+        resources: *id001
+        securityContext: *id002
+        volumeMounts:
+        - name: config
+          mountPath: /etc/hydra
+          readOnly: true
+        readinessProbe:
+          httpGet:
+            path: /health/ready
+            port: admin
+          initialDelaySeconds: 5
+          periodSeconds: 5
+        livenessProbe:
+          httpGet:
+            path: /health/alive
+            port: admin
+          initialDelaySeconds: 20
+          periodSeconds: 20
+      volumes:
+      - name: config
+        configMap:
+          name: hydra-config

apps/hydra/external-secret.yaml

@@ -0,0 +1,16 @@
+apiVersion: external-secrets.io/v1
+kind: ExternalSecret
+metadata:
+  name: hydra
+  namespace: hydra
+spec:
+  refreshInterval: 1h
+  secretStoreRef:
+    kind: ClusterSecretStore
+    name: openbao
+  target:
+    name: hydra
+    creationPolicy: Owner
+  dataFrom:
+  - extract:
+      key: k8s/hydra

apps/hydra/httproutes.yaml

@@ -0,0 +1,33 @@
+apiVersion: gateway.networking.k8s.io/v1
+kind: HTTPRoute
+metadata:
+  name: hydra-public
+  namespace: hydra
+spec:
+  parentRefs:
+  - name: eg
+    namespace: envoy-gateway-system
+    sectionName: https
+  hostnames:
+  - hydra.ad.ddupan.top
+  rules:
+  - backendRefs:
+    - name: hydra-public
+      port: 4444
+---
+apiVersion: gateway.networking.k8s.io/v1
+kind: HTTPRoute
+metadata:
+  name: hydra-login
+  namespace: hydra
+spec:
+  parentRefs:
+  - name: eg
+    namespace: envoy-gateway-system
+    sectionName: https
+  hostnames:
+  - hydra-login.ad.ddupan.top
+  rules:
+  - backendRefs:
+    - name: hydra-login
+      port: 8080

apps/hydra/hydra.yaml

@@ -0,0 +1,25 @@
+serve:
+  public:
+    port: 4444
+  admin:
+    port: 4445
+  tls:
+    allow_termination_from:
+    - 10.42.0.0/16
+  cookies:
+    same_site_mode: Lax
+urls:
+  self:
+    issuer: https://hydra.ad.ddupan.top
+    public: https://hydra.ad.ddupan.top
+  login: https://hydra-login.ad.ddupan.top/login
+  consent: https://hydra-login.ad.ddupan.top/consent
+ttl:
+  access_token: 15m
+  id_token: 15m
+  auth_code: 5m
+log:
+  level: info
+  leak_sensitive_values: false
+oauth2:
+  expose_internal_errors: false

apps/hydra/kustomization.yaml

@@ -0,0 +1,15 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+- namespace.yaml
+- external-secret.yaml
+- deployment.yaml
+- login-deployment.yaml
+- services.yaml
+- httproutes.yaml
+- networkpolicy.yaml
+configMapGenerator:
+- name: hydra-config
+  namespace: hydra
+  files:
+  - hydra.yaml

apps/hydra/login-consent/.gitignore

@@ -0,0 +1 @@
+/login-consent

apps/hydra/login-consent/Dockerfile

@@ -0,0 +1,4 @@
+FROM gcr.io/distroless/static-debian12:nonroot@sha256:afa5c872c891853ca7fcf1f12c3edb23f7eeef36189728842dd51042ff57f7ab
+COPY login-consent /login-consent
+USER 65532:65532
+ENTRYPOINT ["/login-consent"]

apps/hydra/login-consent/go.mod

@@ -0,0 +1,13 @@
+module git.ddupan.top/panxiao81/homelab-infra/apps/hydra/login-consent
+
+go 1.26.0
+
+require (
+	github.com/coreos/go-oidc/v3 v3.14.1
+	golang.org/x/oauth2 v0.37.0
+)
+
+require (
+	github.com/go-jose/go-jose/v4 v4.0.5 // indirect
+	golang.org/x/crypto v0.36.0 // indirect
+)

apps/hydra/login-consent/go.sum

@@ -0,0 +1,18 @@
+github.com/coreos/go-oidc/v3 v3.14.1 h1:9ePWwfdwC4QKRlCXsJGou56adA/owXczOzwKdOumLqk=
+github.com/coreos/go-oidc/v3 v3.14.1/go.mod h1:HaZ3szPaZ0e4r6ebqvsLWlk2Tn+aejfmrfah6hnSYEU=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/go-jose/go-jose/v4 v4.0.5 h1:M6T8+mKZl/+fNNuFHvGIzDz7BTLQPIounk/b9dw3AaE=
+github.com/go-jose/go-jose/v4 v4.0.5/go.mod h1:s3P1lRrkT8igV8D9OjyL4WRyHvjB6a4JSllnOrmmBOA=
+github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
+github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA=
+github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
+golang.org/x/crypto v0.36.0 h1:AnAEvhDddvBdpY+uR+MyHmuZzzNqXSe/GvuDeob5L34=
+golang.org/x/crypto v0.36.0/go.mod h1:Y4J0ReaxCR1IMaabaSMugxJES1EpwhBHhv2bDHklZvc=
+golang.org/x/oauth2 v0.37.0 h1:JUlcxA8oAtauLfiH8FX2/FkAWHAdi0QtGCGc+hofE98=
+golang.org/x/oauth2 v0.37.0/go.mod h1:IxwZNxUULJmpBFf9K/9NTMSIfZZuvuTy1gGxhigP/58=
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
+gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

apps/hydra/login-consent/main.go

@@ -0,0 +1,285 @@
+// Login/Consent adapter for a single trusted upstream and first-party clients.
+package main
+
+import (
+	"bytes"
+	"context"
+	"crypto/rand"
+	"crypto/sha256"
+	"crypto/subtle"
+	"encoding/base64"
+	"encoding/hex"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"log"
+	"net/http"
+	"net/url"
+	"os"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/coreos/go-oidc/v3/oidc"
+	"golang.org/x/oauth2"
+)
+
+const cookieName = "__Host-hydra-login"
+
+type pending struct {
+	Challenge, Nonce, Verifier string
+	Expires                    time.Time
+}
+type claims struct {
+	Username      string   `json:"preferred_username"`
+	Email         string   `json:"email"`
+	EmailVerified bool     `json:"email_verified"`
+	Name          string   `json:"name"`
+	Groups        []string `json:"groups"`
+}
+type flowRequest struct {
+	Client struct {
+		ID string `json:"client_id"`
+	} `json:"client"`
+	Subject  string   `json:"subject"`
+	Scopes   []string `json:"requested_scope"`
+	Audience []string `json:"requested_access_token_audience"`
+	Context  claims   `json:"context"`
+}
+type app struct {
+	admin, public string
+	client        *http.Client
+	oauth         oauth2.Config
+	verifier      *oidc.IDTokenVerifier
+	allowed       map[string]bool
+	mu            sync.Mutex
+	pending       map[string]pending
+}
+
+func required(key string) string {
+	v := os.Getenv(key)
+	if v == "" {
+		log.Fatalf("missing %s", key)
+	}
+	return v
+}
+func random() string {
+	b := make([]byte, 32)
+	if _, err := rand.Read(b); err != nil {
+		panic(err)
+	}
+	return base64.RawURLEncoding.EncodeToString(b)
+}
+func (a *app) api(ctx context.Context, method, path string, in, out any) error {
+	var body io.Reader
+	if in != nil {
+		b, err := json.Marshal(in)
+		if err != nil {
+			return err
+		}
+		body = bytes.NewReader(b)
+	}
+	req, err := http.NewRequestWithContext(ctx, method, a.admin+path, body)
+	if err != nil {
+		return err
+	}
+	req.Header.Set("Content-Type", "application/json")
+	resp, err := a.client.Do(req)
+	if err != nil {
+		return errors.New("Hydra unavailable")
+	}
+	defer resp.Body.Close()
+	if resp.StatusCode < 200 || resp.StatusCode >= 300 {
+		return fmt.Errorf("Hydra status %d", resp.StatusCode)
+	}
+	if out != nil {
+		return json.NewDecoder(io.LimitReader(resp.Body, 1<<20)).Decode(out)
+	}
+	return nil
+}
+func (a *app) request(r *http.Request, kind, challenge string) (flowRequest, error) {
+	var f flowRequest
+	if challenge == "" || len(challenge) > 8192 {
+		return f, errors.New("missing or invalid challenge")
+	}
+	err := a.api(r.Context(), http.MethodGet, "/admin/oauth2/auth/requests/"+kind+"?"+kind+"_challenge="+url.QueryEscape(challenge), nil, &f)
+	if err != nil {
+		return f, err
+	}
+	if !a.allowed[f.Client.ID] {
+		return f, errors.New("client not allowed")
+	}
+	return f, nil
+}
+func (a *app) accept(w http.ResponseWriter, r *http.Request, kind, challenge string, body any) {
+	var result struct {
+		Redirect string `json:"redirect_to"`
+	}
+	if err := a.api(r.Context(), http.MethodPut, "/admin/oauth2/auth/requests/"+kind+"/accept?"+kind+"_challenge="+url.QueryEscape(challenge), body, &result); err != nil {
+		fail(w, 502)
+		return
+	}
+	// Only Hydra's own authorization endpoint can receive a challenge verifier.
+	u, err := url.Parse(result.Redirect)
+	p, _ := url.Parse(a.public)
+	if err != nil || u.Scheme != p.Scheme || u.Host != p.Host || u.User != nil || u.Path != "/oauth2/auth" {
+		fail(w, 502)
+		return
+	}
+	http.Redirect(w, r, result.Redirect, http.StatusSeeOther)
+}
+func fail(w http.ResponseWriter, status int) { http.Error(w, http.StatusText(status), status) }
+func (a *app) login(w http.ResponseWriter, r *http.Request) {
+	challenge := r.URL.Query().Get("login_challenge")
+	if _, err := a.request(r, "login", challenge); err != nil {
+		fail(w, 403)
+		return
+	}
+	state := random()
+	p := pending{challenge, random(), oauth2.GenerateVerifier(), time.Now().Add(10 * time.Minute)}
+	a.mu.Lock()
+	for k, v := range a.pending {
+		if time.Now().After(v.Expires) {
+			delete(a.pending, k)
+		}
+	}
+	if len(a.pending) >= 1024 {
+		a.mu.Unlock()
+		fail(w, 503)
+		return
+	}
+	a.pending[state] = p
+	a.mu.Unlock()
+	http.SetCookie(w, &http.Cookie{Name: cookieName, Value: state, Path: "/", Secure: true, HttpOnly: true, SameSite: http.SameSiteLaxMode, MaxAge: 600})
+	http.Redirect(w, r, a.oauth.AuthCodeURL(state, oidc.Nonce(p.Nonce), oauth2.S256ChallengeOption(p.Verifier)), http.StatusSeeOther)
+}
+func (a *app) take(r *http.Request) (pending, error) {
+	state := r.URL.Query().Get("state")
+	cookie, err := r.Cookie(cookieName)
+	if err != nil || state == "" || subtle.ConstantTimeCompare([]byte(cookie.Value), []byte(state)) != 1 {
+		return pending{}, errors.New("state mismatch")
+	}
+	a.mu.Lock()
+	defer a.mu.Unlock()
+	p, ok := a.pending[state]
+	delete(a.pending, state)
+	if !ok || time.Now().After(p.Expires) {
+		return pending{}, errors.New("expired or used state")
+	}
+	return p, nil
+}
+func (a *app) callback(w http.ResponseWriter, r *http.Request) {
+	p, err := a.take(r)
+	if err != nil {
+		fail(w, 403)
+		return
+	}
+	http.SetCookie(w, &http.Cookie{Name: cookieName, Path: "/", Secure: true, HttpOnly: true, SameSite: http.SameSiteLaxMode, MaxAge: -1})
+	if r.URL.Query().Get("error") != "" || r.URL.Query().Get("code") == "" {
+		fail(w, 403)
+		return
+	}
+	ctx := oidc.ClientContext(r.Context(), a.client)
+	token, err := a.oauth.Exchange(ctx, r.URL.Query().Get("code"), oauth2.VerifierOption(p.Verifier))
+	if err != nil {
+		fail(w, 502)
+		return
+	}
+	raw, ok := token.Extra("id_token").(string)
+	if !ok {
+		fail(w, 502)
+		return
+	}
+	id, err := a.verifier.Verify(ctx, raw)
+	if err != nil || id.Nonce != p.Nonce || id.Subject == "" {
+		fail(w, 403)
+		return
+	}
+	var c claims
+	if id.Claims(&c) != nil || c.Username == "" || c.Email == "" || !c.EmailVerified {
+		fail(w, 403)
+		return
+	}
+	if _, err := a.request(r, "login", p.Challenge); err != nil {
+		fail(w, 403)
+		return
+	}
+	// Stable identity is tied to the verified upstream issuer+subject, never email.
+	sum := sha256.Sum256([]byte(id.Issuer + "\x00" + id.Subject))
+	a.accept(w, r, "login", p.Challenge, map[string]any{"subject": "human:" + hex.EncodeToString(sum[:]), "remember": false, "context": c})
+}
+func consentSession(f flowRequest) (map[string]any, error) {
+	if !strings.HasPrefix(f.Subject, "human:") || f.Context.Username == "" || f.Context.Email == "" || !f.Context.EmailVerified {
+		return nil, errors.New("invalid identity context")
+	}
+	allowed := map[string]bool{"openid": true, "profile": true, "email": true, "groups": true}
+	session := map[string]any{"principal_type": "human"}
+	for _, scope := range f.Scopes {
+		if !allowed[scope] {
+			return nil, errors.New("scope not allowed")
+		}
+		switch scope {
+		case "profile":
+			session["preferred_username"] = f.Context.Username
+			session["name"] = f.Context.Name
+		case "email":
+			session["email"] = f.Context.Email
+			session["email_verified"] = true
+		case "groups":
+			session["groups"] = f.Context.Groups
+		}
+	}
+	if len(f.Audience) > 0 {
+		return nil, errors.New("access token audience not allowed")
+	}
+	return session, nil
+}
+func (a *app) consent(w http.ResponseWriter, r *http.Request) {
+	challenge := r.URL.Query().Get("consent_challenge")
+	f, err := a.request(r, "consent", challenge)
+	if err != nil {
+		fail(w, 403)
+		return
+	}
+	session, err := consentSession(f)
+	if err != nil {
+		fail(w, 403)
+		return
+	}
+	// Explicit policy for pre-approved first-party clients only; no generic auto-consent.
+	a.accept(w, r, "consent", challenge, map[string]any{"grant_scope": f.Scopes, "remember": false, "session": map[string]any{"id_token": session}})
+}
+func (a *app) handler() http.Handler {
+	mux := http.NewServeMux()
+	mux.HandleFunc("GET /healthz", func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(200) })
+	mux.HandleFunc("GET /login", a.login)
+	mux.HandleFunc("GET /callback", a.callback)
+	mux.HandleFunc("GET /consent", a.consent)
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.Header().Set("Cache-Control", "no-store")
+		w.Header().Set("Referrer-Policy", "no-referrer")
+		w.Header().Set("X-Content-Type-Options", "nosniff")
+		w.Header().Set("Content-Security-Policy", "default-src 'none'; frame-ancestors 'none'")
+		mux.ServeHTTP(w, r)
+	})
+}
+func main() {
+	client := &http.Client{Timeout: 15 * time.Second, CheckRedirect: func(r *http.Request, via []*http.Request) error { return http.ErrUseLastResponse }}
+	issuer := required("UPSTREAM_ISSUER")
+	ctx := oidc.ClientContext(context.Background(), client)
+	provider, err := oidc.NewProvider(ctx, issuer)
+	if err != nil {
+		log.Fatal("upstream discovery failed")
+	}
+	clientID := required("UPSTREAM_CLIENT_ID")
+	a := &app{admin: required("HYDRA_ADMIN_URL"), public: required("HYDRA_PUBLIC_URL"), client: client, allowed: map[string]bool{}, pending: map[string]pending{},
+		oauth:    oauth2.Config{ClientID: clientID, ClientSecret: required("UPSTREAM_CLIENT_SECRET"), RedirectURL: required("CALLBACK_URL"), Endpoint: provider.Endpoint(), Scopes: []string{"openid", "profile", "email", "groups"}},
+		verifier: provider.Verifier(&oidc.Config{ClientID: clientID})}
+	for _, id := range strings.Split(required("ALLOWED_CLIENTS"), ",") {
+		a.allowed[id] = true
+	}
+	s := http.Server{Addr: ":8080", Handler: a.handler(), ReadHeaderTimeout: 5 * time.Second, ReadTimeout: 20 * time.Second, WriteTimeout: 45 * time.Second, IdleTimeout: 60 * time.Second, MaxHeaderBytes: 16384}
+	log.Print("login/consent adapter listening on :8080")
+	log.Fatal(s.ListenAndServe())
+}

apps/hydra/login-consent/main_test.go

@@ -0,0 +1,107 @@
+package main
+
+import (
+	"encoding/json"
+	"net/http"
+	"net/http/httptest"
+	"net/url"
+	"strings"
+	"testing"
+	"time"
+
+	"golang.org/x/oauth2"
+)
+
+func TestStateBoundToCookieSingleUseAndExpiry(t *testing.T) {
+	a := &app{pending: map[string]pending{"valid": {Challenge: "challenge", Expires: time.Now().Add(time.Minute)}, "expired": {Expires: time.Now().Add(-time.Minute)}}}
+	request := func(state, cookie string) *http.Request {
+		r := httptest.NewRequest("GET", "https://login.example/callback?state="+state, nil)
+		if cookie != "" {
+			r.AddCookie(&http.Cookie{Name: cookieName, Value: cookie})
+		}
+		return r
+	}
+	for _, r := range []*http.Request{request("valid", ""), request("valid", "other"), request("expired", "expired")} {
+		if _, err := a.take(r); err == nil {
+			t.Fatal("invalid state accepted")
+		}
+	}
+	if p, err := a.take(request("valid", "valid")); err != nil || p.Challenge != "challenge" {
+		t.Fatal("valid state rejected")
+	}
+	if _, err := a.take(request("valid", "valid")); err == nil {
+		t.Fatal("replayed state accepted")
+	}
+}
+func TestConsentRejectsPrivilegeExpansionAndFiltersClaims(t *testing.T) {
+	f := flowRequest{Subject: "human:known", Scopes: []string{"openid", "email"}, Context: claims{Username: "alice", Email: "[email protected]", EmailVerified: true, Groups: []string{"operators"}}}
+	s, err := consentSession(f)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if _, ok := s["groups"]; ok {
+		t.Fatal("groups leaked without scope")
+	}
+	if _, ok := s["preferred_username"]; ok {
+		t.Fatal("profile leaked without scope")
+	}
+	for _, scope := range []string{"admin", "offline_access", "unknown"} {
+		bad := f
+		bad.Scopes = append([]string{"openid"}, scope)
+		if _, err := consentSession(bad); err == nil {
+			t.Fatalf("accepted %s", scope)
+		}
+	}
+	f.Audience = []string{"other-service"}
+	if _, err := consentSession(f); err == nil {
+		t.Fatal("unexpected audience accepted")
+	}
+	f.Audience = nil
+	f.Context.EmailVerified = false
+	if _, err := consentSession(f); err == nil {
+		t.Fatal("unverified email accepted")
+	}
+}
+func TestLoginValidatesClientAndUsesPKCEAndNonce(t *testing.T) {
+	admin := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		json.NewEncoder(w).Encode(map[string]any{"client": map[string]string{"client_id": r.URL.Query().Get("login_challenge")}})
+	}))
+	defer admin.Close()
+	a := &app{admin: admin.URL, client: admin.Client(), allowed: map[string]bool{"gitea": true}, pending: map[string]pending{}, oauth: oauth2.Config{ClientID: "hydra-login", RedirectURL: "https://login.example/callback", Endpoint: oauth2.Endpoint{AuthURL: "https://upstream.example/authorize"}}}
+	w := httptest.NewRecorder()
+	a.handler().ServeHTTP(w, httptest.NewRequest("GET", "https://login.example/login?login_challenge=rogue", nil))
+	if w.Code != 403 {
+		t.Fatal("unknown client accepted")
+	}
+	w = httptest.NewRecorder()
+	a.handler().ServeHTTP(w, httptest.NewRequest("GET", "https://login.example/login?login_challenge=gitea", nil))
+	if w.Code != 303 {
+		t.Fatalf("status %d", w.Code)
+	}
+	u, _ := url.Parse(w.Header().Get("Location"))
+	q := u.Query()
+	if q.Get("code_challenge_method") != "S256" || q.Get("code_challenge") == "" || q.Get("nonce") == "" || q.Get("state") == "" {
+		t.Fatal("missing protocol binding")
+	}
+	cookies := w.Result().Cookies()
+	if len(cookies) != 1 || !cookies[0].Secure || !cookies[0].HttpOnly || cookies[0].SameSite != http.SameSiteLaxMode || cookies[0].Value != q.Get("state") {
+		t.Fatal("unsafe cookie")
+	}
+	if w.Header().Get("Cache-Control") != "no-store" {
+		t.Fatal("missing cache protection")
+	}
+}
+func TestHydraRedirectCannotLeaveTrustedOrigin(t *testing.T) {
+	for _, target := range []string{"https://evil.example/oauth2/auth", "https://[email protected]/oauth2/auth", "https://hydra.example/other"} {
+		admin := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			json.NewEncoder(w).Encode(map[string]string{"redirect_to": target})
+		}))
+		a := &app{admin: admin.URL, public: "https://hydra.example", client: admin.Client()}
+		w := httptest.NewRecorder()
+		a.accept(w, httptest.NewRequest("GET", "https://login.example/login", nil), "login", "challenge", map[string]string{"subject": "human:test"})
+		if w.Code != 502 || strings.Contains(w.Header().Get("Location"), "evil") {
+			t.Fatal("untrusted redirect accepted")
+		}
+		admin.Close()
+	}
+}

apps/hydra/login-deployment.yaml

@@ -0,0 +1,69 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: hydra-login
+  namespace: hydra
+spec:
+  replicas: 1
+  strategy:
+    type: Recreate
+  selector:
+    matchLabels:
+      app: hydra-login
+  template:
+    metadata:
+      labels:
+        app: hydra-login
+    spec:
+      automountServiceAccountToken: false
+      securityContext:
+        runAsNonRoot: true
+        runAsUser: 65532
+        runAsGroup: 65532
+        seccompProfile:
+          type: RuntimeDefault
+      containers:
+      - name: login-consent
+        image: zot.ad.ddupan.top/iam/oidc-login-consent@sha256:fede9b9e93c457c4b7a8a6022d9df86ff5d5900d3f6b6c4f3439851a0ae1e944
+        env:
+        - name: HYDRA_ADMIN_URL
+          value: http://hydra-admin.hydra.svc.cluster.local:4445
+        - name: HYDRA_PUBLIC_URL
+          value: https://hydra.ad.ddupan.top
+        - name: UPSTREAM_ISSUER
+          value: https://auth.ddupan.top
+        - name: UPSTREAM_CLIENT_ID
+          value: hydra-login
+        - name: CALLBACK_URL
+          value: https://hydra-login.ad.ddupan.top/callback
+        - name: ALLOWED_CLIENTS
+          value: gitea
+        - name: UPSTREAM_CLIENT_SECRET
+          valueFrom:
+            secretKeyRef:
+              name: hydra
+              key: upstream_client_secret
+        ports:
+        - name: http
+          containerPort: 8080
+        resources:
+          requests:
+            cpu: 50m
+            memory: 64Mi
+          limits:
+            memory: 256Mi
+        securityContext:
+          allowPrivilegeEscalation: false
+          readOnlyRootFilesystem: true
+          capabilities:
+            drop:
+            - ALL
+        readinessProbe:
+          httpGet:
+            path: /healthz
+            port: http
+        livenessProbe:
+          httpGet:
+            path: /healthz
+            port: http
+          initialDelaySeconds: 10

apps/hydra/namespace.yaml

@@ -0,0 +1,6 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: hydra
+  labels:
+    pod-security.kubernetes.io/enforce: restricted

apps/hydra/networkpolicy.yaml

@@ -0,0 +1,46 @@
+apiVersion: networking.k8s.io/v1
+kind: NetworkPolicy
+metadata:
+  name: hydra
+  namespace: hydra
+spec:
+  podSelector:
+    matchLabels:
+      app: hydra
+  policyTypes:
+  - Ingress
+  ingress:
+  - from:
+    - namespaceSelector:
+        matchLabels:
+          kubernetes.io/metadata.name: envoy-gateway-system
+    ports:
+    - port: 4444
+      protocol: TCP
+  - from:
+    - podSelector:
+        matchLabels:
+          app: hydra-login
+    ports:
+    - port: 4445
+      protocol: TCP
+---
+apiVersion: networking.k8s.io/v1
+kind: NetworkPolicy
+metadata:
+  name: hydra-login
+  namespace: hydra
+spec:
+  podSelector:
+    matchLabels:
+      app: hydra-login
+  policyTypes:
+  - Ingress
+  ingress:
+  - from:
+    - namespaceSelector:
+        matchLabels:
+          kubernetes.io/metadata.name: envoy-gateway-system
+    ports:
+    - port: 8080
+      protocol: TCP

apps/hydra/services.yaml

@@ -0,0 +1,35 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: hydra-public
+  namespace: hydra
+spec:
+  selector:
+    app: hydra
+  ports:
+  - port: 4444
+    targetPort: 4444
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: hydra-admin
+  namespace: hydra
+spec:
+  selector:
+    app: hydra
+  ports:
+  - port: 4445
+    targetPort: 4445
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: hydra-login
+  namespace: hydra
+spec:
+  selector:
+    app: hydra-login
+  ports:
+  - port: 8080
+    targetPort: 8080

apps/nexus/README.md

@@ -0,0 +1,157 @@
+# Nexus Repository POC
+
+本目录声明一个 Nexus Repository Community Edition POC，用来验证一次性 CI runner 通过
+网络服务复用 Ansible Galaxy、Go Modules 与 OCI/BuildKit 缓存。Nexus 固定为 `3.96.1`，
+镜像固定到官方 multi-arch index digest；LAN 入口为
+`https://nexus.ad.ddupan.top`。
+
+## POC 边界
+
+- 单副本 Deployment，`Recreate` 更新，避免一个 RWO 卷被两个 Pod 同时挂载。
+- `/nexus-data` 使用 `localpv-zfs-ceph` 上的 50 GiB RWO PVC。
+- 资源预算为 250m/2 GiB request、2 CPU/4 GiB limit；JVM heap 上限 2 GiB。
+- 使用容器默认的 embedded H2。它只用于 POC；正式接管 OCI 制品前必须迁移到外部
+  PostgreSQL，并验证备份恢复。
+- 入口只在 LAN wildcard Gateway 上发布，不创建公网 DNS 或 Cloudflare route。
+- 不套 Authelia forward-auth；它会破坏 Go、Ansible 与 OCI 非浏览器客户端。
+- 现有 zot 保持不变。Nexus 完成 OCI、BuildKit cache 和恢复验收前不得迁移或删除 zot。
+
+Terraform provider 创建 `ansible-public`、`go-public`、最小匿名权限与 OCI Bearer Token
+Realm。Nexus 3.94 才加入的原生 OCI repository 已有 REST API，但当前锁定的 community
+provider 尚未暴露 OCI resource；`terraform/reconcile-oci.sh` 因此根据 3.96.1 实例 Swagger
+固定的 JSON schema，幂等调和 `oci-hosted`、`oci-proxy` 与 `oci-public`。不得绕过该入口在
+UI 中创建无人管理的长期 repository。
+
+## 部署
+
+Flux 从 `clusters/homelab/apps/nexus.yaml` 协调本目录，并依赖 Envoy Gateway 与 OpenEBS。
+合并前只渲染配置，不直接 apply：
+
+```bash
+kubectl kustomize apps/nexus
+```
+
+合并并由 Flux 部署后检查：
+
+```bash
+kubectl -n flux-system get kustomization nexus
+kubectl -n nexus get pod,pvc,service,httproute
+kubectl -n nexus logs deployment/nexus --tail=100
+```
+
+启动可能需要数分钟，startup probe 允许最多十分钟。不要因初次启动较慢反复删除 Pod；
+先确认 PVC 已 Bound、Pod 没有 OOM，以及日志仍在推进。
+
+DNS 期望状态已加入 `infrastructure/dns/records.yml`，需从 Samba AD Ansible root 以
+`--check --diff` 核对后再按其 README 应用 DNS tag。没有 DNS 时可先用 port-forward
+验证应用，但不能据此宣称 Gateway 路径已通过。
+
+## 首次初始化与 Terraform
+
+初始管理员密码生成在 PVC 的 `/nexus-data/admin.password`。只在交互式终端中读取并立即
+完成首次密码轮换；不得把密码复制进 shell tracing、工单、Git 或命令参数。随后将
+Terraform 管理账号的凭据存入 OpenBao，由 CI 通过 Terraform input variable 注入以下
+环境变量：
+
+```text
+TF_VAR_nexus_url=https://nexus.ad.ddupan.top
+TF_VAR_nexus_username=admin
+TF_VAR_nexus_password=<OpenBao kv/infra/nexus 的 admin_password 字段>
+```
+
+`terraform/` 使用 `sonatype-nexus-community/sonatyperepo` 1.17.0，当前声明：
+
+- `ansible-galaxy-proxy` → `https://galaxy.ansible.com`
+- `ansible-public` group
+- `go-proxy` → `https://proxy.golang.org`
+- `go-public` group
+
+provider credential 不写入 HCL 或 tfvars。正式 apply 前还必须为这个独立 Terraform root
+配置远端 backend；本地 state 只允许用于可丢弃的 POC，不提交。验证命令：
+
+```bash
+terraform -chdir=apps/nexus/terraform init -backend=false
+terraform -chdir=apps/nexus/terraform validate
+```
+
+先以 `--check` 查看 OCI repository 漂移，再明确 apply；脚本只从上述环境变量取得凭据，
+用临时 `0600` netrc 调用 REST API，退出时删除：
+
+```bash
+apps/nexus/terraform/reconcile-oci.sh --check
+apps/nexus/terraform/reconcile-oci.sh --apply
+```
+
+Terraform 同时把内置 `anonymous` 用户从默认的全仓库 `nx-anonymous` 角色收窄到
+`ansible-public`、其返回制品 URL 使用的 `ansible-galaxy-proxy`、`go-public`，以及
+`oci-public`/`oci-proxy` 的 `browse/read` 权限。`oci-hosted` 不向匿名用户开放。首次接管
+已有实例时先执行
+`terraform import sonatyperepo_user.anonymous anonymous,default`，再 apply；不要先启用默认的
+全仓库匿名读取。
+
+2026-09-20 的 POC 现场验收已确认：Flux 与 Pod Ready、PVC Bound、HTTPRoute 通过 HTTPS
+返回 Nexus 状态 200，Samba DNS 已幂等收敛。全新客户端目录通过匿名入口下载
+`community.general:11.2.0` 时冷缓存为 8.49 秒、热缓存为 1.89 秒，两次 tarball SHA-256
+一致；`golang.org/x/[email protected]` 为 2.92 秒与 1.51 秒。
+
+## 客户端验收
+
+先验证冷缓存，再原样重复命令验证热缓存；记录 Nexus 请求、上游流量和耗时，不只观察
+命令成功。Ansible 配置的 URL 必须以 `/` 结尾：
+
+```ini
+[galaxy]
+server_list = nexus
+
+[galaxy_server.nexus]
+url = https://nexus.ad.ddupan.top/repository/ansible-public/
+```
+
+```bash
+ansible-galaxy collection install -r collections/requirements.yml \
+  -p .ansible/collections
+```
+
+Go POC 使用：
+
+```bash
+GOPROXY=https://nexus.ad.ddupan.top/repository/go-public/ go mod download
+```
+
+私有 `git.ddupan.top/*` module 的 `GOPRIVATE`、凭据与是否允许 `direct` fallback 在实际
+workflow 中单独决定；不要让私有 module path 意外发往公共 proxy。
+
+OCI 使用 path-based routing：匿名公共拉取地址形如
+`nexus.ad.ddupan.top/oci-public/library/alpine:3.22`，认证写入地址形如
+`nexus.ad.ddupan.top/oci-hosted/<namespace>/<image>:<tag>`。2026-09-20 现场验收结果：
+
+- `oci-public` 匿名代理拉取 Alpine 冷缓存 4.75 秒、热缓存 0.80 秒，digest 一致；
+- `oci-hosted` 认证 push/pull 成功，匿名 pull 返回 401；
+- amd64/arm64 OCI image index push 成功，两个平台 manifest 可见；
+- Helm chart push/pull digest 与本地 tarball SHA-256 一致；
+- Cosign 3.1.3 使用一次性密钥签名并验证成功，OCI 1.1 referrers API 返回一个
+  `application/vnd.dev.sigstore.bundle.v0.3+json` artifact；
+- BuildKit `registry` cache 以 `mode=max` 导出成功，销毁首个 builder 后由新 builder 导入，
+  两个 `RUN` step 均明确命中 `CACHED`。
+
+本机安装的测试客户端包括 `/usr/local/bin/cosign` 3.1.3；安装时核对官方 Linux amd64
+binary SHA-256 `4629c757b7618056f8ddd7e2625ae9fdd94c0372a65049520bc7d9df9efc7f71`。
+上述结果仍不代表备份恢复、外部 PostgreSQL 或正式 publisher service account 已完成；
+这些项目通过前不得迁移或删除 zot。
+
+## 数据与恢复
+
+POC 的数据库、配置、blob、初始管理员状态都位于 `nexus-data` PVC。删除 Deployment
+不会删除 PVC；删除 PVC 会永久删除整个 POC。当前没有独立备份，不能将它用于唯一副本的
+正式制品。
+
+恢复验证至少包括：停止写入、取得一致备份、在独立 PVC/实例恢复、登录、列出 repository、
+拉取已缓存的 Ansible/Go 制品，并核对 OCI digest/referrers。正式化时再把数据库迁移至
+外部 PostgreSQL，并分别定义数据库与 blob 的备份、恢复顺序和 RPO。
+
+参考：
+
+- [Nexus OCI repositories](https://help.sonatype.com/en/oci-repositories.html)
+- [Nexus Ansible repositories](https://help.sonatype.com/en/ansible-repositories.html)
+- [Nexus Go repositories](https://help.sonatype.com/en/go-repositories.html)
+- [官方容器镜像](https://hub.docker.com/r/sonatype/nexus3)

apps/nexus/deployment.yaml

@@ -0,0 +1,82 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: nexus
+  namespace: nexus
+  labels:
+    app.kubernetes.io/name: nexus
+spec:
+  replicas: 1
+  strategy:
+    type: Recreate
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: nexus
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: nexus
+    spec:
+      automountServiceAccountToken: false
+      securityContext:
+        fsGroup: 200
+        fsGroupChangePolicy: OnRootMismatch
+        runAsGroup: 200
+        runAsNonRoot: true
+        runAsUser: 200
+        seccompProfile:
+          type: RuntimeDefault
+      terminationGracePeriodSeconds: 120
+      containers:
+        - name: nexus
+          image: docker.io/sonatype/nexus3:3.96.1@sha256:56142f13432cf072e017aebb2025f201e42ae36ff40bb82618c702504c61f7dd
+          imagePullPolicy: IfNotPresent
+          env:
+            - name: INSTALL4J_ADD_VM_PARAMS
+              value: >-
+                -Xms1024m -Xmx2048m -XX:MaxDirectMemorySize=1024m
+                -Djava.util.prefs.userRoot=/nexus-data/javaprefs
+          ports:
+            - name: http
+              containerPort: 8081
+              protocol: TCP
+          resources:
+            requests:
+              cpu: 250m
+              memory: 2Gi
+            limits:
+              cpu: "2"
+              memory: 4Gi
+          securityContext:
+            allowPrivilegeEscalation: false
+            capabilities:
+              drop:
+                - ALL
+          startupProbe:
+            httpGet:
+              path: /service/rest/v1/status
+              port: http
+            failureThreshold: 60
+            periodSeconds: 10
+            timeoutSeconds: 5
+          readinessProbe:
+            httpGet:
+              path: /service/rest/v1/status
+              port: http
+            failureThreshold: 6
+            periodSeconds: 10
+            timeoutSeconds: 5
+          livenessProbe:
+            httpGet:
+              path: /service/rest/v1/status
+              port: http
+            failureThreshold: 6
+            periodSeconds: 30
+            timeoutSeconds: 5
+          volumeMounts:
+            - name: data
+              mountPath: /nexus-data
+      volumes:
+        - name: data
+          persistentVolumeClaim:
+            claimName: nexus-data

apps/nexus/httproute.yaml

@@ -0,0 +1,16 @@
+apiVersion: gateway.networking.k8s.io/v1
+kind: HTTPRoute
+metadata:
+  name: nexus
+  namespace: nexus
+spec:
+  parentRefs:
+    - name: eg
+      namespace: envoy-gateway-system
+      sectionName: https
+  hostnames:
+    - nexus.ad.ddupan.top
+  rules:
+    - backendRefs:
+        - name: nexus
+          port: 8081

apps/nexus/kustomization.yaml

@@ -0,0 +1,9 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - namespace.yaml
+  - pvc.yaml
+  - deployment.yaml
+  - service.yaml
+  - httproute.yaml
+  - networkpolicy.yaml

apps/nexus/namespace.yaml

@@ -0,0 +1,8 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: nexus
+  labels:
+    pod-security.kubernetes.io/enforce: restricted
+    pod-security.kubernetes.io/audit: restricted
+    pod-security.kubernetes.io/warn: restricted

apps/nexus/networkpolicy.yaml

@@ -0,0 +1,23 @@
+apiVersion: networking.k8s.io/v1
+kind: NetworkPolicy
+metadata:
+  name: nexus-ingress
+  namespace: nexus
+spec:
+  podSelector:
+    matchLabels:
+      app.kubernetes.io/name: nexus
+  policyTypes:
+    - Ingress
+  ingress:
+    - from:
+        - namespaceSelector:
+            matchLabels:
+              kubernetes.io/metadata.name: envoy-gateway-system
+          podSelector:
+            matchLabels:
+              gateway.envoyproxy.io/owning-gateway-name: eg
+              gateway.envoyproxy.io/owning-gateway-namespace: envoy-gateway-system
+      ports:
+        - protocol: TCP
+          port: 8081

apps/nexus/pvc.yaml

@@ -0,0 +1,12 @@
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: nexus-data
+  namespace: nexus
+spec:
+  accessModes:
+    - ReadWriteOnce
+  resources:
+    requests:
+      storage: 50Gi
+  storageClassName: localpv-zfs-ceph

apps/nexus/service.yaml

@@ -0,0 +1,14 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: nexus
+  namespace: nexus
+spec:
+  type: ClusterIP
+  selector:
+    app.kubernetes.io/name: nexus
+  ports:
+    - name: http
+      port: 8081
+      protocol: TCP
+      targetPort: http

apps/nexus/terraform/.gitignore

@@ -0,0 +1,6 @@
+.terraform/
+*.tfstate
+*.tfstate.*
+*.tfplan
+crash.log
+crash.*.log

apps/nexus/terraform/.terraform.lock.hcl

@@ -0,0 +1,24 @@
+# This file is maintained automatically by "terraform init".
+# Manual edits may be lost in future updates.
+
+provider "registry.terraform.io/sonatype-nexus-community/sonatyperepo" {
+  version     = "1.17.0"
+  constraints = "1.17.0"
+  hashes = [
+    "h1:uKhvhhhI7B+HBsh0zq/ybqKt+EnOGyI6rjcRCtj79ZA=",
+    "zh:0dde99e7b343fa01f8eefc378171fb8621bedb20f59157d6cc8e3d46c738105f",
+    "zh:3315929df254a3a6ac27c8c846c2006f7d2a91fadc014351bc4d617f948e5bf9",
+    "zh:36be5a455af3ce4e187de26753de63e78c1ee9a32dba0135c6cf96a6c1fff25f",
+    "zh:3f73f7ff57b8c339a7c7ac37653e2dc0b2dd9dcc3f3a538788e7e3ac838337b2",
+    "zh:40286ecca4c22ab7ae90618ac6d2743f5055199dac81cf5204a4a397c784d439",
+    "zh:4d24e5c0195fb3155b1967583ee64cfeda402d7cc7f3c73369438f6c69f4245b",
+    "zh:828a9d7aceaac36af7f9c07af43ec8d20a89148780645d170ffb1c68b2da792d",
+    "zh:a5ab04de3fe626ec57c832618c6f990abd6610f81e132621651e0b180b970cff",
+    "zh:a959fa6090a8c0f53739879184e7346423494aee598003df0d1ab4a22b2eee91",
+    "zh:bdda26c2f03f918bbe59e75abea44868fafda019c3a543725331195df126350b",
+    "zh:d8048e149ee97ba62971e6a79355d59887bc6d10fcf72cc2feff3d0a2582670c",
+    "zh:dd36f9988af4e1ca5b1ca7b7bb6f658df9a220dfcda7fec7392fedfe9064f652",
+    "zh:dda2688d46c7e539fe97e8fe9d3ec81fb364170e018d9c6a681364c8955d4e9d",
+    "zh:e6b519afe2dea1c0434f766eb6bc9ba78cc5b6ef2c311c2ca3c65cb24744f31f",
+  ]
+}

apps/nexus/terraform/oci/oci-hosted.json

@@ -0,0 +1,17 @@
+{
+  "name": "oci-hosted",
+  "online": true,
+  "storage": {
+    "blobStoreName": "default",
+    "strictContentTypeValidation": true,
+    "writePolicy": "ALLOW",
+    "latestPolicy": false
+  },
+  "oci": {
+    "v1Enabled": false,
+    "forceBasicAuth": false,
+    "pathEnabled": true
+  },
+  "component": { "proprietaryComponents": false },
+  "cosign": { "enforcement": "NONE" }
+}

apps/nexus/terraform/oci/oci-proxy.json

@@ -0,0 +1,27 @@
+{
+  "name": "oci-proxy",
+  "online": true,
+  "storage": {
+    "blobStoreName": "default",
+    "strictContentTypeValidation": true
+  },
+  "oci": {
+    "v1Enabled": false,
+    "forceBasicAuth": false,
+    "pathEnabled": true
+  },
+  "ociProxy": {
+    "indexType": "HUB",
+    "cacheForeignLayers": false,
+    "foreignLayerUrlWhitelist": []
+  },
+  "proxy": {
+    "remoteUrl": "https://registry-1.docker.io",
+    "contentMaxAge": 1440,
+    "metadataMaxAge": 60,
+    "preserveEncodedCharacters": false
+  },
+  "negativeCache": { "enabled": true, "timeToLive": 60 },
+  "httpClient": { "blocked": false, "autoBlock": true },
+  "cosign": { "enforcement": "NONE" }
+}

apps/nexus/terraform/oci/oci-public.json

@@ -0,0 +1,15 @@
+{
+  "name": "oci-public",
+  "online": true,
+  "storage": {
+    "blobStoreName": "default",
+    "strictContentTypeValidation": true
+  },
+  "group": { "memberNames": ["oci-proxy"] },
+  "oci": {
+    "v1Enabled": false,
+    "forceBasicAuth": false,
+    "pathEnabled": true
+  },
+  "cosign": { "enforcement": "NONE" }
+}

apps/nexus/terraform/reconcile-oci.sh

@@ -0,0 +1,80 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+mode="${1:---check}"
+case "$mode" in
+  --check | --apply) ;;
+  *) echo "usage: $0 [--check|--apply]" >&2; exit 2 ;;
+esac
+
+: "${TF_VAR_nexus_url:?set TF_VAR_nexus_url}"
+: "${TF_VAR_nexus_username:?set TF_VAR_nexus_username}"
+: "${TF_VAR_nexus_password:?set TF_VAR_nexus_password}"
+
+script_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"
+auth_file="$(mktemp /tmp/nexus-oci-auth.XXXXXX)"
+trap 'rm -f -- "$auth_file"' EXIT
+chmod 0600 "$auth_file"
+printf 'machine %s\nlogin %s\npassword %s\n' \
+  "${TF_VAR_nexus_url#*://}" "$TF_VAR_nexus_username" \
+  "$TF_VAR_nexus_password" >"$auth_file"
+
+drift=0
+for entry in \
+  "hosted:$script_dir/oci/oci-hosted.json" \
+  "proxy:$script_dir/oci/oci-proxy.json" \
+  "group:$script_dir/oci/oci-public.json"; do
+  repository_type="${entry%%:*}"
+  desired_file="${entry#*:}"
+  repository_name="$(jq -er '.name' "$desired_file")"
+  endpoint="$TF_VAR_nexus_url/service/rest/v1/repositories/oci/$repository_type"
+  current_file="$(mktemp /tmp/nexus-oci-current.XXXXXX)"
+
+  status="$(curl --silent --show-error --netrc-file "$auth_file" \
+    --output "$current_file" --write-out '%{http_code}' \
+    "$endpoint/$repository_name")"
+
+  if [[ "$status" == 404 ]]; then
+    drift=1
+    if [[ "$mode" == --apply ]]; then
+      curl --fail --silent --show-error --netrc-file "$auth_file" \
+        --header 'Content-Type: application/json' \
+        --data-binary "@$desired_file" "$endpoint"
+      echo "created $repository_name"
+    else
+      echo "missing $repository_name" >&2
+    fi
+  elif [[ "$status" == 200 ]]; then
+    if jq -e --slurpfile desired "$desired_file" '
+      def subset($actual; $wanted):
+        if ($wanted | type) == "object" then
+          all($wanted | keys[];
+            ($actual[.] != null) and subset($actual[.]; $wanted[.]))
+        else
+          $actual == $wanted
+        end;
+      subset(.; $desired[0])
+    ' "$current_file" >/dev/null; then
+      echo "in sync $repository_name"
+    else
+      drift=1
+      if [[ "$mode" == --apply ]]; then
+        curl --fail --silent --show-error --netrc-file "$auth_file" \
+          --request PUT --header 'Content-Type: application/json' \
+          --data-binary "@$desired_file" "$endpoint/$repository_name"
+        echo "updated $repository_name"
+      else
+        echo "drifted $repository_name" >&2
+      fi
+    fi
+  else
+    cat "$current_file" >&2
+    echo "unexpected HTTP $status for $repository_name" >&2
+    exit 1
+  fi
+  rm -f -- "$current_file"
+done
+
+if [[ "$mode" == --check && "$drift" -ne 0 ]]; then
+  exit 1
+fi

apps/nexus/terraform/repositories.tf

@@ -0,0 +1,64 @@
+locals {
+  proxy_http_client = {
+    auto_block = true
+    blocked    = false
+  }
+
+  proxy_negative_cache = {
+    enabled      = true
+    time_to_live = 60
+  }
+
+  repository_storage = {
+    blob_store_name                = "default"
+    strict_content_type_validation = true
+  }
+}
+
+resource "sonatyperepo_repository_ansiblegalaxy_proxy" "galaxy" {
+  name   = "ansible-galaxy-proxy"
+  online = true
+
+  http_client    = local.proxy_http_client
+  negative_cache = local.proxy_negative_cache
+  proxy = {
+    remote_url       = "https://galaxy.ansible.com"
+    content_max_age  = 1440
+    metadata_max_age = 60
+  }
+  storage = local.repository_storage
+}
+
+resource "sonatyperepo_repository_ansiblegalaxy_group" "public" {
+  name   = "ansible-public"
+  online = true
+
+  group = {
+    member_names = [sonatyperepo_repository_ansiblegalaxy_proxy.galaxy.name]
+  }
+  storage = local.repository_storage
+}
+
+resource "sonatyperepo_repository_go_proxy" "public" {
+  name   = "go-proxy"
+  online = true
+
+  http_client    = local.proxy_http_client
+  negative_cache = local.proxy_negative_cache
+  proxy = {
+    remote_url       = "https://proxy.golang.org"
+    content_max_age  = 1440
+    metadata_max_age = 60
+  }
+  storage = local.repository_storage
+}
+
+resource "sonatyperepo_repository_go_group" "public" {
+  name   = "go-public"
+  online = true
+
+  group = {
+    member_names = [sonatyperepo_repository_go_proxy.public.name]
+  }
+  storage = local.repository_storage
+}

apps/nexus/terraform/security.tf

@@ -0,0 +1,75 @@
+resource "sonatyperepo_privilege_repository_view" "anonymous_ansible" {
+  name        = "ci-anonymous-ansible-read"
+  description = "Anonymous read access to the Ansible Galaxy group"
+  actions     = ["BROWSE", "READ"]
+  format      = "ansiblegalaxy"
+  repository  = sonatyperepo_repository_ansiblegalaxy_group.public.name
+}
+
+resource "sonatyperepo_privilege_repository_view" "anonymous_ansible_proxy" {
+  name        = "ci-anonymous-ansible-proxy-read"
+  description = "Anonymous artifact read access to the Ansible Galaxy proxy"
+  actions     = ["BROWSE", "READ"]
+  format      = "ansiblegalaxy"
+  repository  = sonatyperepo_repository_ansiblegalaxy_proxy.galaxy.name
+}
+
+resource "sonatyperepo_privilege_repository_view" "anonymous_go" {
+  name        = "ci-anonymous-go-read"
+  description = "Anonymous read access to the Go module group"
+  actions     = ["BROWSE", "READ"]
+  format      = "go"
+  repository  = sonatyperepo_repository_go_group.public.name
+}
+
+resource "sonatyperepo_privilege_repository_view" "anonymous_oci_public" {
+  name        = "ci-anonymous-oci-public-read"
+  description = "Anonymous read access to the public OCI group"
+  actions     = ["BROWSE", "READ"]
+  format      = "oci"
+  repository  = "oci-public"
+}
+
+resource "sonatyperepo_privilege_repository_view" "anonymous_oci_proxy" {
+  name        = "ci-anonymous-oci-proxy-read"
+  description = "Anonymous read access to the OCI proxy member"
+  actions     = ["BROWSE", "READ"]
+  format      = "oci"
+  repository  = "oci-proxy"
+}
+
+resource "sonatyperepo_role" "anonymous_ci" {
+  id          = "ci-anonymous-read"
+  name        = "CI anonymous read"
+  description = "Read-only access to public CI dependency proxy groups"
+  privileges = [
+    sonatyperepo_privilege_repository_view.anonymous_ansible.name,
+    sonatyperepo_privilege_repository_view.anonymous_ansible_proxy.name,
+    sonatyperepo_privilege_repository_view.anonymous_go.name,
+    sonatyperepo_privilege_repository_view.anonymous_oci_public.name,
+    sonatyperepo_privilege_repository_view.anonymous_oci_proxy.name,
+  ]
+  roles = []
+}
+
+resource "sonatyperepo_user" "anonymous" {
+  user_id       = "anonymous"
+  first_name    = "Anonymous"
+  last_name     = "User"
+  email_address = "[email protected]"
+  status        = "active"
+  roles         = [sonatyperepo_role.anonymous_ci.id]
+}
+
+resource "sonatyperepo_system_anonymous_access" "ci" {
+  enabled    = true
+  user_id    = sonatyperepo_user.anonymous.user_id
+  realm_name = "NexusAuthorizingRealm"
+}
+
+resource "sonatyperepo_security_realms" "active" {
+  active = [
+    "NexusAuthenticatingRealm",
+    "OciBearerToken",
+  ]
+}

apps/nexus/terraform/versions.tf

@@ -0,0 +1,33 @@
+terraform {
+  required_version = ">= 1.11.0"
+
+  required_providers {
+    sonatyperepo = {
+      source  = "sonatype-nexus-community/sonatyperepo"
+      version = "1.17.0"
+    }
+  }
+}
+
+provider "sonatyperepo" {
+  url      = var.nexus_url
+  username = var.nexus_username
+  password = var.nexus_password
+}
+
+variable "nexus_url" {
+  description = "Nexus Repository base URL"
+  type        = string
+}
+
+variable "nexus_username" {
+  description = "Nexus Terraform management username"
+  type        = string
+  sensitive   = true
+}
+
+variable "nexus_password" {
+  description = "Nexus Terraform management password"
+  type        = string
+  sensitive   = true
+}

apps/ps3netsrv/docker-compose.yml

@@ -18,4 +18,4 @@ networks:
   default:
     ipam:
       config:
-      - subnet: 172.28.1.0/24
+        - subnet: 172.28.1.0/24

apps/zot/values.yaml

@@ -70,11 +70,11 @@ configFiles:
         },
         "accessControl": {
           "repositories": {
-            "panxiao81/gitea-dynamic-runner-controller": {
+            "panxiao81/backstage": {
               "policies": [
                 {
                   "users": [
-                    "spiffe://ddupan.top/ci/panxiao81/gitea-dynamic-runner/publish-images",
+                    "spiffe://ddupan.top/ci/panxiao81/backstage/image",
                     "spiffe://ddupan.top/dev/panxiao81"
                   ],
                   "actions": [
@@ -88,18 +88,42 @@ configFiles:
                 "read"
               ]
             },
-            "panxiao81/gitea-dynamic-runner-runner": {
+            "panxiao81/gitea-dynamic-runner-controller": {
               "policies": [
                 {
                   "users": [
-                    "spiffe://ddupan.top/ci/panxiao81/gitea-dynamic-runner/publish-images",
+                    "spiffe://ddupan.top/ci/panxiao81/gitea-dynamic-runner/publish-images"
-                    "spiffe://ddupan.top/dev/panxiao81"
                   ],
                   "actions": [
                     "read",
                     "create",
                     "update"
                   ]
+                },
+                {
+                  "users": ["spiffe://ddupan.top/dev/panxiao81"],
+                  "actions": ["read", "create", "update", "delete"]
+                }
+              ],
+              "defaultPolicy": [
+                "read"
+              ]
+            },
+            "panxiao81/gitea-dynamic-runner-runner": {
+              "policies": [
+                {
+                  "users": [
+                    "spiffe://ddupan.top/ci/panxiao81/gitea-dynamic-runner/publish-images"
+                  ],
+                  "actions": [
+                    "read",
+                    "create",
+                    "update"
+                  ]
+                },
+                {
+                  "users": ["spiffe://ddupan.top/dev/panxiao81"],
+                  "actions": ["read", "create", "update", "delete"]
                 }
               ],
               "defaultPolicy": [

clusters/homelab/README.md

@@ -50,6 +50,9 @@ sudo k3s kubectl -n flux-system get gitrepositories,kustomizations
 - VictoriaMetrics Operator 已固定现有 chart `0.66.2` 并完成分阶段 Flux HelmRelease
   接管；Metrics、Logs、Traces 与 Grafana 也已统一完成 Flux 接管；
 - External Secrets Operator 已固定 chart `2.8.0` 并完成分阶段接管；
-- SPIRE 已按官方 hardened chart `0.30.2`（SPIRE `1.15.3`）声明，使用共享
+- SPIRE 已按 hardened chart 内部 fork `0.30.2-ddupan.1`（基于上游 `0.30.2`，SPIRE
+  `1.15.3`）声明，使用共享
   PostgreSQL 与独立 signing-key PVC；首次上线和 OpenBao JWT-SVID PoC 尚待合并后验证；
+- Nexus Repository CE POC 已加入 GitOps 声明，计划验证 Ansible Galaxy、Go Modules 与
+  OCI/BuildKit 缓存；尚未部署或完成现场验收，现有 zot 保持不变；
 - root Kustomization 与所有 brownfield 子 Kustomization 继续保持 `prune: false`。

clusters/homelab/apps/backstage.yaml

@@ -0,0 +1,22 @@
+apiVersion: kustomize.toolkit.fluxcd.io/v1
+kind: Kustomization
+metadata:
+  name: backstage
+  namespace: flux-system
+spec:
+  dependsOn:
+    - name: envoy-gateway
+    - name: external-secrets
+  healthChecks:
+    - apiVersion: apps/v1
+      kind: Deployment
+      name: backstage
+      namespace: backstage
+  interval: 10m
+  path: ./apps/backstage
+  prune: true
+  sourceRef:
+    kind: GitRepository
+    name: flux-system
+  timeout: 5m
+  wait: true

clusters/homelab/apps/dynamic-runner.yaml

@@ -6,11 +6,12 @@ metadata:
 spec:
   dependsOn:
     - name: external-secrets
-    - name: nats
     - name: spire
   interval: 10m
   path: ./platform/dynamic-runner
-  prune: false
+  # The runner backends are replaceable. Prune is required when a retired
+  # worker is removed from the component; otherwise it keeps consuming work.
+  prune: true
   sourceRef:
     kind: GitRepository
     name: flux-system

clusters/homelab/apps/hydra.yaml

@@ -0,0 +1,17 @@
+apiVersion: kustomize.toolkit.fluxcd.io/v1
+kind: Kustomization
+metadata:
+  name: hydra
+  namespace: flux-system
+spec:
+  dependsOn:
+  - name: envoy-gateway
+  - name: external-secrets
+  interval: 10m
+  path: ./apps/hydra
+  prune: false
+  sourceRef:
+    kind: GitRepository
+    name: flux-system
+  timeout: 5m
+  wait: true

clusters/homelab/apps/nexus.yaml

@@ -0,0 +1,22 @@
+apiVersion: kustomize.toolkit.fluxcd.io/v1
+kind: Kustomization
+metadata:
+  name: nexus
+  namespace: flux-system
+spec:
+  dependsOn:
+    - name: envoy-gateway
+    - name: openebs
+  healthChecks:
+    - apiVersion: apps/v1
+      kind: Deployment
+      name: nexus
+      namespace: nexus
+  interval: 10m
+  path: ./apps/nexus
+  prune: false
+  sourceRef:
+    kind: GitRepository
+    name: flux-system
+  timeout: 15m
+  wait: true

clusters/homelab/kustomization.yaml

@@ -7,7 +7,7 @@ resources:
   - apps/envoy-gateway.yaml
   - apps/external-secrets.yaml
   - apps/gitea.yaml
-  - apps/gitea-actions.yaml
+  - apps/backstage.yaml
   - apps/http-echo.yaml
   - apps/openebs.yaml
   - apps/nats.yaml
@@ -15,3 +15,5 @@ resources:
   - apps/spire.yaml
   - apps/observability.yaml
   - apps/zot.yaml
+  - apps/nexus.yaml
+  - apps/hydra.yaml

clusters/sandbox/README.md

@@ -0,0 +1,72 @@
+# Sandbox 集群
+
+这里是 OpenSandbox、CI 和 AI Agent workload 所在双节点 k3s 集群的 Flux
+reconciliation 入口。LXC、PostgreSQL、K3s、固定版本的 Flux controllers 与 root
+sync 由 `infrastructure/sandbox-cluster/` 中的 Ansible 管理；本目录只组合集群内
+workload。
+
+Flux 通过 `https://git.ddupan.top/panxiao81/homelab-infra.git` 读取公开仓库。
+Ansible 将 homelab CA 注入 `GitRepository/flux-system` 引用的同名 Secret，不使用
+长期 Git 凭据。root Kustomization 从 `./clusters/sandbox` 开始 reconciliation，
+初始保持 `prune: false`。
+
+Root bootstrap 已完成。后续按依赖顺序分别引入：
+
+1. 监控 CRD、kube-state-metrics 以及 kubelet/cAdvisor 抓取配置；
+2. SPIRE Agent、SPIFFE CSI Driver 与 workload registration；
+3. Kata Containers、`block-plain` RuntimeClass；
+4. 独立 External Secrets Operator 与 sandbox 专用 OpenBao auth backend；
+5. OpenSandbox controller/server；CI Pool 与 runner 调度器随后独立接入。
+
+每一阶段单独合并并等待对应 Flux Kustomization Ready，不在 bootstrap 时一次性部署。
+第一阶段监控拆为 `monitoring-operator` 与依赖它的 `monitoring`，防止 VM CR 在
+VictoriaMetrics Operator CRD Ready 前进入 reconciliation。
+
+SPIRE 阶段先由 `spire-bootstrap` 安装 CRD，并声明按上游 k8s_psat Server plugin
+要求收窄的 reviewer：它可以调用 TokenReview，并只读查询用于证明的 Pod 与 Node。
+Agent ServiceAccount 留给后续 HelmRelease 创建，避免两个声明方争夺同一资源。随后运行
+`infrastructure/sandbox-cluster/ansible/spire-bootstrap.yml`：playbook 从 sandbox
+读取 reviewer token，在内存中组成受限 kubeconfig，再通过 stdin reconcile 到 central
+集群的 `spire-server/spire-external-kubeconfigs` Secret。凭据不写入仓库、日志或控制机
+文件；该 Secret 准备完成后，才能启用 central external PSAT/controller-manager 和
+sandbox Agent/CSI。
+
+External controller-manager 使用独立的 `spire-controller-manager` ServiceAccount；其
+RBAC 与上游 controller-manager 所需权限一致，用于读取 workload selectors、维护
+SPIFFE CR status/finalizer 和 leader election。它不复用只允许 TokenReview 的 Server
+reviewer。Ansible 将两份 kubeconfig 写入同一个 central Secret 的不同 key，便于 central
+chart 分别绑定 `sandbox` 与 `sandbox-controller`。
+
+Central SPIRE Server 通过内网 `spire-server.ad.ddupan.top:8081` 接收 sandbox Agent
+attestation。Server 使用 external bundle publisher 持续维护 sandbox
+`spire-system/spire-bundle`，Agent 不固定或复制 trust bundle。Sandbox HelmRelease
+显式关闭 Server 与 OIDC Provider，只部署 Agent DaemonSet 和 SPIFFE CSI Driver；因此
+不会产生第二个 trust root。
+
+`spire-smoke` namespace、ServiceAccount 和 `sandbox-spire-smoke` ClusterSPIFFEID 只用于
+普通 Pod 的 CSI 回归夹具，稳定身份为 `spiffe://ddupan.top/sandbox/smoke`。Kata guest
+不能复用 node Agent 暴露的 Unix socket；virtio-fs 只能呈现 socket 路径，不能把连接
+跨过 VM 边界。Kata workload 必须使用 guest 内 Agent，具体约束见
+`platform/sandbox-kata/README.md`。测试 Pod 临时创建并在验收后删除，普通 Pod 的身份
+声明保留。
+
+Kata 阶段使用官方 4.1.0 `kata-deploy` chart 的短生命周期 `job` 模式，逐节点安装并
+重启 K3s。只启用 `kata-clh-runtime-rs`，不创建默认 `kata` 别名；该 handler 的
+`emptyDir` 固定使用 `block-plain`，为 Docker/BuildKit overlay2 与 kind 提供 guest
+内块设备文件系统。详细限制与上线验收见 `platform/sandbox-kata/README.md`。
+
+Sandbox 的 ESO 通过独立 `auth/kubernetes-sandbox` 向 OpenBao 证明 ServiceAccount
+身份，只能读取共享的 `kv/k8s/opensandbox-api`。它不保存 reviewer JWT 或长期 Bao token；相关
+Terraform 与 Flux 边界见 `platform/sandbox-external-secrets/README.md`。
+
+OpenSandbox 阶段固定官方源码 commit 与 umbrella chart `0.2.2`，只部署 controller、
+ClusterIP server 和 CRD。API key 由 ESO 从 OpenBao 投影，明文不进入 Git。
+
+## 监控边界
+
+这里只管理 sandbox LXC 内的 Kubernetes 监控，不负责 PVE 宿主监控。LXC 与宿主共享
+内核，即使 lxcfs 虚拟化了内存和 uptime，容器内 `/proc/stat` 仍是宿主 CPU 视图；
+在 LXC 内运行 node_exporter 会生成混合语义并重复采集宿主指标，因此禁止部署。
+
+Sandbox 节点与 workload 指标来自 kubelet/cAdvisor 和 kube-state-metrics；K3s 或 LXC
+特有但上述接口未覆盖的指标，应使用目标明确的 collector，不以 node_exporter 补齐。

clusters/sandbox/apps/ci-runners.yaml

@@ -0,0 +1,18 @@
+---
+apiVersion: kustomize.toolkit.fluxcd.io/v1
+kind: Kustomization
+metadata:
+  name: ci-runners
+  namespace: flux-system
+spec:
+  dependsOn:
+    - name: opensandbox
+    - name: spire-agents
+  interval: 10m
+  path: ./platform/sandbox-ci-runners
+  prune: true
+  sourceRef:
+    kind: GitRepository
+    name: flux-system
+  timeout: 20m
+  wait: true

clusters/sandbox/apps/external-secrets-operator.yaml

@@ -0,0 +1,17 @@
+---
+apiVersion: kustomize.toolkit.fluxcd.io/v1
+kind: Kustomization
+metadata:
+  name: external-secrets-operator
+  namespace: flux-system
+spec:
+  dependsOn:
+    - name: monitoring-operator
+  interval: 10m
+  path: ./platform/sandbox-external-secrets/operator
+  prune: true
+  sourceRef:
+    kind: GitRepository
+    name: flux-system
+  timeout: 10m
+  wait: true

clusters/sandbox/apps/external-secrets.yaml

@@ -0,0 +1,17 @@
+---
+apiVersion: kustomize.toolkit.fluxcd.io/v1
+kind: Kustomization
+metadata:
+  name: external-secrets
+  namespace: flux-system
+spec:
+  dependsOn:
+    - name: external-secrets-operator
+  interval: 10m
+  path: ./platform/sandbox-external-secrets/config
+  prune: true
+  sourceRef:
+    kind: GitRepository
+    name: flux-system
+  timeout: 10m
+  wait: true

clusters/sandbox/apps/kata.yaml

@@ -0,0 +1,18 @@
+---
+apiVersion: kustomize.toolkit.fluxcd.io/v1
+kind: Kustomization
+metadata:
+  name: kata
+  namespace: flux-system
+spec:
+  dependsOn:
+    - name: monitoring-operator
+    - name: spire-agents
+  interval: 10m
+  path: ./platform/sandbox-kata
+  prune: true
+  sourceRef:
+    kind: GitRepository
+    name: flux-system
+  timeout: 35m
+  wait: true

clusters/sandbox/apps/monitoring-operator.yaml

@@ -0,0 +1,15 @@
+---
+apiVersion: kustomize.toolkit.fluxcd.io/v1
+kind: Kustomization
+metadata:
+  name: monitoring-operator
+  namespace: flux-system
+spec:
+  interval: 10m
+  path: ./platform/sandbox-monitoring/operator
+  prune: true
+  sourceRef:
+    kind: GitRepository
+    name: flux-system
+  timeout: 10m
+  wait: true

clusters/sandbox/apps/monitoring.yaml

@@ -0,0 +1,17 @@
+---
+apiVersion: kustomize.toolkit.fluxcd.io/v1
+kind: Kustomization
+metadata:
+  name: monitoring
+  namespace: flux-system
+spec:
+  dependsOn:
+    - name: monitoring-operator
+  interval: 10m
+  path: ./platform/sandbox-monitoring/workloads
+  prune: true
+  sourceRef:
+    kind: GitRepository
+    name: flux-system
+  timeout: 10m
+  wait: true

clusters/sandbox/apps/opensandbox.yaml

@@ -0,0 +1,19 @@
+---
+apiVersion: kustomize.toolkit.fluxcd.io/v1
+kind: Kustomization
+metadata:
+  name: opensandbox
+  namespace: flux-system
+spec:
+  dependsOn:
+    - name: external-secrets
+    - name: kata
+    - name: monitoring-operator
+  interval: 10m
+  path: ./platform/sandbox-opensandbox
+  prune: true
+  sourceRef:
+    kind: GitRepository
+    name: flux-system
+  timeout: 15m
+  wait: true

clusters/sandbox/apps/spire-agents.yaml

@@ -0,0 +1,18 @@
+---
+apiVersion: kustomize.toolkit.fluxcd.io/v1
+kind: Kustomization
+metadata:
+  name: spire-agents
+  namespace: flux-system
+spec:
+  dependsOn:
+    - name: spire-bootstrap
+    - name: monitoring-operator
+  interval: 10m
+  path: ./platform/sandbox-spire/agents
+  prune: true
+  sourceRef:
+    kind: GitRepository
+    name: flux-system
+  timeout: 15m
+  wait: true

clusters/sandbox/apps/spire-bootstrap.yaml

@@ -1,14 +1,15 @@
+---
 apiVersion: kustomize.toolkit.fluxcd.io/v1
 kind: Kustomization
 metadata:
-  name: gitea-actions
+  name: spire-bootstrap
   namespace: flux-system
 spec:
   interval: 10m
-  path: ./platform/gitea-runner
+  path: ./platform/sandbox-spire/bootstrap
-  prune: false
+  prune: true
   sourceRef:
     kind: GitRepository
     name: flux-system
-  timeout: 3m
+  timeout: 10m
-  wait: false
+  wait: true

clusters/sandbox/kustomization.yaml

@@ -0,0 +1,13 @@
+---
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - apps/monitoring-operator.yaml
+  - apps/monitoring.yaml
+  - apps/spire-bootstrap.yaml
+  - apps/spire-agents.yaml
+  - apps/kata.yaml
+  - apps/external-secrets-operator.yaml
+  - apps/external-secrets.yaml
+  - apps/opensandbox.yaml
+  - apps/ci-runners.yaml

infrastructure/dns/records.yml

@@ -15,10 +15,15 @@ homelab_dns:
       - { zone: ad.ddupan.top, name: sandbox-k8s, type: A, values: [10.60.0.13] }
       - { zone: ad.ddupan.top, name: retrolab, type: A, values: [10.60.0.10] }
       - { zone: ad.ddupan.top, name: grafana, type: A, values: [192.168.10.127] }
+      - { zone: ad.ddupan.top, name: metrics-write, type: A, values: [192.168.10.127] }
       - { zone: ad.ddupan.top, name: netbox, type: A, values: [192.168.10.127] }
       - { zone: ad.ddupan.top, name: nats, type: A, values: [192.168.10.127] }
+      - { zone: ad.ddupan.top, name: nexus, type: A, values: [192.168.10.127] }
       - { zone: ad.ddupan.top, name: s3, type: A, values: [192.168.10.127] }
+      - { zone: ad.ddupan.top, name: hydra, type: A, values: [192.168.10.127] }
+      - { zone: ad.ddupan.top, name: hydra-login, type: A, values: [192.168.10.127] }
       - { zone: ad.ddupan.top, name: spire-oidc, type: A, values: [192.168.10.127] }
+      - { zone: ad.ddupan.top, name: spire-server, type: A, values: [192.168.10.127] }
       - { zone: ad.ddupan.top, name: zot, type: A, values: [192.168.10.127] }
       - { zone: ad.ddupan.top, name: zot-push, type: A, values: [192.168.10.127] }
 
@@ -60,14 +65,14 @@ homelab_dns:
       - name: oci-arm.ddupan.top
         type: A
         values:
-        - 129.225.138.179
+          - 129.225.138.179
         proxied: false
         ttl: 300
         terraform: { managed: false }
       - name: oci-amd.ddupan.top
         type: A
         values:
-        - 129.225.176.134
+          - 129.225.176.134
         proxied: false
         ttl: 300
         terraform: { managed: false }

infrastructure/openbao/README.md

@@ -51,7 +51,7 @@ few things Terraform must not own.**
 | secrets engine mounts (`kv`, `pki`, `ssh-client-signer`) | the daemon, Raft, TLS files, systemd |
 | PKI role, issuing/CRL URLs, cluster paths, **ACME** | `bao operator init` / unseal (manual, PGP-wrapped) |
 | SSH signing role (`ai-agent`) | **PKI root CA + SSH CA signing key** |
-| OIDC auth *mount* and *role* | OIDC **client secret** (`auth/oidc/config`) |
+| OIDC/Kubernetes auth mount、config 与 role | OIDC **client secret** (`auth/oidc/config`) |
 | all policies | snapshot token + script + systemd timer |
 |  | host-level CA trust distribution (`openbao_ssh_ca_trust`) |
 
@@ -92,6 +92,13 @@ then `VAULT_ADDR`/`VAULT_TOKEN`), mirroring how `smtp-relay/terraform` uses
 native `openbao/openbao` provider is published only to the OpenTofu registry and
 cannot be resolved by the HashiCorp `terraform` CLI.
 
+Sandbox 集群使用独立的 `auth/kubernetes-sandbox`。其 API 地址、公开 Kubernetes CA、
+ESO role 与只读 `kv/k8s/opensandbox-api` policy 全部由 Terraform 管理；CA 位于
+`terraform/certs/sandbox-kubernetes-ca.crt`。集群重建并轮换 CA 后，先更新该文件并
+apply，再让 Flux 恢复 ESO reconciliation。该 backend 不保存 reviewer JWT，而是使用
+ESO 的短期登录 JWT 执行 TokenReview。该组资源已于 2026-09-18 apply，随后复验 plan
+为 zero-diff。
+
 ## DNS
 
 `bao.ad.ddupan.top` is an **internal-only** name — not a public Cloudflare record and

infrastructure/openbao/terraform/auth.tf

@@ -59,3 +59,33 @@ resource "vault_kubernetes_auth_backend_role" "external_secrets" {
   # in a long TTL and every extra hour is a longer-lived credential in memory.
   token_ttl = 3600
 }
+
+# A Kubernetes auth mount can validate identities from only the API server it is
+# configured against. The sandbox cluster therefore cannot reuse auth/kubernetes,
+# whose TokenReview endpoint belongs to homelab.
+resource "vault_auth_backend" "sandbox_kubernetes" {
+  type = "kubernetes"
+  path = "kubernetes-sandbox"
+}
+
+resource "vault_kubernetes_auth_backend_config" "sandbox" {
+  backend              = vault_auth_backend.sandbox_kubernetes.path
+  kubernetes_host      = "https://10.60.0.13:6443"
+  kubernetes_ca_cert   = file("${path.module}/certs/sandbox-kubernetes-ca.crt")
+  disable_local_ca_jwt = true
+
+  # Deliberately omit token_reviewer_jwt. OpenBao uses the login JWT for
+  # TokenReview; the sandbox external-secrets ServiceAccount is bound only to
+  # system:auth-delegator and all issued JWTs remain short-lived.
+}
+
+resource "vault_kubernetes_auth_backend_role" "sandbox_external_secrets" {
+  backend   = vault_auth_backend.sandbox_kubernetes.path
+  role_name = "external-secrets"
+
+  bound_service_account_names      = ["external-secrets"]
+  bound_service_account_namespaces = ["external-secrets"]
+
+  token_policies = [vault_policy.sandbox_external_secrets.name]
+  token_ttl      = 3600
+}

infrastructure/openbao/terraform/certs/sandbox-kubernetes-ca.crt

@@ -0,0 +1,10 @@
+-----BEGIN CERTIFICATE-----
+MIIBdjCCAR2gAwIBAgIBADAKBggqhkjOPQQDAjAjMSEwHwYDVQQDDBhrM3Mtc2Vy
+dmVyLWNhQDE3ODk2NTI4NTcwHhcNMjYwOTE3MTI0NzM3WhcNMzYwOTE0MTI0NzM3
+WjAjMSEwHwYDVQQDDBhrM3Mtc2VydmVyLWNhQDE3ODk2NTI4NTcwWTATBgcqhkjO
+PQIBBggqhkjOPQMBBwNCAAR4SbqzTXZnlZdUPz7viN6+dYbB1Maw44Qiepn9r5XG
+sOzkYkN8t1aG3Ugo8TqQ3xJaKkM89n1Rluj0vbOhiNajo0IwQDAOBgNVHQ8BAf8E
+BAMCAqQwDwYDVR0TAQH/BAUwAwEB/zAdBgNVHQ4EFgQUyLSGoKAKAJuiniuRdBLG
+XYaDQC8wCgYIKoZIzj0EAwIDRwAwRAIgFkVzyUZexk/ynnxBEOg+3foJv3WKqAei
+hTSRjO1gL0UCIFbBKR7BMrJJAgW3DJFeeBM+b+tTg93jNx55qZACbFOL
+-----END CERTIFICATE-----

infrastructure/openbao/terraform/policies.tf

@@ -35,3 +35,10 @@ resource "vault_policy" "external_secrets" {
   name   = "external-secrets"
   policy = file("${path.module}/policies/external-secrets.hcl")
 }
+
+# The sandbox cluster has its own Kubernetes auth backend and a deliberately
+# narrower KV view than the homelab ESO instance.
+resource "vault_policy" "sandbox_external_secrets" {
+  name   = "sandbox-external-secrets"
+  policy = file("${path.module}/policies/sandbox-external-secrets.hcl")
+}

infrastructure/openbao/terraform/policies/sandbox-external-secrets.hcl

@@ -0,0 +1,9 @@
+# Read only the shared OpenSandbox control-plane API key. The same Bao object is
+# consumed by the server in sandbox and, later, by the scheduler in homelab.
+path "kv/data/k8s/opensandbox-api" {
+  capabilities = ["read"]
+}
+
+path "kv/metadata/k8s/opensandbox-api" {
+  capabilities = ["read"]
+}

infrastructure/proxmox/ansible/roles/vyos_router/defaults/main.yml

@@ -77,3 +77,19 @@ vyos_ospf_networks:
 vyos_postgresql_listen_address: "10.60.0.1"
 vyos_postgresql_port: 5432
 vyos_postgresql_primary_address: "10.60.0.11"
+
+# Stable API endpoint for the two-node sandbox K3s control plane. HAProxy owns
+# this /32 on the existing labnet interface and performs only TCP health checks.
+vyos_sandbox_k3s_api_address: "10.60.0.13"
+vyos_sandbox_k3s_api_interface: eth1
+vyos_sandbox_k3s_api_port: 6443
+vyos_opensandbox_api_port: 8080
+vyos_opensandbox_node_port: 30080
+# Lifecycle create is synchronous and a cold Kata image pull can exceed the
+# HAProxy 50-second default. Keep this below OpenSandbox's overall timeout.
+vyos_opensandbox_api_timeout: 600
+vyos_sandbox_k3s_servers:
+  - name: sandbox1
+    address: "10.60.0.11"
+  - name: sandbox2
+    address: "10.60.0.12"

infrastructure/proxmox/ansible/roles/vyos_router/tasks/main.yml

@@ -3,6 +3,23 @@
 # `set` lines against the running config and issues only the differences, so
 # re-runs are no-ops. Supports --check and --diff.
 
+- name: Read current VyOS configuration commands
+  vyos.vyos.vyos_command:
+    commands:
+      - show configuration commands
+  register: _vyos_running_commands
+  changed_when: false
+
+- name: Remove obsolete HAProxy PostgreSQL protocol check
+  vyos.vyos.vyos_config:
+    lines:
+      - delete load-balancing haproxy backend sandbox-postgresql-primary health-check
+    save: true
+    comment: Remove unsupported PostgreSQL protocol check
+  when: >-
+    "set load-balancing haproxy backend sandbox-postgresql-primary health-check 'pgsql'"
+    in _vyos_running_commands.stdout[0]
+
 - name: Render the desired configuration
   ansible.builtin.set_fact:
     _vyos_lines: >-

infrastructure/proxmox/ansible/roles/vyos_router/templates/vyos.conf.j2

@@ -15,6 +15,20 @@ set interfaces ethernet {{ i.iface }} address {{ i.address }}
 set interfaces ethernet {{ i.iface }} description '{{ i.description }}'
 {% endfor %}
 
+{# OpenSandbox stays on a NodePort; VyOS owns the stable routed frontend. #}
+set load-balancing haproxy service opensandbox-api listen-address {{ vyos_sandbox_k3s_api_address }}
+set load-balancing haproxy service opensandbox-api port {{ vyos_opensandbox_api_port }}
+set load-balancing haproxy service opensandbox-api mode 'tcp'
+set load-balancing haproxy service opensandbox-api backend 'opensandbox-api'
+set load-balancing haproxy service opensandbox-api timeout client {{ vyos_opensandbox_api_timeout }}
+set load-balancing haproxy backend opensandbox-api mode 'tcp'
+set load-balancing haproxy backend opensandbox-api timeout server {{ vyos_opensandbox_api_timeout }}
+{% for server in vyos_sandbox_k3s_servers %}
+set load-balancing haproxy backend opensandbox-api server {{ server.name }} address {{ server.address }}
+set load-balancing haproxy backend opensandbox-api server {{ server.name }} port {{ vyos_opensandbox_node_port }}
+set load-balancing haproxy backend opensandbox-api server {{ server.name }} check
+{% endfor %}
+
 {# --- default route out; OSPF carries the rest --- #}
 set protocols static route 0.0.0.0/0 next-hop {{ vyos_lan_gateway }}
 
@@ -35,11 +49,23 @@ set load-balancing haproxy service sandbox-postgresql port {{ vyos_postgresql_po
 set load-balancing haproxy service sandbox-postgresql mode 'tcp'
 set load-balancing haproxy service sandbox-postgresql backend 'sandbox-postgresql-primary'
 set load-balancing haproxy backend sandbox-postgresql-primary mode 'tcp'
-set load-balancing haproxy backend sandbox-postgresql-primary health-check 'pgsql'
 set load-balancing haproxy backend sandbox-postgresql-primary server primary address {{ vyos_postgresql_primary_address }}
 set load-balancing haproxy backend sandbox-postgresql-primary server primary port {{ vyos_postgresql_port }}
 set load-balancing haproxy backend sandbox-postgresql-primary server primary check
 
+{# --- Sandbox K3s API: stable address with both control-plane nodes --- #}
+set interfaces ethernet {{ vyos_sandbox_k3s_api_interface }} address '{{ vyos_sandbox_k3s_api_address }}/32'
+set load-balancing haproxy service sandbox-k3s-api listen-address {{ vyos_sandbox_k3s_api_address }}
+set load-balancing haproxy service sandbox-k3s-api port {{ vyos_sandbox_k3s_api_port }}
+set load-balancing haproxy service sandbox-k3s-api mode 'tcp'
+set load-balancing haproxy service sandbox-k3s-api backend 'sandbox-k3s-api'
+set load-balancing haproxy backend sandbox-k3s-api mode 'tcp'
+{% for server in vyos_sandbox_k3s_servers %}
+set load-balancing haproxy backend sandbox-k3s-api server {{ server.name }} address {{ server.address }}
+set load-balancing haproxy backend sandbox-k3s-api server {{ server.name }} port {{ vyos_sandbox_k3s_api_port }}
+set load-balancing haproxy backend sandbox-k3s-api server {{ server.name }} check
+{% endfor %}
+
 {# --- management --- #}
 set service ssh port 22
 

infrastructure/sandbox-cluster/README.md

@@ -0,0 +1,152 @@
+# Sandbox 集群
+
+该目录管理供 OpenSandbox、CI 和 AI Agent workload 使用的独立双节点 k3s 集群。
+基础设施和节点生命周期由 Ansible 管理；Kubernetes API 可用后，集群内组件由
+`clusters/sandbox/` 下的 Flux desired state 管理。
+
+## 所有权边界
+
+Ansible 管理以下持久状态：
+
+- pve1/pve2 上的 privileged LXC、磁盘、固定网络与 KVM/vhost/TUN/kmsg 设备；
+- LXC OS 基线、内部 CA、外部 PostgreSQL 与 K3s；
+- 固定版本的 Flux controllers 和 root sync bootstrap；
+- 只读状态验证和数据库切换 runbook。
+
+Flux 管理以下 Kubernetes 资源：
+
+- Kata Containers 和 CI 专用的 `block-plain` RuntimeClass；
+- SPIRE Agent、SPIFFE CSI Driver 与 workload identity 声明；
+- vmagent、kube-state-metrics、kubelet/cAdvisor scrape 配置和告警；
+- sandbox External Secrets Operator、OpenSandbox controller/server；CI Pool 与 runner
+  调度器由 runner 项目接入。
+
+同一个对象只能有一个 owner。Ansible 不直接部署上述集群内 workload；Flux 不管理
+LXC、K3s datastore 或 K3s 本身。
+
+监控范围仅覆盖 sandbox LXC 内的 Kubernetes 与 workload。不得在 LXC 内部署
+node_exporter：LXC 的 `/proc` 是 lxcfs 虚拟视图与宿主内核视图的混合，尤其
+`/proc/stat` 会重复暴露 PVE 宿主 CPU 数据。PVE 自身的 node_exporter 或其他宿主
+监控不属于本目录；sandbox 使用 kubelet/cAdvisor 和 kube-state-metrics。
+
+## 声明拓扑
+
+| 对象 | PVE 节点 | VMID | 地址 | 资源 |
+|---|---:|---:|---|---|
+| `sandbox1` | pve1 | 148 | `10.60.0.11/24` | 4 vCPU / 6 GiB / 2 GiB swap |
+| `sandbox2` | pve2 | 149 | `10.60.0.12/24` | 4 vCPU / 4 GiB / 2 GiB swap |
+| K3s API VIP | VyOS | — | `10.60.0.13:6443` | HAProxy TCP LB |
+
+节点与 API VIP 均位于现有 PVE `labnet`（VLAN 100，`10.60.0.0/24`），通过
+VyOS `10.60.0.1` 路由；不为 sandbox 新建 VNet，也不占用 `192.168.10.0/24`
+地址。需要从 LAN 访问的服务统一经 VyOS 路由或 LB 暴露。两个 LXC 均使用
+`pve-rg-hdd` 上的 32 GiB rootfs；不为 `/var/lib/kubelet` 单独创建 volume。Kata
+`block-plain` 产生的数据随 Pod 生命周期清理，当前规模没有额外磁盘故障域的需求。
+K3s API 的 `10.60.0.13/32` 由 VyOS 现有 labnet interface 持有，HAProxy 以 TCP
+健康检查把 `6443` 分发到两个 control-plane 节点；它与 PostgreSQL 主从切换逻辑无关。
+LXC 内通过 `/etc/tmpfiles.d/kmsg.conf` 持久维护 `/dev/kmsg -> /dev/console`；否则
+kubelet 会因 LXC 不提供真实 host `/dev/kmsg` 而反复退出。
+每个 LXC 还以只读 bind mount 使用宿主的 `/lib/modules`。LXC 与 PVE 宿主共享内核，
+guest 若看不到对应版本的模块目录，K3s 无法加载 `br_netfilter` 和 `overlay`，Flannel
+也不会生成节点的 subnet 配置。
+
+## PostgreSQL 写入口与切换
+
+K3s 使用外部 PostgreSQL，首期采用 primary + synchronous standby。VyOS 在 labnet
+gateway `10.60.0.1:5432` 提供固定 TCP 入口，backend 只包含当前声明的 primary；
+不把普通 TCP 或 PostgreSQL 存活检查等同于“节点可写”，也不自动把流量切到 standby。
+VyOS 2025.11 的 PostgreSQL protocol check 会生成缺少必需 `user` 参数的 HAProxy
+配置，因此这里只使用基础 TCP check；真正的可写性由 `verify.yml` 通过 SQL 验证。
+
+数据库切换必须由 Ansible runbook 受控完成：先隔离旧 primary，再提升 standby，最后
+更新 VyOS backend。首期不部署 PgBouncer、Patroni、repmgr 或额外 DCS，也不宣称两节点
+PostgreSQL 能够自动 HA。以后具备第三个仲裁节点时再重新评估自动 failover。
+
+数据库凭据位于 Bao `kv/infra/sandbox-postgresql`，包含 K3s 登录密码和 physical
+replication 密码。运行 Ansible 前由本机 SPIFFE identity 获取短期 Bao token，再把
+两个值注入 `SANDBOX_K3S_DB_PASSWORD` 与 `SANDBOX_REPLICATION_PASSWORD`；凭据不写入
+inventory、Git 或 Ansible fact cache。首次创建 standby 只允许覆盖不含任何业务库的
+Ubuntu 默认空集群，之后重复执行不会 reseed。
+
+## Ansible
+
+安装固定依赖：
+
+```bash
+cd infrastructure/sandbox-cluster/ansible
+uv venv .venv
+uv pip install --python .venv/bin/python -r requirements.txt
+source .venv/bin/activate
+ansible-galaxy collection install -r requirements.yml
+export PVE_API_PYTHON="$PWD/.venv/bin/python"
+```
+
+部署入口最终为：
+
+```bash
+ansible-playbook site.yml
+```
+
+只安装或 reconcile K3s（LXC、OS baseline 和 PostgreSQL 已就绪时）：
+
+```bash
+ansible-playbook k3s.yml
+```
+
+只 reconcile Flux controllers 与 root sync manifest（不触碰 LXC、PostgreSQL 或
+K3s lifecycle）：
+
+```bash
+ansible-playbook flux.yml
+```
+
+K3s 外部 datastore URI 由运行时 `SANDBOX_K3S_DB_PASSWORD` 生成，密码在 URI 中
+进行 URL 编码，最终仅持久化于节点 root 可读的 `/etc/rancher/k3s/config.yaml`
+（mode `0600`）。首节点生成的 K3s join token 仅在同一次 Ansible run 内传给第二节点；
+不在 inventory 或 Git 中维护副本。首期关闭内建 Traefik 和 ServiceLB。
+
+部署后使用同一组运行时凭据执行只读验收：
+
+```bash
+ansible-playbook verify.yml
+```
+
+当前已经声明 LXC 生命周期、最小 OS baseline、PostgreSQL 和 K3s，包括系统级
+homelab CA trust。Flux `v2.9.5` controllers 与 root sync 也由 Ansible 通过 K3s
+server manifests 管理；root 使用 homelab CA 访问公开 Gitea 仓库，不保存 Git token。
+集群内 workload 由 `clusters/sandbox/` 分阶段纳入 Flux。root Kustomization 的健康检查
+timeout 为 40 分钟，用于覆盖 Kata 等首次安装时会逐节点重启 K3s 的子
+Kustomization；各子项仍保留自己的更短 timeout，故障会在对应子项先行暴露。
+
+## SPIRE 跨集群 bootstrap
+
+Sandbox 复用 homelab 的 SPIRE Server 与 `ddupan.top` trust domain。Flux 首先安装
+SPIRE CRD，并创建供 k8s_psat 使用的 reviewer。它按上游 Server chart 的权限模型调用
+TokenReview，并以 `get/list` 读取用于证明的 Pod 与 Node；不具有修改 workload 的权限。
+在该 Kustomization Ready 后运行：
+
+```bash
+cd infrastructure/sandbox-cluster/ansible
+ansible-playbook spire-bootstrap.yml
+```
+
+Playbook 不把 reviewer token 或生成的 kubeconfig 落盘，而是将目标 Secret manifest
+通过 stdin 交给本机 homelab `k3s kubectl`。目标 Secret
+`spire-server/spire-external-kubeconfigs` 由 Ansible 单独拥有；Flux 和人工操作不得写入。
+第二次运行必须为零变更。
+
+Secret 的 `sandbox` key 仅供 SPIRE Server 的 external PSAT plugin 验证 token 与对应的
+Pod/Node；
+`sandbox-controller` key 供 external controller-manager 读取 Pod/Node、reconcile SPIFFE
+CR 及执行 leader election。两者使用不同 ServiceAccount，不得合并权限或互换。
+
+## 已验证的 Kata CI 前置条件
+
+- Cloud Hypervisor 必须报告 `vm.info.config.memory.shared=true`；
+- CI RuntimeClass 必须使用 `[runtime] emptydir_mode = "block-plain"`；
+- dockerd bootstrap 在 guest 内创建 `/dev/kmsg`：`mknod /dev/kmsg c 1 11`；
+- `/var/lib/docker` 必须是 guest block device 上的 ext4，Docker driver 必须为
+  `overlay2`，不能静默退化到 `vfs` 或 `fuse-overlayfs`；
+- Pod 删除后必须不存在遗留 `disk.img`、VMM 或临时 credential。
+
+PoC 的完整数据和陷阱见 `../kata-lxc-lab/README.md`。

infrastructure/sandbox-cluster/ansible/ansible.cfg

@@ -0,0 +1,11 @@
+[defaults]
+inventory = inventory/hosts.yml
+host_key_checking = True
+interpreter_python = auto_silent
+retry_files_enabled = False
+local_tmp = /tmp/ansible-sandbox-cluster-local
+remote_tmp = /tmp
+roles_path = roles
+
+[ssh_connection]
+pipelining = True

infrastructure/sandbox-cluster/ansible/flux.yml

@@ -0,0 +1,6 @@
+---
+- name: Reconcile Flux controllers and root sync
+  hosts: sandbox1
+  gather_facts: false
+  roles:
+    - sandbox_flux

infrastructure/sandbox-cluster/ansible/inventory/group_vars/all.yml

@@ -0,0 +1,61 @@
+---
+sandbox_pve_api_host: "{{ lookup('env', 'PROXMOX_HOST') }}"
+sandbox_pve_api_user: "{{ lookup('env', 'PROXMOX_USER') }}"
+sandbox_pve_api_token_id: "{{ lookup('env', 'PROXMOX_TOKEN_ID') }}"
+sandbox_pve_api_token_secret: "{{ lookup('env', 'PROXMOX_TOKEN_SECRET') }}"
+sandbox_pve_validate_certs: true
+sandbox_pve_api_python_interpreter: "{{ lookup('env', 'PVE_API_PYTHON') | default(ansible_playbook_python, true) }}"
+
+sandbox_bridge: labnet
+sandbox_gateway: 10.60.0.1
+sandbox_nameserver: 192.168.10.5
+sandbox_search_domain: ad.ddupan.top
+sandbox_lxc_template: laptop:vztmpl/ubuntu-24.04-standard_24.04-2_amd64.tar.zst
+sandbox_rootfs_storage: pve-rg-hdd
+sandbox_rootfs_gb: 32
+sandbox_lxc_cores: 4
+sandbox_lxc_swap_mb: 2048
+sandbox_lxc_pubkey: "{{ lookup('file', '~/.ssh/id_ed25519.pub') }}"
+sandbox_internal_ca_url: https://bao.ad.ddupan.top:8200/v1/pki/ca/pem
+sandbox_base_packages:
+  - ca-certificates
+  - curl
+  - jq
+  - openssh-server
+
+sandbox_postgresql_version: 16
+sandbox_postgresql_primary_address: 10.60.0.11
+sandbox_postgresql_standby_address: 10.60.0.12
+sandbox_postgresql_lb_address: 10.60.0.1
+sandbox_postgresql_database: k3s
+sandbox_postgresql_user: k3s
+sandbox_postgresql_replication_user: k3s_replication
+sandbox_postgresql_replication_slot: sandbox2
+sandbox_postgresql_rotate_passwords: false
+sandbox_postgresql_k3s_password: "{{ lookup('env', 'SANDBOX_K3S_DB_PASSWORD') }}"
+sandbox_postgresql_replication_password: "{{ lookup('env', 'SANDBOX_REPLICATION_PASSWORD') }}"
+
+# This is rendered only at playbook runtime from the Bao-provided password. K3s
+# necessarily persists its datastore credential, but config.yaml is root-only.
+sandbox_k3s_datastore_password_encoded: >-
+  {{ sandbox_postgresql_k3s_password | urlencode | replace('/', '%2F') }}
+sandbox_k3s_datastore_endpoint: >-
+  {{
+    [
+      'postgres://', sandbox_postgresql_user, ':', sandbox_k3s_datastore_password_encoded,
+      '@', sandbox_postgresql_lb_address, ':5432/', sandbox_postgresql_database, '?sslmode=disable'
+    ] | join
+  }}
+server_config_yaml: |
+  datastore-endpoint: {{ sandbox_k3s_datastore_endpoint }}
+  disable:
+    - servicelb
+    - traefik
+user_kubectl: false
+
+sandbox_flux_version: v2.9.5
+sandbox_flux_install_url: >-
+  https://github.com/fluxcd/flux2/releases/download/{{ sandbox_flux_version }}/install.yaml
+sandbox_flux_repository_url: https://git.ddupan.top/panxiao81/homelab-infra.git
+sandbox_flux_repository_branch: main
+sandbox_flux_path: ./clusters/sandbox

infrastructure/sandbox-cluster/ansible/inventory/hosts.yml

@@ -0,0 +1,48 @@
+---
+all:
+  children:
+    sandbox_pve:
+      hosts:
+        pve1:
+          ansible_host: 192.168.10.4
+          sandbox_lxc:
+            vmid: 148
+            hostname: sandbox1
+            address: 10.60.0.11/24
+            memory_mb: 6144
+        pve2:
+          ansible_host: 192.168.10.7
+          sandbox_lxc:
+            vmid: 149
+            hostname: sandbox2
+            address: 10.60.0.12/24
+            memory_mb: 4096
+
+    k3s_cluster:
+      children:
+        server:
+          hosts:
+            sandbox1:
+              ansible_host: 10.60.0.11
+            sandbox2:
+              ansible_host: 10.60.0.12
+        agent:
+          hosts: {}
+      vars:
+        ansible_user: root
+        k3s_version: v1.36.4+k3s1
+        api_endpoint: 10.60.0.13
+        k3s_server_config_mode: "0600"
+        use_external_database: true
+
+    postgres_primary:
+      hosts:
+        sandbox1:
+
+    postgres_standby:
+      hosts:
+        sandbox2:
+
+  vars:
+    ansible_user: root
+    ansible_python_interpreter: /usr/bin/python3

infrastructure/sandbox-cluster/ansible/k3s.yml

@@ -0,0 +1,17 @@
+---
+- name: Validate runtime K3s datastore credential
+  hosts: k3s_cluster
+  gather_facts: false
+  tasks:
+    - name: Require the Bao-provided PostgreSQL password
+      ansible.builtin.assert:
+        that:
+          - sandbox_postgresql_k3s_password | length > 0
+          - server_config_yaml is defined
+          - (server_config_yaml | from_yaml)['datastore-endpoint'] == sandbox_k3s_datastore_endpoint
+        fail_msg: SANDBOX_K3S_DB_PASSWORD must be populated from Bao
+        quiet: true
+      no_log: true
+
+- name: Install the sandbox K3s control plane
+  ansible.builtin.import_playbook: k3s.orchestration.site

infrastructure/sandbox-cluster/ansible/requirements.txt

@@ -0,0 +1,4 @@
+ansible-core==2.21.2
+proxmoxer==2.3.0
+requests==2.34.2
+netaddr==1.3.0

infrastructure/sandbox-cluster/ansible/requirements.yml

@@ -0,0 +1,15 @@
+---
+collections:
+  - name: community.proxmox
+    version: 2.0.0
+  - name: community.postgresql
+    version: 4.2.0
+  - name: ansible.posix
+    version: 2.2.2
+  - name: community.general
+    version: 13.4.0
+  - name: community.library_inventory_filtering_v1
+    version: 1.1.5
+  - name: https://github.com/k3s-io/k3s-ansible.git
+    type: git
+    version: 1a600b60d37e0f8a6e2e79b0e474147b5b108ae5

infrastructure/sandbox-cluster/ansible/roles/sandbox_base/handlers/main.yml

@@ -0,0 +1,5 @@
+---
+- name: Update sandbox CA certificates
+  ansible.builtin.command:
+    cmd: update-ca-certificates
+  changed_when: true

infrastructure/sandbox-cluster/ansible/roles/sandbox_base/tasks/main.yml

@@ -0,0 +1,58 @@
+---
+- name: Install sandbox base packages
+  ansible.builtin.apt:
+    name: "{{ sandbox_base_packages }}"
+    state: present
+    update_cache: true
+    cache_valid_time: 3600
+
+- name: Fetch the homelab internal CA
+  ansible.builtin.uri:
+    url: "{{ sandbox_internal_ca_url }}"
+    return_content: true
+    validate_certs: true
+  register: sandbox_internal_ca
+  changed_when: false
+  retries: 3
+  delay: 5
+  until: sandbox_internal_ca is succeeded
+
+- name: Validate the homelab internal CA response
+  ansible.builtin.assert:
+    that:
+      - "'BEGIN CERTIFICATE' in sandbox_internal_ca.content"
+    fail_msg: OpenBao did not return a PEM certificate
+    quiet: true
+
+- name: Install the homelab internal CA
+  ansible.builtin.copy:
+    dest: /usr/local/share/ca-certificates/ddupan-internal-ca.crt
+    content: "{{ sandbox_internal_ca.content }}"
+    owner: root
+    group: root
+    mode: "0644"
+  notify: Update sandbox CA certificates
+
+- name: Enable the SSH service
+  ansible.builtin.service:
+    name: ssh
+    enabled: true
+    state: started
+
+# LXC does not expose the host kernel log device. Kubelet only needs a writable
+# kmsg-compatible character device, and the container console is the established
+# LXC mapping for this purpose. tmpfiles recreates the link after every boot.
+- name: Persist the LXC kubelet kmsg mapping
+  ansible.builtin.copy:
+    dest: /etc/tmpfiles.d/kmsg.conf
+    content: "L+ /dev/kmsg - - - - /dev/console\n"
+    owner: root
+    group: root
+    mode: "0644"
+
+- name: Expose the LXC console as kmsg
+  ansible.builtin.file:
+    src: /dev/console
+    dest: /dev/kmsg
+    state: link
+    force: true

infrastructure/sandbox-cluster/ansible/roles/sandbox_flux/tasks/main.yml

@@ -0,0 +1,30 @@
+---
+- name: Wait for the K3s server manifest directory
+  ansible.builtin.wait_for:
+    path: /var/lib/rancher/k3s/server/manifests
+    state: present
+    timeout: 120
+
+- name: Download the pinned Flux controller manifest
+  ansible.builtin.get_url:
+    url: "{{ sandbox_flux_install_url }}"
+    dest: /var/lib/rancher/k3s/server/manifests/flux-install.yaml
+    owner: root
+    group: root
+    mode: "0644"
+    timeout: 120
+
+- name: Read the installed homelab CA
+  ansible.builtin.slurp:
+    src: /usr/local/share/ca-certificates/ddupan-internal-ca.crt
+  register: sandbox_flux_ca
+  no_log: true
+
+- name: Install the Flux source and root sync manifest
+  ansible.builtin.template:
+    src: flux-sync.yaml.j2
+    dest: /var/lib/rancher/k3s/server/manifests/flux-sync.yaml
+    owner: root
+    group: root
+    mode: "0600"
+  no_log: true

infrastructure/sandbox-cluster/ansible/roles/sandbox_flux/templates/flux-sync.yaml.j2

@@ -0,0 +1,39 @@
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: flux-system
+  namespace: flux-system
+type: Opaque
+stringData:
+  caFile: |
+{{ sandbox_flux_ca.content | b64decode | indent(4, true) }}
+---
+apiVersion: source.toolkit.fluxcd.io/v1
+kind: GitRepository
+metadata:
+  name: flux-system
+  namespace: flux-system
+spec:
+  interval: 1m
+  ref:
+    branch: {{ sandbox_flux_repository_branch }}
+  secretRef:
+    name: flux-system
+  timeout: 60s
+  url: {{ sandbox_flux_repository_url }}
+---
+apiVersion: kustomize.toolkit.fluxcd.io/v1
+kind: Kustomization
+metadata:
+  name: flux-system
+  namespace: flux-system
+spec:
+  interval: 10m
+  path: {{ sandbox_flux_path }}
+  prune: false
+  sourceRef:
+    kind: GitRepository
+    name: flux-system
+  timeout: 40m
+  wait: true

infrastructure/sandbox-cluster/ansible/roles/sandbox_lxc/tasks/main.yml

@@ -0,0 +1,214 @@
+---
+- name: Validate sandbox declaration and API inputs
+  ansible.builtin.assert:
+    that:
+      - sandbox_lxc.vmid | int > 100
+      - sandbox_lxc.hostname | length > 0
+      - sandbox_lxc.address is match('^10\.60\.0\.[0-9]{1,3}/24$')
+      - sandbox_lxc.memory_mb | int >= 4096
+      - sandbox_pve_api_host | length > 0
+      - sandbox_pve_api_user | length > 0
+      - sandbox_pve_api_token_id | length > 0
+      - sandbox_pve_api_token_secret | length > 0
+    fail_msg: Invalid sandbox declaration or missing Proxmox API environment
+  no_log: true
+
+- name: Check whether the LXC already exists
+  ansible.builtin.stat:
+    path: /etc/pve/lxc/{{ sandbox_lxc.vmid }}.conf
+  register: sandbox_lxc_config
+
+- name: Read current sandbox LXC runtime state
+  community.proxmox.proxmox_vm_info:
+    api_host: "{{ sandbox_pve_api_host }}"
+    api_user: "{{ sandbox_pve_api_user }}"
+    api_token_id: "{{ sandbox_pve_api_token_id }}"
+    api_token_secret: "{{ sandbox_pve_api_token_secret }}"
+    validate_certs: "{{ sandbox_pve_validate_certs }}"
+    ca_path: /etc/ssl/certs/ca-certificates.crt
+    api_timeout: 120
+    vmid: "{{ sandbox_lxc.vmid }}"
+  register: sandbox_lxc_info
+  delegate_to: localhost
+  no_log: true
+  vars:
+    ansible_python_interpreter: "{{ sandbox_pve_api_python_interpreter }}"
+
+- name: Read current sandbox LXC configuration
+  ansible.builtin.uri:
+    url: >-
+      https://{{ sandbox_pve_api_host }}:8006/api2/json/nodes/{{ inventory_hostname }}/lxc/{{ sandbox_lxc.vmid }}/config
+    headers:
+      Authorization: >-
+        PVEAPIToken={{ sandbox_pve_api_user }}!{{ sandbox_pve_api_token_id }}={{ sandbox_pve_api_token_secret }}
+    ca_path: /etc/ssl/certs/ca-certificates.crt
+    return_content: true
+    status_code: [200, 404, 500]
+  register: sandbox_lxc_current_config
+  delegate_to: localhost
+  no_log: true
+
+- name: Reconcile sandbox LXC through the PVE API
+  community.proxmox.proxmox:
+    api_host: "{{ sandbox_pve_api_host }}"
+    api_user: "{{ sandbox_pve_api_user }}"
+    api_token_id: "{{ sandbox_pve_api_token_id }}"
+    api_token_secret: "{{ sandbox_pve_api_token_secret }}"
+    validate_certs: "{{ sandbox_pve_validate_certs }}"
+    ca_path: /etc/ssl/certs/ca-certificates.crt
+    api_timeout: 120
+    node: "{{ inventory_hostname }}"
+    vmid: "{{ sandbox_lxc.vmid }}"
+    hostname: "{{ sandbox_lxc.hostname }}"
+    ostemplate: "{{ sandbox_lxc_template }}"
+    ostype: ubuntu
+    cmode: shell
+    disk_volume:
+      storage: "{{ sandbox_rootfs_storage }}"
+      size: "{{ sandbox_rootfs_gb if not sandbox_lxc_config.stat.exists else omit }}"
+    cores: "{{ sandbox_lxc_cores }}"
+    memory: "{{ sandbox_lxc.memory_mb }}"
+    swap: "{{ sandbox_lxc_swap_mb }}"
+    netif:
+      net0: >-
+        {{
+          'name=eth0,bridge=' ~ sandbox_bridge ~
+          ',ip=' ~ sandbox_lxc.address ~
+          ',gw=' ~ sandbox_gateway ~
+          ',type=veth' ~
+          (',hwaddr=' ~ _sandbox_existing_mac if _sandbox_existing_mac | length > 0 else '')
+        }}
+    nameserver: "{{ sandbox_nameserver }}"
+    searchdomain: "{{ sandbox_search_domain }}"
+    unprivileged: false
+    onboot: true
+    startup: ["order=30", "up=30", "down=60"]
+    tags: [ansible, sandbox, k3s, kata]
+    pubkey: "{{ sandbox_lxc_pubkey }}"
+    update: true
+    state: present
+    timeout: 120
+  register: sandbox_lxc_api
+  when: not sandbox_lxc_config.stat.exists or _sandbox_api_drift
+  delegate_to: localhost
+  no_log: true
+  vars:
+    ansible_python_interpreter: "{{ sandbox_pve_api_python_interpreter }}"
+    _sandbox_existing_mac: >-
+      {{ sandbox_lxc_current_config.json.data.net0.split('hwaddr=')[1].split(',')[0]
+         if 'hwaddr=' in (sandbox_lxc_current_config.json.data.net0 | default(''))
+         else '' }}
+    _sandbox_api_drift: >-
+      {{
+        sandbox_lxc_current_config.status != 200 or
+        sandbox_lxc_current_config.json.data.hostname | default('') != sandbox_lxc.hostname or
+        sandbox_lxc_current_config.json.data.cores | default(0) | int != sandbox_lxc_cores | int or
+        sandbox_lxc_current_config.json.data.memory | default(0) | int != sandbox_lxc.memory_mb | int or
+        sandbox_lxc_current_config.json.data.swap | default(0) | int != sandbox_lxc_swap_mb | int or
+        sandbox_lxc_current_config.json.data.nameserver | default('') != sandbox_nameserver or
+        sandbox_lxc_current_config.json.data.searchdomain | default('') != sandbox_search_domain or
+        sandbox_lxc_current_config.json.data.cmode | default('') != 'shell' or
+        sandbox_lxc_current_config.json.data.onboot | default(0) | int != 1 or
+        sandbox_lxc_current_config.json.data.tags | default('') != 'ansible;k3s;kata;sandbox' or
+        ('bridge=' ~ sandbox_bridge) not in (sandbox_lxc_current_config.json.data.net0 | default('')) or
+        ('ip=' ~ sandbox_lxc.address) not in (sandbox_lxc_current_config.json.data.net0 | default('')) or
+        ('gw=' ~ sandbox_gateway) not in (sandbox_lxc_current_config.json.data.net0 | default('')) or
+        (sandbox_rootfs_storage ~ ':') not in (sandbox_lxc_current_config.json.data.rootfs | default(''))
+      }}
+
+- name: Grow existing sandbox rootfs through the PVE resize API
+  ansible.builtin.uri:
+    url: >-
+      https://{{ sandbox_pve_api_host }}:8006/api2/json/nodes/{{ inventory_hostname }}/lxc/{{ sandbox_lxc.vmid }}/resize
+    method: PUT
+    headers:
+      Authorization: >-
+        PVEAPIToken={{ sandbox_pve_api_user }}!{{ sandbox_pve_api_token_id }}={{ sandbox_pve_api_token_secret }}
+    body_format: form-urlencoded
+    body:
+      disk: rootfs
+      size: "{{ sandbox_rootfs_gb }}G"
+    ca_path: /etc/ssl/certs/ca-certificates.crt
+    status_code: 200
+  when:
+    - sandbox_lxc_config.stat.exists
+    - _sandbox_current_rootfs_gb | int < sandbox_rootfs_gb | int
+  delegate_to: localhost
+  no_log: true
+  vars:
+    _sandbox_current_rootfs_gb: >-
+      {{
+        sandbox_lxc_current_config.json.data.rootfs |
+        default('') |
+        regex_findall('size=([0-9]+)G') |
+        first |
+        default('0')
+      }}
+
+# These properties are not exposed by community.proxmox. Every retained line
+# is required by the already validated nested Kata/kind runtime.
+- name: Reconcile Kata-specific native LXC properties
+  ansible.builtin.lineinfile:
+    path: /etc/pve/lxc/{{ sandbox_lxc.vmid }}.conf
+    regexp: "^{{ item.key | regex_escape }}:"
+    line: "{{ item.key }}: {{ item.value }}"
+  loop:
+    - { key: features, value: "nesting=1,keyctl=1,fuse=1,mknod=1,force_rw_sys=1" }
+    - { key: dev0, value: "path=/dev/kvm,mode=0660" }
+    - { key: dev1, value: "path=/dev/vhost-net,mode=0660" }
+    - { key: dev2, value: "path=/dev/vhost-vsock,mode=0660" }
+    - { key: dev3, value: "path=/dev/net/tun,mode=0666" }
+    - { key: lxc.apparmor.profile, value: unconfined }
+    - { key: lxc.cgroup2.devices.allow, value: a }
+    - { key: lxc.cap.drop, value: "" }
+    - { key: lxc.mount.auto, value: "proc:rw sys:rw" }
+    - { key: lxc.mount.entry, value: "/lib/modules usr/lib/modules none bind,ro,create=dir 0 0" }
+  loop_control:
+    label: "{{ item.key }}"
+  register: sandbox_lxc_native
+
+- name: Start sandbox LXC
+  community.proxmox.proxmox:
+    api_host: "{{ sandbox_pve_api_host }}"
+    api_user: "{{ sandbox_pve_api_user }}"
+    api_token_id: "{{ sandbox_pve_api_token_id }}"
+    api_token_secret: "{{ sandbox_pve_api_token_secret }}"
+    validate_certs: "{{ sandbox_pve_validate_certs }}"
+    ca_path: /etc/ssl/certs/ca-certificates.crt
+    api_timeout: 120
+    vmid: "{{ sandbox_lxc.vmid }}"
+    state: started
+    timeout: 120
+  when: >-
+    not _sandbox_was_running or
+    not (sandbox_lxc_api is changed or sandbox_lxc_native is changed)
+  delegate_to: localhost
+  no_log: true
+  vars:
+    ansible_python_interpreter: "{{ sandbox_pve_api_python_interpreter }}"
+    _sandbox_was_running: >-
+      {{ (sandbox_lxc_info.proxmox_vms | default([]) | first | default({})).status |
+         default('absent') == 'running' }}
+
+- name: Restart existing sandbox LXC after configuration changes
+  community.proxmox.proxmox:
+    api_host: "{{ sandbox_pve_api_host }}"
+    api_user: "{{ sandbox_pve_api_user }}"
+    api_token_id: "{{ sandbox_pve_api_token_id }}"
+    api_token_secret: "{{ sandbox_pve_api_token_secret }}"
+    validate_certs: "{{ sandbox_pve_validate_certs }}"
+    ca_path: /etc/ssl/certs/ca-certificates.crt
+    api_timeout: 120
+    vmid: "{{ sandbox_lxc.vmid }}"
+    state: restarted
+    timeout: 120
+  when:
+    - _sandbox_was_running
+    - sandbox_lxc_api is changed or sandbox_lxc_native is changed
+  delegate_to: localhost
+  no_log: true
+  vars:
+    ansible_python_interpreter: "{{ sandbox_pve_api_python_interpreter }}"
+    _sandbox_was_running: >-
+      {{ (sandbox_lxc_info.proxmox_vms | default([]) | first | default({})).status |
+         default('absent') == 'running' }}

infrastructure/sandbox-cluster/ansible/roles/sandbox_postgresql/handlers/main.yml

@@ -0,0 +1,6 @@
+---
+- name: Restart sandbox PostgreSQL
+  ansible.builtin.service:
+    name: postgresql
+    enabled: true
+    state: restarted

infrastructure/sandbox-cluster/ansible/roles/sandbox_postgresql/tasks/main.yml

@@ -0,0 +1,50 @@
+---
+- name: Validate sandbox PostgreSQL inputs
+  ansible.builtin.assert:
+    that:
+      - sandbox_postgresql_k3s_password | length >= 32
+      - sandbox_postgresql_replication_password | length >= 32
+      - inventory_hostname in groups['postgres_primary'] or inventory_hostname in groups['postgres_standby']
+    fail_msg: Missing Bao-provided PostgreSQL credentials or invalid inventory role
+  no_log: true
+
+- name: Install PostgreSQL packages
+  ansible.builtin.apt:
+    name:
+      - postgresql-{{ sandbox_postgresql_version }}
+      - postgresql-client-{{ sandbox_postgresql_version }}
+      - python3-psycopg2
+    state: present
+    update_cache: true
+    cache_valid_time: 3600
+
+- name: Check whether synchronous replication has completed bootstrap
+  ansible.builtin.stat:
+    path: /etc/postgresql/{{ sandbox_postgresql_version }}/main/sandbox-synchronous-ready
+  register: sandbox_postgresql_synchronous_marker
+
+- name: Install sandbox PostgreSQL access policy
+  ansible.builtin.template:
+    src: pg_hba.conf.j2
+    dest: /etc/postgresql/{{ sandbox_postgresql_version }}/main/pg_hba.conf
+    owner: postgres
+    group: postgres
+    mode: "0640"
+  notify: Restart sandbox PostgreSQL
+
+- name: Install sandbox PostgreSQL server settings
+  ansible.builtin.template:
+    src: sandbox.conf.j2
+    dest: /etc/postgresql/{{ sandbox_postgresql_version }}/main/conf.d/99-sandbox.conf
+    owner: postgres
+    group: postgres
+    mode: "0644"
+  notify: Restart sandbox PostgreSQL
+
+- name: Configure sandbox PostgreSQL primary
+  ansible.builtin.include_tasks: primary.yml
+  when: inventory_hostname in groups['postgres_primary']
+
+- name: Configure sandbox PostgreSQL standby
+  ansible.builtin.include_tasks: standby.yml
+  when: inventory_hostname in groups['postgres_standby']

infrastructure/sandbox-cluster/ansible/roles/sandbox_postgresql/tasks/primary.yml

@@ -0,0 +1,45 @@
+---
+- name: Apply primary PostgreSQL configuration before replication setup
+  ansible.builtin.meta: flush_handlers
+
+- name: Create K3s PostgreSQL login role
+  community.postgresql.postgresql_user:
+    name: "{{ sandbox_postgresql_user }}"
+    password: "{{ sandbox_postgresql_k3s_password }}"
+    no_password_changes: "{{ not sandbox_postgresql_rotate_passwords }}"
+    role_attr_flags: LOGIN,NOSUPERUSER,NOCREATEDB,NOCREATEROLE,NOREPLICATION
+    state: present
+  become: true
+  become_user: postgres
+  no_log: true
+  environment:
+    PGOPTIONS: -c password_encryption=scram-sha-256
+
+- name: Create K3s PostgreSQL database
+  community.postgresql.postgresql_db:
+    name: "{{ sandbox_postgresql_database }}"
+    owner: "{{ sandbox_postgresql_user }}"
+    state: present
+  become: true
+  become_user: postgres
+
+- name: Create PostgreSQL replication login role
+  community.postgresql.postgresql_user:
+    name: "{{ sandbox_postgresql_replication_user }}"
+    password: "{{ sandbox_postgresql_replication_password }}"
+    no_password_changes: "{{ not sandbox_postgresql_rotate_passwords }}"
+    role_attr_flags: LOGIN,REPLICATION,NOSUPERUSER,NOCREATEDB,NOCREATEROLE
+    state: present
+  become: true
+  become_user: postgres
+  no_log: true
+  environment:
+    PGOPTIONS: -c password_encryption=scram-sha-256
+
+- name: Create physical replication slot for sandbox2
+  community.postgresql.postgresql_slot:
+    name: "{{ sandbox_postgresql_replication_slot }}"
+    slot_type: physical
+    state: present
+  become: true
+  become_user: postgres

infrastructure/sandbox-cluster/ansible/roles/sandbox_postgresql/tasks/standby.yml

@@ -0,0 +1,114 @@
+---
+- name: Detect whether sandbox2 is already a standby
+  community.postgresql.postgresql_query:
+    login_db: postgres
+    query: SELECT pg_is_in_recovery() AS in_recovery
+  become: true
+  become_user: postgres
+  register: sandbox_postgresql_recovery_state
+
+- name: Inspect databases before the initial standby seed
+  community.postgresql.postgresql_query:
+    login_db: postgres
+    query: >-
+      SELECT datname FROM pg_database
+      WHERE NOT datistemplate AND datname <> 'postgres'
+  become: true
+  become_user: postgres
+  register: sandbox_postgresql_existing_databases
+  when: not sandbox_postgresql_recovery_state.query_result[0].in_recovery
+
+- name: Refuse to overwrite a non-empty PostgreSQL node
+  ansible.builtin.assert:
+    that:
+      - sandbox_postgresql_existing_databases.query_result | length == 0
+    fail_msg: Refusing to reseed sandbox2 because it contains non-system databases
+  when: not sandbox_postgresql_recovery_state.query_result[0].in_recovery
+
+- name: Stop PostgreSQL before the initial standby seed
+  ansible.builtin.service:
+    name: postgresql
+    state: stopped
+  when: not sandbox_postgresql_recovery_state.query_result[0].in_recovery
+
+- name: Remove the verified-empty standby data directory
+  ansible.builtin.file:
+    path: /var/lib/postgresql/{{ sandbox_postgresql_version }}/main
+    state: absent
+  when: not sandbox_postgresql_recovery_state.query_result[0].in_recovery
+
+- name: Recreate the standby data directory
+  ansible.builtin.file:
+    path: /var/lib/postgresql/{{ sandbox_postgresql_version }}/main
+    state: directory
+    owner: postgres
+    group: postgres
+    mode: "0700"
+  when: not sandbox_postgresql_recovery_state.query_result[0].in_recovery
+
+- name: Install standby replication password file
+  ansible.builtin.copy:
+    dest: /var/lib/postgresql/.pgpass
+    content: >-
+      {{ sandbox_postgresql_primary_address }}:5432:*:{{ sandbox_postgresql_replication_user }}:{{ sandbox_postgresql_replication_password }}
+    owner: postgres
+    group: postgres
+    mode: "0600"
+  no_log: true
+
+- name: Seed sandbox2 from the PostgreSQL primary
+  ansible.builtin.command:
+    argv:
+      - /usr/bin/pg_basebackup
+      - --host={{ sandbox_postgresql_primary_address }}
+      - --username={{ sandbox_postgresql_replication_user }}
+      - --pgdata=/var/lib/postgresql/{{ sandbox_postgresql_version }}/main
+      - --format=plain
+      - --wal-method=stream
+      - --write-recovery-conf
+      - --slot={{ sandbox_postgresql_replication_slot }}
+  become: true
+  become_user: postgres
+  environment:
+    PGPASSFILE: /var/lib/postgresql/.pgpass
+  when: not sandbox_postgresql_recovery_state.query_result[0].in_recovery
+  no_log: true
+
+- name: Set the standby connection identity
+  ansible.builtin.lineinfile:
+    path: /var/lib/postgresql/{{ sandbox_postgresql_version }}/main/postgresql.auto.conf
+    regexp: ^primary_conninfo =
+    line: >-
+      primary_conninfo = 'host={{ sandbox_postgresql_primary_address }} port=5432
+      user={{ sandbox_postgresql_replication_user }} application_name=sandbox2
+      passfile=/var/lib/postgresql/.pgpass'
+    owner: postgres
+    group: postgres
+    mode: "0600"
+  no_log: true
+
+- name: Start PostgreSQL standby
+  ansible.builtin.service:
+    name: postgresql
+    enabled: true
+    state: started
+
+- name: Wait for sandbox2 to enter recovery
+  community.postgresql.postgresql_query:
+    login_db: postgres
+    query: SELECT pg_is_in_recovery() AS in_recovery
+  become: true
+  become_user: postgres
+  register: sandbox_postgresql_standby_ready
+  retries: 12
+  delay: 5
+  until: sandbox_postgresql_standby_ready.query_result[0].in_recovery
+
+- name: Mark synchronous replication bootstrap complete on the primary
+  ansible.builtin.copy:
+    dest: /etc/postgresql/{{ sandbox_postgresql_version }}/main/sandbox-synchronous-ready
+    content: "sandbox2\n"
+    owner: postgres
+    group: postgres
+    mode: "0644"
+  delegate_to: "{{ groups['postgres_primary'][0] }}"

infrastructure/sandbox-cluster/ansible/roles/sandbox_postgresql/templates/pg_hba.conf.j2

@@ -0,0 +1,5 @@
+# Ansible managed
+local   all             postgres                                peer
+local   all             all                                     peer
+host    {{ sandbox_postgresql_database }} {{ sandbox_postgresql_user }} 10.60.0.0/24 scram-sha-256
+host    replication     {{ sandbox_postgresql_replication_user }} 10.60.0.0/24 scram-sha-256

infrastructure/sandbox-cluster/ansible/roles/sandbox_postgresql/templates/sandbox.conf.j2

@@ -0,0 +1,11 @@
+# Ansible managed
+listen_addresses = '{{ ansible_host }},127.0.0.1'
+password_encryption = 'scram-sha-256'
+wal_level = 'replica'
+max_wal_senders = 10
+max_replication_slots = 10
+hot_standby = on
+{% if inventory_hostname in groups['postgres_primary'] and sandbox_postgresql_synchronous_marker.stat.exists %}
+synchronous_standby_names = 'FIRST 1 (sandbox2)'
+synchronous_commit = 'remote_apply'
+{% endif %}

infrastructure/sandbox-cluster/ansible/roles/sandbox_spire_bootstrap/defaults/main.yml

@@ -0,0 +1,7 @@
+---
+sandbox_spire_bootstrap_api_server: https://10.60.0.13:6443
+sandbox_spire_bootstrap_source_namespace: spire-system
+sandbox_spire_bootstrap_source_secret: spire-server-token-reviewer-token
+sandbox_spire_bootstrap_controller_secret: spire-controller-manager-token
+sandbox_spire_bootstrap_target_namespace: spire-server
+sandbox_spire_bootstrap_target_secret: spire-external-kubeconfigs

infrastructure/sandbox-cluster/ansible/roles/sandbox_spire_bootstrap/tasks/main.yml

@@ -0,0 +1,115 @@
+---
+- name: Wait for the sandbox SPIRE token reviewer credential
+  ansible.builtin.command:
+    argv:
+      - k3s
+      - kubectl
+      - --namespace
+      - "{{ sandbox_spire_bootstrap_source_namespace }}"
+      - get
+      - secret
+      - "{{ sandbox_spire_bootstrap_source_secret }}"
+      - --output=json
+  register: sandbox_spire_bootstrap_reviewer_secret
+  changed_when: false
+  retries: 60
+  delay: 10
+  until:
+    - sandbox_spire_bootstrap_reviewer_secret.rc == 0
+    - (sandbox_spire_bootstrap_reviewer_secret.stdout | from_json).data.token is defined
+    - (sandbox_spire_bootstrap_reviewer_secret.stdout | from_json).data['ca.crt'] is defined
+  no_log: true
+
+- name: Wait for the sandbox SPIRE controller credential
+  ansible.builtin.command:
+    argv:
+      - k3s
+      - kubectl
+      - --namespace
+      - "{{ sandbox_spire_bootstrap_source_namespace }}"
+      - get
+      - secret
+      - "{{ sandbox_spire_bootstrap_controller_secret }}"
+      - --output=json
+  register: sandbox_spire_bootstrap_controller_secret_result
+  changed_when: false
+  retries: 60
+  delay: 10
+  until:
+    - sandbox_spire_bootstrap_controller_secret_result.rc == 0
+    - (sandbox_spire_bootstrap_controller_secret_result.stdout | from_json).data.token is defined
+    - (sandbox_spire_bootstrap_controller_secret_result.stdout | from_json).data['ca.crt'] is defined
+  no_log: true
+
+- name: Extract the sandbox TokenReview credential data
+  ansible.builtin.set_fact:
+    sandbox_spire_bootstrap_secret_data: >-
+      {{ (sandbox_spire_bootstrap_reviewer_secret.stdout | from_json).data }}
+    sandbox_spire_bootstrap_controller_data: >-
+      {{ (sandbox_spire_bootstrap_controller_secret_result.stdout | from_json).data }}
+  no_log: true
+
+- name: Build the restricted sandbox TokenReview kubeconfig
+  ansible.builtin.set_fact:
+    sandbox_spire_bootstrap_kubeconfig: |
+      apiVersion: v1
+      kind: Config
+      clusters:
+        - name: sandbox
+          cluster:
+            server: {{ sandbox_spire_bootstrap_api_server }}
+            certificate-authority-data: {{ sandbox_spire_bootstrap_secret_data['ca.crt'] }}
+      users:
+        - name: spire-server-token-reviewer
+          user:
+            token: {{ sandbox_spire_bootstrap_secret_data.token | b64decode }}
+      contexts:
+        - name: sandbox
+          context:
+            cluster: sandbox
+            user: spire-server-token-reviewer
+      current-context: sandbox
+    sandbox_spire_bootstrap_controller_kubeconfig: |
+      apiVersion: v1
+      kind: Config
+      clusters:
+        - name: sandbox
+          cluster:
+            server: {{ sandbox_spire_bootstrap_api_server }}
+            certificate-authority-data: {{ sandbox_spire_bootstrap_controller_data['ca.crt'] }}
+      users:
+        - name: spire-controller-manager
+          user:
+            token: {{ sandbox_spire_bootstrap_controller_data.token | b64decode }}
+      contexts:
+        - name: sandbox
+          context:
+            cluster: sandbox
+            user: spire-controller-manager
+      current-context: sandbox
+  no_log: true
+
+- name: Reconcile the central SPIRE external kubeconfig Secret
+  ansible.builtin.command:
+    argv:
+      - k3s
+      - kubectl
+      - apply
+      - --filename=-
+    stdin: |
+      apiVersion: v1
+      kind: Secret
+      metadata:
+        name: {{ sandbox_spire_bootstrap_target_secret }}
+        namespace: {{ sandbox_spire_bootstrap_target_namespace }}
+      type: Opaque
+      data:
+        sandbox: {{ sandbox_spire_bootstrap_kubeconfig | b64encode }}
+        sandbox-controller: {{ sandbox_spire_bootstrap_controller_kubeconfig | b64encode }}
+  delegate_to: localhost
+  become: true
+  register: sandbox_spire_bootstrap_target
+  changed_when: >-
+    ' created' in sandbox_spire_bootstrap_target.stdout or
+    ' configured' in sandbox_spire_bootstrap_target.stdout
+  no_log: true

infrastructure/sandbox-cluster/ansible/site.yml

@@ -0,0 +1,60 @@
+---
+- name: Reconcile sandbox LXC nodes
+  hosts: sandbox_pve
+  gather_facts: false
+  serial: 1
+  roles:
+    - sandbox_lxc
+
+- name: Reconcile sandbox node OS baseline
+  hosts: k3s_cluster
+  gather_facts: false
+  roles:
+    - sandbox_base
+
+- name: Reconcile sandbox PostgreSQL primary
+  hosts: postgres_primary
+  gather_facts: false
+  roles:
+    - sandbox_postgresql
+
+- name: Reconcile sandbox PostgreSQL synchronous standby
+  hosts: postgres_standby
+  gather_facts: false
+  roles:
+    - sandbox_postgresql
+
+- name: Finalize synchronous PostgreSQL settings
+  hosts: postgres_primary
+  gather_facts: false
+  roles:
+    - sandbox_postgresql
+
+- name: Validate runtime K3s datastore credential
+  hosts: k3s_cluster
+  gather_facts: false
+  tasks:
+    - name: Require the Bao-provided PostgreSQL password
+      ansible.builtin.assert:
+        that:
+          - sandbox_postgresql_k3s_password | length > 0
+          - server_config_yaml is defined
+          - (server_config_yaml | from_yaml)['datastore-endpoint'] == sandbox_k3s_datastore_endpoint
+        fail_msg: SANDBOX_K3S_DB_PASSWORD must be populated from Bao
+        quiet: true
+      no_log: true
+
+- name: Install the sandbox K3s control plane
+  ansible.builtin.import_playbook: k3s.orchestration.site
+
+- name: Reconcile Flux controllers and root sync
+  hosts: sandbox1
+  gather_facts: false
+  roles:
+    - sandbox_flux
+
+- name: Reconcile central SPIRE access to sandbox Kubernetes
+  hosts: sandbox1
+  gather_facts: false
+  roles:
+    - sandbox_spire_bootstrap

infrastructure/sandbox-cluster/ansible/spire-bootstrap.yml

@@ -0,0 +1,6 @@
+---
+- name: Reconcile central SPIRE access to sandbox Kubernetes
+  hosts: sandbox1
+  gather_facts: false
+  roles:
+    - sandbox_spire_bootstrap