启用 Backstage 显式目录位置 (#153)

Co-authored-by: panxiao81 <[email protected]>

.gitea/workflows/ansible.yml

@@ -16,12 +16,9 @@ on:
       - '.ansible-lint'
       - '.gitea/workflows/ansible.yml'
 
-env:
-  ANSIBLE_COLLECTIONS_PATH: /root/.ansible/collections
-
 jobs:
   lint:
-    runs-on: self-hosted
+    runs-on: [self-hosted, pod]
     steps:
       - uses: actions/checkout@v4
 
@@ -30,6 +27,7 @@ jobs:
           python3 -m pip install --user --break-system-packages \
             --index-url https://pypi.org/simple --quiet uv==0.11.7
           echo "$HOME/.local/bin" >> "$GITHUB_PATH"
+          echo "ANSIBLE_COLLECTIONS_PATH=$HOME/.ansible/collections" >> "$GITHUB_ENV"
 
       - name: Install ansible-lint and collections
         run: |
@@ -50,7 +48,17 @@ jobs:
       - name: ansible-lint
         run: |
           export PATH="$HOME/.local/bin:$PATH"
-          export ANSIBLE_COLLECTIONS_PATH="$PWD/infrastructure/samba-ad/ansible/collections:/root/.ansible/collections"
+          export ANSIBLE_COLLECTIONS_PATH="$PWD/infrastructure/samba-ad/ansible/collections:$HOME/.ansible/collections"
+          # 静态检查不应依赖生产 vault 凭据。一次性 checkout 可以去掉加密变量文件；
+          # syntax-check 只验证结构，不需要解析变量的运行时值。
+          rm -f \
+            infrastructure/openbao/ansible/group_vars/all/vault.yml \
+            infrastructure/samba-ad/ansible/group_vars/all/vault.yml
+          # ansible.cfg still declares vault_password_file. Even with encrypted
+          # vars removed, ansible-lint validates that the configured file exists
+          # before syntax-check starts. This throwaway value decrypts nothing.
+          export ANSIBLE_VAULT_PASSWORD_FILE="$RUNNER_TEMP/ansible-lint-vault-pass"
+          printf '%s\n' 'ci-placeholder-not-a-production-secret' > "$ANSIBLE_VAULT_PASSWORD_FILE"
           rc=0
           for p in infrastructure/openbao infrastructure/samba-ad infrastructure/proxmox; do
             echo "::group::$p"
@@ -60,7 +68,7 @@ jobs:
           exit $rc
 
   collection-test:
-    runs-on: self-hosted
+    runs-on: [self-hosted, pod]
     steps:
       - uses: actions/checkout@v4
 

.gitea/workflows/hydra.yml

@@ -0,0 +1,22 @@
+name: hydra-login
+on:
+  pull_request:
+    paths:
+      - 'apps/hydra/login-consent/**'
+      - '.gitea/workflows/hydra.yml'
+  workflow_dispatch:
+jobs:
+  verify:
+    runs-on: [self-hosted, pod]
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-go@v5
+        with:
+          go-version-file: apps/hydra/login-consent/go.mod
+          cache-dependency-path: apps/hydra/login-consent/go.sum
+      - name: Test authentication boundaries
+        working-directory: apps/hydra/login-consent
+        run: |
+          go test -race ./...
+          go vet ./...
+          CGO_ENABLED=0 go build -trimpath .

.gitea/workflows/kind-on-kata-smoke.yml

@@ -1,63 +0,0 @@
----
-name: kind-on-kata-smoke
-
-on:
-  push:
-    branches:
-      - poc/kind-on-kata
-    paths:
-      - .gitea/workflows/kind-on-kata-smoke.yml
-  workflow_dispatch:
-
-jobs:
-  smoke:
-    runs-on: kata-poc
-    steps:
-      - name: Create nested kind cluster
-        shell: sh
-        env:
-          KIND_VERSION: v0.27.0
-          KIND_NODE_IMAGE: kindest/node:v1.32.2@sha256:f226345927d7e348497136874b6d207e0b32cc52154ad8323129352923a3142f
-        run: |
-          set -eu
-          apk add --no-cache ca-certificates curl docker-cli
-          curl --retry 5 --retry-all-errors --connect-timeout 15 -fsSLo /tmp/kind \
-            "https://kind.sigs.k8s.io/dl/${KIND_VERSION}/kind-linux-amd64"
-          curl --retry 5 --retry-all-errors --connect-timeout 15 -fsSLo /tmp/kind.sha256sum \
-            "https://kind.sigs.k8s.io/dl/${KIND_VERSION}/kind-linux-amd64.sha256sum"
-          expected="$(awk '{print $1}' /tmp/kind.sha256sum)"
-          printf '%s  %s\n' "$expected" /tmp/kind | sha256sum -c -
-          install -m 0755 /tmp/kind /usr/local/bin/kind
-          docker info --format 'kernel={{.KernelVersion}} driver={{.Driver}}'
-          test "$(docker info --format '{{.Driver}}')" = overlay2
-
-          cleanup() {
-            kind delete cluster --name nested >/dev/null 2>&1 || true
-          }
-          trap cleanup EXIT
-
-          cat >/tmp/kind-config.yaml <<'EOF'
-          kind: Cluster
-          apiVersion: kind.x-k8s.io/v1alpha4
-          name: nested
-          nodes:
-            - role: control-plane
-              extraMounts:
-                - hostPath: /dev/kmsg
-                  containerPath: /dev/kmsg
-          EOF
-          kind create cluster -v 9 --retain --config /tmp/kind-config.yaml --image "$KIND_NODE_IMAGE" --wait 5m
-          docker exec nested-control-plane kubectl \
-            --kubeconfig=/etc/kubernetes/admin.conf wait \
-            --for=condition=Ready node/nested-control-plane --timeout=2m
-          docker exec nested-control-plane kubectl \
-            --kubeconfig=/etc/kubernetes/admin.conf run smoke \
-            --image=docker.io/library/busybox:1.37 --restart=Never \
-            --command -- sh -c 'echo kind-on-kata-ok'
-          docker exec nested-control-plane kubectl \
-            --kubeconfig=/etc/kubernetes/admin.conf wait \
-            --for=jsonpath='{.status.phase}'=Succeeded pod/smoke --timeout=2m
-          test "$(docker exec nested-control-plane kubectl \
-            --kubeconfig=/etc/kubernetes/admin.conf logs smoke)" = kind-on-kata-ok
-          kind delete cluster --name nested
-          trap - EXIT

.gitea/workflows/lint.yml

@@ -13,18 +13,22 @@ on:
     paths:
       - '**/*.yaml'
       - '**/*.yml'
+      - 'infrastructure/dns/**'
+      - 'infrastructure/cloudflared/terraform/dns.generated.tf'
       - '.yamllint.yml'
       - '.gitea/workflows/lint.yml'
   pull_request:
     paths:
       - '**/*.yaml'
       - '**/*.yml'
+      - 'infrastructure/dns/**'
+      - 'infrastructure/cloudflared/terraform/dns.generated.tf'
       - '.yamllint.yml'
       - '.gitea/workflows/lint.yml'
 
 jobs:
   yaml:
-    runs-on: self-hosted
+    runs-on: [self-hosted, pod]
     steps:
       - uses: actions/checkout@v4
 
@@ -52,3 +56,8 @@ jobs:
           export PATH="$HOME/.local/bin:$PATH"
           files=$(git ls-files '*.yaml' '*.yml' | grep -vE '^apps/netboot/')
           yamllint -c .yamllint.yml --no-warnings -f parsable $files
+
+      - name: Verify generated DNS configuration
+        run: |
+          export PATH="$HOME/.local/bin:$PATH"
+          uv run infrastructure/dns/generate.py --check

.gitea/workflows/terraform.yml

@@ -16,7 +16,7 @@ on:
 
 jobs:
   validate:
-    runs-on: self-hosted
+    runs-on: [self-hosted, pod]
     steps:
       - uses: actions/checkout@v4
       - uses: hashicorp/setup-terraform@v3

.gitignore

@@ -49,6 +49,7 @@ authelia/secret.yaml
 **/secret.yaml
 **/credentials.yml
 **/terraform.tfvars
+**/credentials.auto.tfvars
 **/tailscale/helm.sh
 **/cloudflared/backup/
 **/cloudflared/secret.yaml
@@ -118,3 +119,4 @@ apps/netboot/config/log/
 # Blocky's per-day query logs. Bind-mounted into the container, one file per
 # day, and every DNS query the LAN makes ends up in them.
 apps/blocky/logs/
+.venv/

apps/authelia/values.yaml

@@ -229,7 +229,7 @@ configMap:
           require_pkce: false
           token_endpoint_auth_method: 'client_secret_basic'
           redirect_uris:
-            - 'https://grafana.tail7e769.ts.net/login/generic_oauth'
+            - 'https://grafana.ad.ddupan.top/login/generic_oauth'
           scopes:
             - 'openid'
             - 'profile'

apps/backstage/README.md

@@ -0,0 +1,20 @@
+# Backstage
+
+Backstage 作为 homelab 的只读开发者门户运行。应用源码、插件、测试和镜像构建归
+`panxiao81/backstage` 仓库管理；本目录只保存 Kubernetes/Flux 部署声明，并通过
+OCI digest 固定镜像。
+
+## 外部前置
+
+- OpenBao `kv/k8s/backstage` 必须包含以下与容器环境变量同名的字段：
+  `BACKEND_SECRET`、`AUTH_SESSION_SECRET`、`AUTH_OIDC_CLIENT_ID`、
+  `AUTH_OIDC_CLIENT_SECRET`、`POSTGRES_PASSWORD`、`GITEA_TOKEN`。
+- PostgreSQL 需要在共享集群中预先创建由 `backstage` 角色拥有的 `backstage`
+  数据库；密码必须与 OpenBao 中的 `POSTGRES_PASSWORD` 一致。
+- Authelia OIDC 客户端回调地址为
+  `https://backstage.ad.ddupan.top/api/auth/oidc/handler/frame`。
+- AD DNS 需要将 `backstage.ad.ddupan.top` 指向 Envoy Gateway
+  `192.168.10.127`。
+
+Flux 等待 ExternalSecret 和 Deployment 就绪；任何前置缺失都会使该
+Kustomization 保持 NotReady，而不会回退到明文 Secret。

apps/backstage/deployment.yaml

@@ -0,0 +1,83 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: backstage
+  namespace: backstage
+  labels:
+    app.kubernetes.io/name: backstage
+spec:
+  replicas: 1
+  strategy:
+    type: Recreate
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: backstage
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: backstage
+    spec:
+      serviceAccountName: backstage
+      securityContext:
+        fsGroup: 1000
+        runAsNonRoot: true
+        seccompProfile:
+          type: RuntimeDefault
+      containers:
+        - name: backstage
+          image: zot.ad.ddupan.top/panxiao81/backstage@sha256:493aed471c58b21db7a108a333417498fc03682d119282b89cd0318563d6cae6
+          imagePullPolicy: IfNotPresent
+          env:
+            - name: BACKSTAGE_BASE_URL
+              value: https://backstage.ad.ddupan.top
+            - name: POSTGRES_HOST
+              value: shared-postgresql-rw.shared-db.svc.cluster.local
+            - name: POSTGRES_PORT
+              value: "5432"
+            - name: POSTGRES_USER
+              value: backstage
+            - name: POSTGRES_DATABASE
+              value: backstage
+            - name: GITEA_HOST
+              value: git.ddupan.top
+          envFrom:
+            - secretRef:
+                name: backstage
+          ports:
+            - containerPort: 7007
+              name: http
+              protocol: TCP
+          readinessProbe:
+            httpGet:
+              path: /.backstage/health/v1/readiness
+              port: http
+            initialDelaySeconds: 10
+            periodSeconds: 10
+            timeoutSeconds: 3
+          livenessProbe:
+            httpGet:
+              path: /.backstage/health/v1/liveness
+              port: http
+            initialDelaySeconds: 30
+            periodSeconds: 20
+            timeoutSeconds: 3
+          resources:
+            requests:
+              cpu: 100m
+              memory: 512Mi
+            limits:
+              cpu: "1"
+              memory: 1Gi
+          securityContext:
+            allowPrivilegeEscalation: false
+            capabilities:
+              drop: [ALL]
+            readOnlyRootFilesystem: true
+            runAsNonRoot: true
+            runAsUser: 1000
+          volumeMounts:
+            - mountPath: /tmp
+              name: tmp
+      volumes:
+        - emptyDir: {}
+          name: tmp

apps/backstage/external-secret.yaml

@@ -0,0 +1,16 @@
+apiVersion: external-secrets.io/v1
+kind: ExternalSecret
+metadata:
+  name: backstage
+  namespace: backstage
+spec:
+  refreshInterval: 1h
+  secretStoreRef:
+    kind: ClusterSecretStore
+    name: openbao
+  target:
+    creationPolicy: Owner
+    name: backstage
+  dataFrom:
+    - extract:
+        key: k8s/backstage

apps/backstage/httproute.yaml

@@ -0,0 +1,20 @@
+apiVersion: gateway.networking.k8s.io/v1
+kind: HTTPRoute
+metadata:
+  name: backstage
+  namespace: backstage
+spec:
+  parentRefs:
+    - name: eg
+      namespace: envoy-gateway-system
+      sectionName: https
+  hostnames:
+    - backstage.ad.ddupan.top
+  rules:
+    - matches:
+        - path:
+            type: PathPrefix
+            value: /
+      backendRefs:
+        - name: backstage
+          port: 7007

apps/backstage/kustomization.yaml

@@ -0,0 +1,10 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - namespace.yaml
+  - serviceaccount.yaml
+  - external-secret.yaml
+  - deployment.yaml
+  - service.yaml
+  - networkpolicy.yaml
+  - httproute.yaml

apps/backstage/namespace.yaml

@@ -0,0 +1,7 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: backstage
+  labels:
+    pod-security.kubernetes.io/enforce: restricted
+    pod-security.kubernetes.io/enforce-version: v1.36

apps/backstage/networkpolicy.yaml

@@ -0,0 +1,24 @@
+apiVersion: networking.k8s.io/v1
+kind: NetworkPolicy
+metadata:
+  name: backstage-ingress
+  namespace: backstage
+spec:
+  podSelector:
+    matchLabels:
+      app.kubernetes.io/name: backstage
+  policyTypes: [Ingress]
+  ingress:
+    - from:
+        - namespaceSelector:
+            matchLabels:
+              kubernetes.io/metadata.name: envoy-gateway-system
+      ports:
+        - port: 7007
+          protocol: TCP
+    - from:
+        - ipBlock:
+            cidr: 192.168.10.127/32
+      ports:
+        - port: 7007
+          protocol: TCP

apps/backstage/service.yaml

@@ -0,0 +1,13 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: backstage
+  namespace: backstage
+spec:
+  selector:
+    app.kubernetes.io/name: backstage
+  ports:
+    - name: http
+      port: 7007
+      protocol: TCP
+      targetPort: http

apps/backstage/serviceaccount.yaml

@@ -0,0 +1,6 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: backstage
+  namespace: backstage
+automountServiceAccountToken: false

apps/blocky/compose.yaml

@@ -1,4 +1,4 @@
-# Blocky — LAN DNS. STAGED, NOT DEPLOYED. See README.md.
+# Blocky — 已部署的 LAN 主 DNS。见 README.md。
 #
 # WHY compose on the laptop and NOT a k3s Deployment, given everything else here
 # is Kubernetes:
@@ -52,3 +52,10 @@ services:
       options:
         max-size: "10m"
         max-file: "3"
+
+# 避免与 DN42 的 172.20.0.0/14 重叠。
+networks:
+  default:
+    ipam:
+      config:
+        - subnet: 172.28.0.0/24

apps/blocky/config.yml

@@ -1,9 +1,7 @@
 # Blocky — LAN resolver, ad-blocker and split-horizon DNS.
 #
-# DEPLOYED 2026-07-28 and verified, but NOT yet the LAN resolver — clients still
+# LAN 主 DNS 为 192.168.10.127，NEC IX 192.168.10.1 为备用。
-# get the DC/router pair from DHCP. Making it the resolver needs a DHCP change on
+# DN42 条件转发经 VyOS，参见 README.md。
-# the NEC IX; see README.md. Until then only clients that query 192.168.10.127
-# explicitly are affected, so this is safely reversible.
 
 ports:
   # These are the CONTAINER's listen addresses, so they must be unqualified —
@@ -35,6 +33,13 @@ conditional:
   # Queries for the AD zone go straight to the DC, which is authoritative. This
   # replaces the "DC first, router second" resolver ordering that clients use today.
   mapping:
+    # DN42 由 VyOS 使用注册地址转发，避免 LAN 私网源地址缺少回程。
+    dn42: 192.168.10.2
+    20.172.in-addr.arpa: 192.168.10.2
+    21.172.in-addr.arpa: 192.168.10.2
+    22.172.in-addr.arpa: 192.168.10.2
+    23.172.in-addr.arpa: 192.168.10.2
+    d.f.ip6.arpa: 192.168.10.2
     ad.ddupan.top: 192.168.10.5
     # Reverse lookups for LAN hosts — the DC holds the reverse zone.
     10.168.192.in-addr.arpa: 192.168.10.5
@@ -54,9 +59,11 @@ customDNS:
   # laptop's only global IPv6 belongs to tun0, so a AAAA answer would send LAN
   # traffic into the VPN. See CLAUDE.md.
   mapping:
+    # BEGIN GENERATED: homelab DNS (blocky)
     git.ddupan.top: 192.168.10.127
     auth.ddupan.top: 192.168.10.127
     obj.ddupan.top: 192.168.10.127
+    # END GENERATED: homelab DNS (blocky)
 
 blocking:
   denylists:

apps/gitea/README.md

@@ -44,3 +44,15 @@ API、OIDC、Git/Flux 和 runner 均已验证。第二跳按明确决定跳过
 结构，或者 release notes 指出相关 breaking migration 时，才恢复停机一致备份、分阶段
 suspend、详细日志审计和扩展验收。出现启动失败或 migration error 时也立即升级为完整
 故障流程。
+
+## Hydra 人类登录 PoC
+
+新增 `hydra` OIDC 登录源，旧 `authelia` 入口保留。Hydra 通过通用 OIDC Login/Consent
+适配器转到现有 Authelia 完成人类认证；不是 Gitea 直接验证 LDAP 或 SPIFFE。
+入口为 `https://git.ddupan.top/user/oauth2/hydra`，需 LAN/Tailscale 可达 Hydra 内网域名。
+
+新 client secret 通过 `ExternalSecret/gitea-hydra-oidc` 从 OpenBao 投射。沿用
+preferred_username、已验证邮箱与 groups；当前仍映射 gitea-admins，不在本轮切换组模型。
+先部署并验证 Hydra discovery 后再接入本配置，避免 Gitea init 因上游不可达而失败。
+
+实际登录验收与部署状态见 wiki；依赖和回退见 [Hydra README](../hydra/README.md)。

apps/gitea/gitea-values.yaml

@@ -48,6 +48,11 @@ gitea:
       # github.com is reachable from this network (verified 2026-07-28) even when
       # pypi.org/Fastly is not — see the flaky-WAN notes in the lint workflow.
       DEFAULT_ACTIONS_URL: github
+    webhook:
+      # Keep the default public-internet access for existing hooks while allowing
+      # only the dynamic Runner controller's exact in-cluster DNS name. Do not
+      # broaden this to the built-in `private` network group.
+      ALLOWED_HOST_LIST: external,dynamic-runner-controller.dynamic-runner.svc.cluster.local
     mailer:
       # Outbound mail via the in-cluster Postfix+OAuth relay (see ../smtp-relay/).
       # Plain SMTP on :25 — the relay does STARTTLS + OAuth to M365. From must be the
@@ -111,6 +116,13 @@ gitea:
       scopes: openid profile email groups
       groupClaimName: groups
       adminGroup: gitea-admins
+    - name: hydra
+      provider: openidConnect
+      existingSecret: gitea-hydra-oidc
+      autoDiscoverUrl: https://hydra.ad.ddupan.top/.well-known/openid-configuration
+      scopes: openid profile email groups
+      groupClaimName: groups
+      adminGroup: gitea-admins
 
 persistence:
   size: 20Gi

apps/gitea/hydra-external-secret.yaml

@@ -0,0 +1,22 @@
+apiVersion: external-secrets.io/v1
+kind: ExternalSecret
+metadata:
+  name: gitea-hydra-oidc
+  namespace: gitea
+spec:
+  refreshInterval: 1h
+  secretStoreRef:
+    kind: ClusterSecretStore
+    name: openbao
+  target:
+    name: gitea-hydra-oidc
+    creationPolicy: Owner
+    template:
+      data:
+        key: gitea
+        secret: "{{ .client_secret }}"
+  data:
+    - secretKey: client_secret
+      remoteRef:
+        key: k8s/hydra
+        property: gitea_client_secret

apps/gitea/kustomization.yaml

@@ -16,3 +16,4 @@ resources:
   - helmrepository.yaml
   - helmrelease.yaml
   - httproute.yaml
+  - hydra-external-secret.yaml

apps/hydra/README.md

@@ -0,0 +1,102 @@
+# Hydra 与 OIDC Login/Consent PoC
+
+本目录提供独立 Hydra 签发服务，以及一个薄的 **OIDC 上游适配器**。当前上游配置为
+Authelia；适配器不连接 LDAP，也不管理用户目录。Samba AD、密码和 MFA 继续由现有
+Authelia 链路负责。第一轮只接入人类和 Gitea，不实现 agent 动态授权。
+
+```text
+Gitea → Hydra → OIDC Login/Consent → Authelia → Samba AD
+      ← OIDC ← 经验证的上游身份 ← OIDC callback
+```
+
+目标入口：
+
+- `https://hydra.ad.ddupan.top`：Hydra 公共 OAuth2/OIDC endpoint。
+- `https://hydra-login.ad.ddupan.top`：上游 OIDC 登录及 consent 适配器。
+- `hydra-admin.hydra.svc.cluster.local:4445`：仅集群内管理接口，无 HTTPRoute。
+
+均为 LAN/Tailscale 入口，复用 Envoy `eg/https` wildcard TLS。没有增加公网 tunnel。
+部署及实际验收状态以 wiki 和对应 PR 为准，文件存在不表示登录已验收。
+
+## 首次使用与边界
+
+在 Gitea 登录页选择 `hydra`，跳转到 Authelia 完成现有人类认证，再返回原有 Gitea
+账号。旧 `authelia` 登录源保留。Gitea 的账号关联和资源权限仍由 Gitea 维护。
+
+适配器要求验证上游 issuer、audience、签名、过期时间和 nonce，使用 PKCE S256，
+并把单次 state 绑定到 Secure/HttpOnly/SameSite=Lax cookie。短期登录事务只存内存，
+最多 1024 个、10 分钟过期；单副本重启后正在登录的用户需重试，不存人类密码或 token。
+
+Hydra subject 为上游 `(issuer, sub)` 的 SHA-256 加 `human:` 前缀，与可变邮箱/用户名
+分离。第一轮要求上游返回经过验证的 email 及 preferred_username；这些 claims 必须
+明确配置进 ID token。更换 issuer 会改变本 PoC 的 subject，正式迁移前需要身份绑定设计。
+
+仅为显式 `ALLOWED_CLIENTS=gitea` 自动 consent，scope 限于 openid/profile/email/groups；
+拒绝额外 access-token audience，不发 refresh token。只按实际请求 scope 释放 claims。
+这不是通用的无人确认授权服务。组当前透传，沿用 Gitea 的 gitea-admins 映射；统一组
+模型和 agent 认证均在后续阶段。不存在对 Authelia 专有协议的调用。
+
+NetworkPolicy 限制公共端口只接收 Envoy 流量，Hydra admin 只允许适配器访问。
+Hydra 使用正式模式，TLS 由 Envoy 终止；不使用 `--dev`。管理操作使用受控
+`kubectl port-forward`，不要将 admin 接口暴露到 Gateway。
+
+## 依赖、秘密与初始化
+
+依赖共享 CloudNativePG、OpenBao/ESO、Authelia OIDC、Envoy、Samba DNS、zot 镜像仓库。
+Hydra 使用独立 `hydra` database/role，不与其他应用共享数据库角色。
+
+`kv/k8s/hydra` 保存 dsn、system_secret、upstream_client_secret、upstream_client_digest、
+gitea_client_secret；通过 ExternalSecret 投射，值不写入 Git。Bootstrap 创建角色及数据库
+后才启动 Hydra migration。system_secret 必须持久保存，不得在重启时随机重建。
+
+Authelia 中新增 confidential client `hydra-login`：
+
+- redirect URI：`https://hydra-login.ad.ddupan.top/callback`；
+- authorization policy：two_factor；grant：authorization_code；PKCE：S256；
+- token endpoint auth：client_secret_basic；scope：openid/profile/email/groups；
+- claims policy：把 preferred_username、name、email、email_verified、groups 放入 ID token；
+- client secret 的 PBKDF2 digest 存入 Authelia，原值仅供适配器使用。
+
+Authelia 尚非 Flux 管理。修改 Helm values 时保留所有已有 clients 与 secret 引用，
+通过 `--reuse-values` 和最小 overlay 增加客户端，不能以本目录配置覆盖其完整 values。
+
+Hydra 中注册 confidential client `gitea`，redirect URI 为
+`https://git.ddupan.top/user/oauth2/hydra/callback`，grant/response 为 authorization_code/code，
+scope 为 openid/profile/email/groups，token endpoint auth 为 client_secret_basic。
+Gitea 启动时读取 OIDC discovery，所以应先确认 Hydra 健康和 discovery 可达，再接入 Gitea。
+
+## 构建与检查
+
+```bash
+cd apps/hydra/login-consent
+go test -race ./...
+go vet ./...
+CGO_ENABLED=0 go build -trimpath -ldflags='-s -w' -o login-consent .
+docker build -t hydra-login-consent:VERSION .
+```
+
+Go module 独立，依赖由 go.sum 锁定；Dockerfile 固定基础镜像 digest。
+使用已授权的短期 SPIFFE zot 凭据发布镜像，部署使用匿名拉取入口与不可变 digest。
+不把 registry 凭据写入源码或 build args。
+
+```bash
+kubectl kustomize apps/hydra
+sudo k3s kubectl -n hydra get deployment,pod,externalsecret,httproute
+sudo k3s kubectl -n hydra logs deployment/hydra -c migrate
+sudo k3s kubectl -n hydra logs deployment/hydra-login
+```
+
+日志不输出上游 token、授权 code、challenge 或秘密。登录失败先查两端 Pod 状态、
+DNS/discovery 连通性、client redirect URI 和 scope，再由用户重新发起登录。
+不要在故障排查中关闭签名验证、MFA 或 state/nonce 校验。
+
+## 恢复与回退
+
+保留共享 PostgreSQL 中 Hydra 数据及 OpenBao 秘密；数据库持有 clients、会话及签名密钥，
+单独重建 Deployment 不能替代恢复数据库。先恢复依赖，再启动 Hydra 和适配器。
+当前恢复仍依赖 homelab 共享基础设施，不能声称已完成独立灾备。
+
+第一轮不切换 Authelia 的主入口。撤回 Gitea 的新增 Hydra 登录源即可回到旧入口；
+先撤消费者，再考虑停用 Hydra。不要删除旧 Authelia 登录源、用户或数据库作为回退手段。
+
+跨服务设计见 [独立 IAM 草案](https://git.ddupan.top/panxiao81/homelab-wiki/src/branch/main/architecture/independent-iam-draft.md)。

apps/hydra/deployment.yaml

@@ -0,0 +1,95 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: hydra
+  namespace: hydra
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: hydra
+  template:
+    metadata:
+      labels:
+        app: hydra
+    spec:
+      automountServiceAccountToken: false
+      securityContext:
+        runAsNonRoot: true
+        runAsUser: 1000
+        runAsGroup: 1000
+        seccompProfile:
+          type: RuntimeDefault
+      initContainers:
+      - name: migrate
+        image: docker.io/oryd/hydra:v26.2.0@sha256:ff67c7fb5f95074fa53374d41151713554960504b340cd3f95b09e65deaea2a9
+        args:
+        - migrate
+        - sql
+        - -e
+        - --yes
+        env:
+        - name: DSN
+          valueFrom:
+            secretKeyRef:
+              name: hydra
+              key: dsn
+        securityContext: &id002
+          allowPrivilegeEscalation: false
+          readOnlyRootFilesystem: true
+          capabilities:
+            drop:
+            - ALL
+        resources: &id001
+          requests:
+            cpu: 50m
+            memory: 64Mi
+          limits:
+            memory: 256Mi
+      containers:
+      - name: hydra
+        image: docker.io/oryd/hydra:v26.2.0@sha256:ff67c7fb5f95074fa53374d41151713554960504b340cd3f95b09e65deaea2a9
+        args:
+        - serve
+        - all
+        - --config
+        - /etc/hydra/hydra.yaml
+        - --sqa-opt-out
+        env:
+        - name: DSN
+          valueFrom:
+            secretKeyRef:
+              name: hydra
+              key: dsn
+        - name: SECRETS_SYSTEM
+          valueFrom:
+            secretKeyRef:
+              name: hydra
+              key: system_secret
+        ports:
+        - name: public
+          containerPort: 4444
+        - name: admin
+          containerPort: 4445
+        resources: *id001
+        securityContext: *id002
+        volumeMounts:
+        - name: config
+          mountPath: /etc/hydra
+          readOnly: true
+        readinessProbe:
+          httpGet:
+            path: /health/ready
+            port: admin
+          initialDelaySeconds: 5
+          periodSeconds: 5
+        livenessProbe:
+          httpGet:
+            path: /health/alive
+            port: admin
+          initialDelaySeconds: 20
+          periodSeconds: 20
+      volumes:
+      - name: config
+        configMap:
+          name: hydra-config

apps/hydra/external-secret.yaml

@@ -0,0 +1,16 @@
+apiVersion: external-secrets.io/v1
+kind: ExternalSecret
+metadata:
+  name: hydra
+  namespace: hydra
+spec:
+  refreshInterval: 1h
+  secretStoreRef:
+    kind: ClusterSecretStore
+    name: openbao
+  target:
+    name: hydra
+    creationPolicy: Owner
+  dataFrom:
+  - extract:
+      key: k8s/hydra

apps/hydra/httproutes.yaml

@@ -0,0 +1,33 @@
+apiVersion: gateway.networking.k8s.io/v1
+kind: HTTPRoute
+metadata:
+  name: hydra-public
+  namespace: hydra
+spec:
+  parentRefs:
+  - name: eg
+    namespace: envoy-gateway-system
+    sectionName: https
+  hostnames:
+  - hydra.ad.ddupan.top
+  rules:
+  - backendRefs:
+    - name: hydra-public
+      port: 4444
+---
+apiVersion: gateway.networking.k8s.io/v1
+kind: HTTPRoute
+metadata:
+  name: hydra-login
+  namespace: hydra
+spec:
+  parentRefs:
+  - name: eg
+    namespace: envoy-gateway-system
+    sectionName: https
+  hostnames:
+  - hydra-login.ad.ddupan.top
+  rules:
+  - backendRefs:
+    - name: hydra-login
+      port: 8080

apps/hydra/hydra.yaml

@@ -0,0 +1,25 @@
+serve:
+  public:
+    port: 4444
+  admin:
+    port: 4445
+  tls:
+    allow_termination_from:
+    - 10.42.0.0/16
+  cookies:
+    same_site_mode: Lax
+urls:
+  self:
+    issuer: https://hydra.ad.ddupan.top
+    public: https://hydra.ad.ddupan.top
+  login: https://hydra-login.ad.ddupan.top/login
+  consent: https://hydra-login.ad.ddupan.top/consent
+ttl:
+  access_token: 15m
+  id_token: 15m
+  auth_code: 5m
+log:
+  level: info
+  leak_sensitive_values: false
+oauth2:
+  expose_internal_errors: false

apps/hydra/kustomization.yaml

@@ -0,0 +1,15 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+- namespace.yaml
+- external-secret.yaml
+- deployment.yaml
+- login-deployment.yaml
+- services.yaml
+- httproutes.yaml
+- networkpolicy.yaml
+configMapGenerator:
+- name: hydra-config
+  namespace: hydra
+  files:
+  - hydra.yaml

apps/hydra/login-consent/.gitignore

@@ -0,0 +1 @@
+/login-consent

apps/hydra/login-consent/Dockerfile

@@ -0,0 +1,4 @@
+FROM gcr.io/distroless/static-debian12:nonroot@sha256:afa5c872c891853ca7fcf1f12c3edb23f7eeef36189728842dd51042ff57f7ab
+COPY login-consent /login-consent
+USER 65532:65532
+ENTRYPOINT ["/login-consent"]

apps/hydra/login-consent/go.mod

@@ -0,0 +1,13 @@
+module git.ddupan.top/panxiao81/homelab-infra/apps/hydra/login-consent
+
+go 1.26.0
+
+require (
+	github.com/coreos/go-oidc/v3 v3.14.1
+	golang.org/x/oauth2 v0.37.0
+)
+
+require (
+	github.com/go-jose/go-jose/v4 v4.0.5 // indirect
+	golang.org/x/crypto v0.36.0 // indirect
+)

apps/hydra/login-consent/go.sum

@@ -0,0 +1,18 @@
+github.com/coreos/go-oidc/v3 v3.14.1 h1:9ePWwfdwC4QKRlCXsJGou56adA/owXczOzwKdOumLqk=
+github.com/coreos/go-oidc/v3 v3.14.1/go.mod h1:HaZ3szPaZ0e4r6ebqvsLWlk2Tn+aejfmrfah6hnSYEU=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/go-jose/go-jose/v4 v4.0.5 h1:M6T8+mKZl/+fNNuFHvGIzDz7BTLQPIounk/b9dw3AaE=
+github.com/go-jose/go-jose/v4 v4.0.5/go.mod h1:s3P1lRrkT8igV8D9OjyL4WRyHvjB6a4JSllnOrmmBOA=
+github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
+github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA=
+github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
+golang.org/x/crypto v0.36.0 h1:AnAEvhDddvBdpY+uR+MyHmuZzzNqXSe/GvuDeob5L34=
+golang.org/x/crypto v0.36.0/go.mod h1:Y4J0ReaxCR1IMaabaSMugxJES1EpwhBHhv2bDHklZvc=
+golang.org/x/oauth2 v0.37.0 h1:JUlcxA8oAtauLfiH8FX2/FkAWHAdi0QtGCGc+hofE98=
+golang.org/x/oauth2 v0.37.0/go.mod h1:IxwZNxUULJmpBFf9K/9NTMSIfZZuvuTy1gGxhigP/58=
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
+gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

apps/hydra/login-consent/main.go

@@ -0,0 +1,285 @@
+// Login/Consent adapter for a single trusted upstream and first-party clients.
+package main
+
+import (
+	"bytes"
+	"context"
+	"crypto/rand"
+	"crypto/sha256"
+	"crypto/subtle"
+	"encoding/base64"
+	"encoding/hex"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"log"
+	"net/http"
+	"net/url"
+	"os"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/coreos/go-oidc/v3/oidc"
+	"golang.org/x/oauth2"
+)
+
+const cookieName = "__Host-hydra-login"
+
+type pending struct {
+	Challenge, Nonce, Verifier string
+	Expires                    time.Time
+}
+type claims struct {
+	Username      string   `json:"preferred_username"`
+	Email         string   `json:"email"`
+	EmailVerified bool     `json:"email_verified"`
+	Name          string   `json:"name"`
+	Groups        []string `json:"groups"`
+}
+type flowRequest struct {
+	Client struct {
+		ID string `json:"client_id"`
+	} `json:"client"`
+	Subject  string   `json:"subject"`
+	Scopes   []string `json:"requested_scope"`
+	Audience []string `json:"requested_access_token_audience"`
+	Context  claims   `json:"context"`
+}
+type app struct {
+	admin, public string
+	client        *http.Client
+	oauth         oauth2.Config
+	verifier      *oidc.IDTokenVerifier
+	allowed       map[string]bool
+	mu            sync.Mutex
+	pending       map[string]pending
+}
+
+func required(key string) string {
+	v := os.Getenv(key)
+	if v == "" {
+		log.Fatalf("missing %s", key)
+	}
+	return v
+}
+func random() string {
+	b := make([]byte, 32)
+	if _, err := rand.Read(b); err != nil {
+		panic(err)
+	}
+	return base64.RawURLEncoding.EncodeToString(b)
+}
+func (a *app) api(ctx context.Context, method, path string, in, out any) error {
+	var body io.Reader
+	if in != nil {
+		b, err := json.Marshal(in)
+		if err != nil {
+			return err
+		}
+		body = bytes.NewReader(b)
+	}
+	req, err := http.NewRequestWithContext(ctx, method, a.admin+path, body)
+	if err != nil {
+		return err
+	}
+	req.Header.Set("Content-Type", "application/json")
+	resp, err := a.client.Do(req)
+	if err != nil {
+		return errors.New("Hydra unavailable")
+	}
+	defer resp.Body.Close()
+	if resp.StatusCode < 200 || resp.StatusCode >= 300 {
+		return fmt.Errorf("Hydra status %d", resp.StatusCode)
+	}
+	if out != nil {
+		return json.NewDecoder(io.LimitReader(resp.Body, 1<<20)).Decode(out)
+	}
+	return nil
+}
+func (a *app) request(r *http.Request, kind, challenge string) (flowRequest, error) {
+	var f flowRequest
+	if challenge == "" || len(challenge) > 8192 {
+		return f, errors.New("missing or invalid challenge")
+	}
+	err := a.api(r.Context(), http.MethodGet, "/admin/oauth2/auth/requests/"+kind+"?"+kind+"_challenge="+url.QueryEscape(challenge), nil, &f)
+	if err != nil {
+		return f, err
+	}
+	if !a.allowed[f.Client.ID] {
+		return f, errors.New("client not allowed")
+	}
+	return f, nil
+}
+func (a *app) accept(w http.ResponseWriter, r *http.Request, kind, challenge string, body any) {
+	var result struct {
+		Redirect string `json:"redirect_to"`
+	}
+	if err := a.api(r.Context(), http.MethodPut, "/admin/oauth2/auth/requests/"+kind+"/accept?"+kind+"_challenge="+url.QueryEscape(challenge), body, &result); err != nil {
+		fail(w, 502)
+		return
+	}
+	// Only Hydra's own authorization endpoint can receive a challenge verifier.
+	u, err := url.Parse(result.Redirect)
+	p, _ := url.Parse(a.public)
+	if err != nil || u.Scheme != p.Scheme || u.Host != p.Host || u.User != nil || u.Path != "/oauth2/auth" {
+		fail(w, 502)
+		return
+	}
+	http.Redirect(w, r, result.Redirect, http.StatusSeeOther)
+}
+func fail(w http.ResponseWriter, status int) { http.Error(w, http.StatusText(status), status) }
+func (a *app) login(w http.ResponseWriter, r *http.Request) {
+	challenge := r.URL.Query().Get("login_challenge")
+	if _, err := a.request(r, "login", challenge); err != nil {
+		fail(w, 403)
+		return
+	}
+	state := random()
+	p := pending{challenge, random(), oauth2.GenerateVerifier(), time.Now().Add(10 * time.Minute)}
+	a.mu.Lock()
+	for k, v := range a.pending {
+		if time.Now().After(v.Expires) {
+			delete(a.pending, k)
+		}
+	}
+	if len(a.pending) >= 1024 {
+		a.mu.Unlock()
+		fail(w, 503)
+		return
+	}
+	a.pending[state] = p
+	a.mu.Unlock()
+	http.SetCookie(w, &http.Cookie{Name: cookieName, Value: state, Path: "/", Secure: true, HttpOnly: true, SameSite: http.SameSiteLaxMode, MaxAge: 600})
+	http.Redirect(w, r, a.oauth.AuthCodeURL(state, oidc.Nonce(p.Nonce), oauth2.S256ChallengeOption(p.Verifier)), http.StatusSeeOther)
+}
+func (a *app) take(r *http.Request) (pending, error) {
+	state := r.URL.Query().Get("state")
+	cookie, err := r.Cookie(cookieName)
+	if err != nil || state == "" || subtle.ConstantTimeCompare([]byte(cookie.Value), []byte(state)) != 1 {
+		return pending{}, errors.New("state mismatch")
+	}
+	a.mu.Lock()
+	defer a.mu.Unlock()
+	p, ok := a.pending[state]
+	delete(a.pending, state)
+	if !ok || time.Now().After(p.Expires) {
+		return pending{}, errors.New("expired or used state")
+	}
+	return p, nil
+}
+func (a *app) callback(w http.ResponseWriter, r *http.Request) {
+	p, err := a.take(r)
+	if err != nil {
+		fail(w, 403)
+		return
+	}
+	http.SetCookie(w, &http.Cookie{Name: cookieName, Path: "/", Secure: true, HttpOnly: true, SameSite: http.SameSiteLaxMode, MaxAge: -1})
+	if r.URL.Query().Get("error") != "" || r.URL.Query().Get("code") == "" {
+		fail(w, 403)
+		return
+	}
+	ctx := oidc.ClientContext(r.Context(), a.client)
+	token, err := a.oauth.Exchange(ctx, r.URL.Query().Get("code"), oauth2.VerifierOption(p.Verifier))
+	if err != nil {
+		fail(w, 502)
+		return
+	}
+	raw, ok := token.Extra("id_token").(string)
+	if !ok {
+		fail(w, 502)
+		return
+	}
+	id, err := a.verifier.Verify(ctx, raw)
+	if err != nil || id.Nonce != p.Nonce || id.Subject == "" {
+		fail(w, 403)
+		return
+	}
+	var c claims
+	if id.Claims(&c) != nil || c.Username == "" || c.Email == "" || !c.EmailVerified {
+		fail(w, 403)
+		return
+	}
+	if _, err := a.request(r, "login", p.Challenge); err != nil {
+		fail(w, 403)
+		return
+	}
+	// Stable identity is tied to the verified upstream issuer+subject, never email.
+	sum := sha256.Sum256([]byte(id.Issuer + "\x00" + id.Subject))
+	a.accept(w, r, "login", p.Challenge, map[string]any{"subject": "human:" + hex.EncodeToString(sum[:]), "remember": false, "context": c})
+}
+func consentSession(f flowRequest) (map[string]any, error) {
+	if !strings.HasPrefix(f.Subject, "human:") || f.Context.Username == "" || f.Context.Email == "" || !f.Context.EmailVerified {
+		return nil, errors.New("invalid identity context")
+	}
+	allowed := map[string]bool{"openid": true, "profile": true, "email": true, "groups": true}
+	session := map[string]any{"principal_type": "human"}
+	for _, scope := range f.Scopes {
+		if !allowed[scope] {
+			return nil, errors.New("scope not allowed")
+		}
+		switch scope {
+		case "profile":
+			session["preferred_username"] = f.Context.Username
+			session["name"] = f.Context.Name
+		case "email":
+			session["email"] = f.Context.Email
+			session["email_verified"] = true
+		case "groups":
+			session["groups"] = f.Context.Groups
+		}
+	}
+	if len(f.Audience) > 0 {
+		return nil, errors.New("access token audience not allowed")
+	}
+	return session, nil
+}
+func (a *app) consent(w http.ResponseWriter, r *http.Request) {
+	challenge := r.URL.Query().Get("consent_challenge")
+	f, err := a.request(r, "consent", challenge)
+	if err != nil {
+		fail(w, 403)
+		return
+	}
+	session, err := consentSession(f)
+	if err != nil {
+		fail(w, 403)
+		return
+	}
+	// Explicit policy for pre-approved first-party clients only; no generic auto-consent.
+	a.accept(w, r, "consent", challenge, map[string]any{"grant_scope": f.Scopes, "remember": false, "session": map[string]any{"id_token": session}})
+}
+func (a *app) handler() http.Handler {
+	mux := http.NewServeMux()
+	mux.HandleFunc("GET /healthz", func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(200) })
+	mux.HandleFunc("GET /login", a.login)
+	mux.HandleFunc("GET /callback", a.callback)
+	mux.HandleFunc("GET /consent", a.consent)
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.Header().Set("Cache-Control", "no-store")
+		w.Header().Set("Referrer-Policy", "no-referrer")
+		w.Header().Set("X-Content-Type-Options", "nosniff")
+		w.Header().Set("Content-Security-Policy", "default-src 'none'; frame-ancestors 'none'")
+		mux.ServeHTTP(w, r)
+	})
+}
+func main() {
+	client := &http.Client{Timeout: 15 * time.Second, CheckRedirect: func(r *http.Request, via []*http.Request) error { return http.ErrUseLastResponse }}
+	issuer := required("UPSTREAM_ISSUER")
+	ctx := oidc.ClientContext(context.Background(), client)
+	provider, err := oidc.NewProvider(ctx, issuer)
+	if err != nil {
+		log.Fatal("upstream discovery failed")
+	}
+	clientID := required("UPSTREAM_CLIENT_ID")
+	a := &app{admin: required("HYDRA_ADMIN_URL"), public: required("HYDRA_PUBLIC_URL"), client: client, allowed: map[string]bool{}, pending: map[string]pending{},
+		oauth:    oauth2.Config{ClientID: clientID, ClientSecret: required("UPSTREAM_CLIENT_SECRET"), RedirectURL: required("CALLBACK_URL"), Endpoint: provider.Endpoint(), Scopes: []string{"openid", "profile", "email", "groups"}},
+		verifier: provider.Verifier(&oidc.Config{ClientID: clientID})}
+	for _, id := range strings.Split(required("ALLOWED_CLIENTS"), ",") {
+		a.allowed[id] = true
+	}
+	s := http.Server{Addr: ":8080", Handler: a.handler(), ReadHeaderTimeout: 5 * time.Second, ReadTimeout: 20 * time.Second, WriteTimeout: 45 * time.Second, IdleTimeout: 60 * time.Second, MaxHeaderBytes: 16384}
+	log.Print("login/consent adapter listening on :8080")
+	log.Fatal(s.ListenAndServe())
+}

apps/hydra/login-consent/main_test.go

@@ -0,0 +1,107 @@
+package main
+
+import (
+	"encoding/json"
+	"net/http"
+	"net/http/httptest"
+	"net/url"
+	"strings"
+	"testing"
+	"time"
+
+	"golang.org/x/oauth2"
+)
+
+func TestStateBoundToCookieSingleUseAndExpiry(t *testing.T) {
+	a := &app{pending: map[string]pending{"valid": {Challenge: "challenge", Expires: time.Now().Add(time.Minute)}, "expired": {Expires: time.Now().Add(-time.Minute)}}}
+	request := func(state, cookie string) *http.Request {
+		r := httptest.NewRequest("GET", "https://login.example/callback?state="+state, nil)
+		if cookie != "" {
+			r.AddCookie(&http.Cookie{Name: cookieName, Value: cookie})
+		}
+		return r
+	}
+	for _, r := range []*http.Request{request("valid", ""), request("valid", "other"), request("expired", "expired")} {
+		if _, err := a.take(r); err == nil {
+			t.Fatal("invalid state accepted")
+		}
+	}
+	if p, err := a.take(request("valid", "valid")); err != nil || p.Challenge != "challenge" {
+		t.Fatal("valid state rejected")
+	}
+	if _, err := a.take(request("valid", "valid")); err == nil {
+		t.Fatal("replayed state accepted")
+	}
+}
+func TestConsentRejectsPrivilegeExpansionAndFiltersClaims(t *testing.T) {
+	f := flowRequest{Subject: "human:known", Scopes: []string{"openid", "email"}, Context: claims{Username: "alice", Email: "[email protected]", EmailVerified: true, Groups: []string{"operators"}}}
+	s, err := consentSession(f)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if _, ok := s["groups"]; ok {
+		t.Fatal("groups leaked without scope")
+	}
+	if _, ok := s["preferred_username"]; ok {
+		t.Fatal("profile leaked without scope")
+	}
+	for _, scope := range []string{"admin", "offline_access", "unknown"} {
+		bad := f
+		bad.Scopes = append([]string{"openid"}, scope)
+		if _, err := consentSession(bad); err == nil {
+			t.Fatalf("accepted %s", scope)
+		}
+	}
+	f.Audience = []string{"other-service"}
+	if _, err := consentSession(f); err == nil {
+		t.Fatal("unexpected audience accepted")
+	}
+	f.Audience = nil
+	f.Context.EmailVerified = false
+	if _, err := consentSession(f); err == nil {
+		t.Fatal("unverified email accepted")
+	}
+}
+func TestLoginValidatesClientAndUsesPKCEAndNonce(t *testing.T) {
+	admin := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		json.NewEncoder(w).Encode(map[string]any{"client": map[string]string{"client_id": r.URL.Query().Get("login_challenge")}})
+	}))
+	defer admin.Close()
+	a := &app{admin: admin.URL, client: admin.Client(), allowed: map[string]bool{"gitea": true}, pending: map[string]pending{}, oauth: oauth2.Config{ClientID: "hydra-login", RedirectURL: "https://login.example/callback", Endpoint: oauth2.Endpoint{AuthURL: "https://upstream.example/authorize"}}}
+	w := httptest.NewRecorder()
+	a.handler().ServeHTTP(w, httptest.NewRequest("GET", "https://login.example/login?login_challenge=rogue", nil))
+	if w.Code != 403 {
+		t.Fatal("unknown client accepted")
+	}
+	w = httptest.NewRecorder()
+	a.handler().ServeHTTP(w, httptest.NewRequest("GET", "https://login.example/login?login_challenge=gitea", nil))
+	if w.Code != 303 {
+		t.Fatalf("status %d", w.Code)
+	}
+	u, _ := url.Parse(w.Header().Get("Location"))
+	q := u.Query()
+	if q.Get("code_challenge_method") != "S256" || q.Get("code_challenge") == "" || q.Get("nonce") == "" || q.Get("state") == "" {
+		t.Fatal("missing protocol binding")
+	}
+	cookies := w.Result().Cookies()
+	if len(cookies) != 1 || !cookies[0].Secure || !cookies[0].HttpOnly || cookies[0].SameSite != http.SameSiteLaxMode || cookies[0].Value != q.Get("state") {
+		t.Fatal("unsafe cookie")
+	}
+	if w.Header().Get("Cache-Control") != "no-store" {
+		t.Fatal("missing cache protection")
+	}
+}
+func TestHydraRedirectCannotLeaveTrustedOrigin(t *testing.T) {
+	for _, target := range []string{"https://evil.example/oauth2/auth", "https://[email protected]/oauth2/auth", "https://hydra.example/other"} {
+		admin := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			json.NewEncoder(w).Encode(map[string]string{"redirect_to": target})
+		}))
+		a := &app{admin: admin.URL, public: "https://hydra.example", client: admin.Client()}
+		w := httptest.NewRecorder()
+		a.accept(w, httptest.NewRequest("GET", "https://login.example/login", nil), "login", "challenge", map[string]string{"subject": "human:test"})
+		if w.Code != 502 || strings.Contains(w.Header().Get("Location"), "evil") {
+			t.Fatal("untrusted redirect accepted")
+		}
+		admin.Close()
+	}
+}

apps/hydra/login-deployment.yaml

@@ -0,0 +1,69 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: hydra-login
+  namespace: hydra
+spec:
+  replicas: 1
+  strategy:
+    type: Recreate
+  selector:
+    matchLabels:
+      app: hydra-login
+  template:
+    metadata:
+      labels:
+        app: hydra-login
+    spec:
+      automountServiceAccountToken: false
+      securityContext:
+        runAsNonRoot: true
+        runAsUser: 65532
+        runAsGroup: 65532
+        seccompProfile:
+          type: RuntimeDefault
+      containers:
+      - name: login-consent
+        image: zot.ad.ddupan.top/iam/oidc-login-consent@sha256:fede9b9e93c457c4b7a8a6022d9df86ff5d5900d3f6b6c4f3439851a0ae1e944
+        env:
+        - name: HYDRA_ADMIN_URL
+          value: http://hydra-admin.hydra.svc.cluster.local:4445
+        - name: HYDRA_PUBLIC_URL
+          value: https://hydra.ad.ddupan.top
+        - name: UPSTREAM_ISSUER
+          value: https://auth.ddupan.top
+        - name: UPSTREAM_CLIENT_ID
+          value: hydra-login
+        - name: CALLBACK_URL
+          value: https://hydra-login.ad.ddupan.top/callback
+        - name: ALLOWED_CLIENTS
+          value: gitea
+        - name: UPSTREAM_CLIENT_SECRET
+          valueFrom:
+            secretKeyRef:
+              name: hydra
+              key: upstream_client_secret
+        ports:
+        - name: http
+          containerPort: 8080
+        resources:
+          requests:
+            cpu: 50m
+            memory: 64Mi
+          limits:
+            memory: 256Mi
+        securityContext:
+          allowPrivilegeEscalation: false
+          readOnlyRootFilesystem: true
+          capabilities:
+            drop:
+            - ALL
+        readinessProbe:
+          httpGet:
+            path: /healthz
+            port: http
+        livenessProbe:
+          httpGet:
+            path: /healthz
+            port: http
+          initialDelaySeconds: 10

apps/hydra/namespace.yaml

@@ -0,0 +1,6 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: hydra
+  labels:
+    pod-security.kubernetes.io/enforce: restricted

apps/hydra/networkpolicy.yaml

@@ -0,0 +1,46 @@
+apiVersion: networking.k8s.io/v1
+kind: NetworkPolicy
+metadata:
+  name: hydra
+  namespace: hydra
+spec:
+  podSelector:
+    matchLabels:
+      app: hydra
+  policyTypes:
+  - Ingress
+  ingress:
+  - from:
+    - namespaceSelector:
+        matchLabels:
+          kubernetes.io/metadata.name: envoy-gateway-system
+    ports:
+    - port: 4444
+      protocol: TCP
+  - from:
+    - podSelector:
+        matchLabels:
+          app: hydra-login
+    ports:
+    - port: 4445
+      protocol: TCP
+---
+apiVersion: networking.k8s.io/v1
+kind: NetworkPolicy
+metadata:
+  name: hydra-login
+  namespace: hydra
+spec:
+  podSelector:
+    matchLabels:
+      app: hydra-login
+  policyTypes:
+  - Ingress
+  ingress:
+  - from:
+    - namespaceSelector:
+        matchLabels:
+          kubernetes.io/metadata.name: envoy-gateway-system
+    ports:
+    - port: 8080
+      protocol: TCP

apps/hydra/services.yaml

@@ -0,0 +1,35 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: hydra-public
+  namespace: hydra
+spec:
+  selector:
+    app: hydra
+  ports:
+  - port: 4444
+    targetPort: 4444
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: hydra-admin
+  namespace: hydra
+spec:
+  selector:
+    app: hydra
+  ports:
+  - port: 4445
+    targetPort: 4445
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: hydra-login
+  namespace: hydra
+spec:
+  selector:
+    app: hydra-login
+  ports:
+  - port: 8080
+    targetPort: 8080

apps/netbox/terraform/topology.yml

@@ -52,11 +52,16 @@ prefix_roles:
 # range. It makes the collision VISIBLE — the range shows 100% utilised and the
 # address never appears as a suggestion — where plain YAML shows nothing at all.
 ip_ranges:
-  - start: 192.168.10.10/24
+  - start: 192.168.10.128/24
     end: 192.168.10.250/24
     status: active
     mark_utilized: true
-    description: "NEC IX DHCP pool — do NOT statically allocate inside this."
+    description: "NEC IX DHCP pool, updated 2026-09-14. Do NOT statically allocate inside this."
+  - start: 192.168.10.251/24
+    end: 192.168.10.254/24
+    status: reserved
+    mark_utilized: true
+    description: "用户确认预留，尚未分配；不可按扫描无响应视为空闲。"
 
 vlan_group:
   name: lab
@@ -144,8 +149,8 @@ devices:
     role: hypervisor
     type: 10vgcto1ww
     serial: PC1AGX1Q
-    description: "Proxmox VE 9.2. LINSTOR satellite. The node that randomly froze."
+    description: "Proxmox VE 9.2. LINSTOR satellite."
-    comments: "AMD Ryzen 5 PRO 2400GE w/ Vega, 8 threads, 7 GiB RAM. BIOS M1XKT45A. Raven Ridge idle bug fixed in BIOS: Power Supply Idle Control = Typical Current Idle."
+    comments: "AMD Ryzen 5 PRO 2400GE w/ Vega, 8 threads, 7 GiB RAM. BIOS M1XKT45A."
     interfaces:
       - { name: vmbr0, type: bridge, ip: 192.168.10.7/24, primary: true, mtu: 9000, dns_name: pve2.ad.ddupan.top }
 
@@ -154,7 +159,7 @@ devices:
     type: 10vgcto1ww
     serial: PC1AGX1P
     description: "Proxmox VE 9.2. LINSTOR satellite."
-    comments: "AMD Ryzen 5 PRO 2400GE w/ Vega, 8 threads, 7 GiB RAM. BIOS M1XKT55A. Same silicon as pve2, so susceptible to the same idle bug in principle."
+    comments: "AMD Ryzen 5 PRO 2400GE w/ Vega, 8 threads, 7 GiB RAM. BIOS M1XKT55A."
     interfaces:
       - { name: vmbr0, type: bridge, ip: 192.168.10.9/24, primary: true, mtu: 9000, dns_name: pve3.ad.ddupan.top }
 
@@ -181,9 +186,8 @@ devices:
   # Wi-Fi. Runs as an AP/bridge, not a router — the NEC IX is the gateway, so this box's
   # routing, NAT and DHCP are not in play. Wireless clients land directly on the flat LAN.
   #
-  # ⚠ Its address .10 is the FIRST ADDRESS OF THE DHCP POOL above. Either it holds a lease
+  # 2026-09-14: NEC IX 为此 MAC 固定分配 .10；动态池已迁到 .128–.250。
-  # (so the address can move) or it is a static that overlaps the pool. NetBox surfaces
+  # 操作与回滚记录：infrastructure/samba-ad/router-dhcp-nec-ix.md。
-  # the overlap; the underlying config still needs a decision. See ../README.md.
   #
   # Identified by MAC OUI d4:2c:46 = BUFFALO.INC plus the model string on its login page.
   - name: ap-buffalo

apps/nexus/README.md

@@ -0,0 +1,157 @@
+# Nexus Repository POC
+
+本目录声明一个 Nexus Repository Community Edition POC，用来验证一次性 CI runner 通过
+网络服务复用 Ansible Galaxy、Go Modules 与 OCI/BuildKit 缓存。Nexus 固定为 `3.96.1`，
+镜像固定到官方 multi-arch index digest；LAN 入口为
+`https://nexus.ad.ddupan.top`。
+
+## POC 边界
+
+- 单副本 Deployment，`Recreate` 更新，避免一个 RWO 卷被两个 Pod 同时挂载。
+- `/nexus-data` 使用 `localpv-zfs-ceph` 上的 50 GiB RWO PVC。
+- 资源预算为 250m/2 GiB request、2 CPU/4 GiB limit；JVM heap 上限 2 GiB。
+- 使用容器默认的 embedded H2。它只用于 POC；正式接管 OCI 制品前必须迁移到外部
+  PostgreSQL，并验证备份恢复。
+- 入口只在 LAN wildcard Gateway 上发布，不创建公网 DNS 或 Cloudflare route。
+- 不套 Authelia forward-auth；它会破坏 Go、Ansible 与 OCI 非浏览器客户端。
+- 现有 zot 保持不变。Nexus 完成 OCI、BuildKit cache 和恢复验收前不得迁移或删除 zot。
+
+Terraform provider 创建 `ansible-public`、`go-public`、最小匿名权限与 OCI Bearer Token
+Realm。Nexus 3.94 才加入的原生 OCI repository 已有 REST API，但当前锁定的 community
+provider 尚未暴露 OCI resource；`terraform/reconcile-oci.sh` 因此根据 3.96.1 实例 Swagger
+固定的 JSON schema，幂等调和 `oci-hosted`、`oci-proxy` 与 `oci-public`。不得绕过该入口在
+UI 中创建无人管理的长期 repository。
+
+## 部署
+
+Flux 从 `clusters/homelab/apps/nexus.yaml` 协调本目录，并依赖 Envoy Gateway 与 OpenEBS。
+合并前只渲染配置，不直接 apply：
+
+```bash
+kubectl kustomize apps/nexus
+```
+
+合并并由 Flux 部署后检查：
+
+```bash
+kubectl -n flux-system get kustomization nexus
+kubectl -n nexus get pod,pvc,service,httproute
+kubectl -n nexus logs deployment/nexus --tail=100
+```
+
+启动可能需要数分钟，startup probe 允许最多十分钟。不要因初次启动较慢反复删除 Pod；
+先确认 PVC 已 Bound、Pod 没有 OOM，以及日志仍在推进。
+
+DNS 期望状态已加入 `infrastructure/dns/records.yml`，需从 Samba AD Ansible root 以
+`--check --diff` 核对后再按其 README 应用 DNS tag。没有 DNS 时可先用 port-forward
+验证应用，但不能据此宣称 Gateway 路径已通过。
+
+## 首次初始化与 Terraform
+
+初始管理员密码生成在 PVC 的 `/nexus-data/admin.password`。只在交互式终端中读取并立即
+完成首次密码轮换；不得把密码复制进 shell tracing、工单、Git 或命令参数。随后将
+Terraform 管理账号的凭据存入 OpenBao，由 CI 通过 Terraform input variable 注入以下
+环境变量：
+
+```text
+TF_VAR_nexus_url=https://nexus.ad.ddupan.top
+TF_VAR_nexus_username=admin
+TF_VAR_nexus_password=<OpenBao kv/infra/nexus 的 admin_password 字段>
+```
+
+`terraform/` 使用 `sonatype-nexus-community/sonatyperepo` 1.17.0，当前声明：
+
+- `ansible-galaxy-proxy` → `https://galaxy.ansible.com`
+- `ansible-public` group
+- `go-proxy` → `https://proxy.golang.org`
+- `go-public` group
+
+provider credential 不写入 HCL 或 tfvars。正式 apply 前还必须为这个独立 Terraform root
+配置远端 backend；本地 state 只允许用于可丢弃的 POC，不提交。验证命令：
+
+```bash
+terraform -chdir=apps/nexus/terraform init -backend=false
+terraform -chdir=apps/nexus/terraform validate
+```
+
+先以 `--check` 查看 OCI repository 漂移，再明确 apply；脚本只从上述环境变量取得凭据，
+用临时 `0600` netrc 调用 REST API，退出时删除：
+
+```bash
+apps/nexus/terraform/reconcile-oci.sh --check
+apps/nexus/terraform/reconcile-oci.sh --apply
+```
+
+Terraform 同时把内置 `anonymous` 用户从默认的全仓库 `nx-anonymous` 角色收窄到
+`ansible-public`、其返回制品 URL 使用的 `ansible-galaxy-proxy`、`go-public`，以及
+`oci-public`/`oci-proxy` 的 `browse/read` 权限。`oci-hosted` 不向匿名用户开放。首次接管
+已有实例时先执行
+`terraform import sonatyperepo_user.anonymous anonymous,default`，再 apply；不要先启用默认的
+全仓库匿名读取。
+
+2026-09-20 的 POC 现场验收已确认：Flux 与 Pod Ready、PVC Bound、HTTPRoute 通过 HTTPS
+返回 Nexus 状态 200，Samba DNS 已幂等收敛。全新客户端目录通过匿名入口下载
+`community.general:11.2.0` 时冷缓存为 8.49 秒、热缓存为 1.89 秒，两次 tarball SHA-256
+一致；`golang.org/x/[email protected]` 为 2.92 秒与 1.51 秒。
+
+## 客户端验收
+
+先验证冷缓存，再原样重复命令验证热缓存；记录 Nexus 请求、上游流量和耗时，不只观察
+命令成功。Ansible 配置的 URL 必须以 `/` 结尾：
+
+```ini
+[galaxy]
+server_list = nexus
+
+[galaxy_server.nexus]
+url = https://nexus.ad.ddupan.top/repository/ansible-public/
+```
+
+```bash
+ansible-galaxy collection install -r collections/requirements.yml \
+  -p .ansible/collections
+```
+
+Go POC 使用：
+
+```bash
+GOPROXY=https://nexus.ad.ddupan.top/repository/go-public/ go mod download
+```
+
+私有 `git.ddupan.top/*` module 的 `GOPRIVATE`、凭据与是否允许 `direct` fallback 在实际
+workflow 中单独决定；不要让私有 module path 意外发往公共 proxy。
+
+OCI 使用 path-based routing：匿名公共拉取地址形如
+`nexus.ad.ddupan.top/oci-public/library/alpine:3.22`，认证写入地址形如
+`nexus.ad.ddupan.top/oci-hosted/<namespace>/<image>:<tag>`。2026-09-20 现场验收结果：
+
+- `oci-public` 匿名代理拉取 Alpine 冷缓存 4.75 秒、热缓存 0.80 秒，digest 一致；
+- `oci-hosted` 认证 push/pull 成功，匿名 pull 返回 401；
+- amd64/arm64 OCI image index push 成功，两个平台 manifest 可见；
+- Helm chart push/pull digest 与本地 tarball SHA-256 一致；
+- Cosign 3.1.3 使用一次性密钥签名并验证成功，OCI 1.1 referrers API 返回一个
+  `application/vnd.dev.sigstore.bundle.v0.3+json` artifact；
+- BuildKit `registry` cache 以 `mode=max` 导出成功，销毁首个 builder 后由新 builder 导入，
+  两个 `RUN` step 均明确命中 `CACHED`。
+
+本机安装的测试客户端包括 `/usr/local/bin/cosign` 3.1.3；安装时核对官方 Linux amd64
+binary SHA-256 `4629c757b7618056f8ddd7e2625ae9fdd94c0372a65049520bc7d9df9efc7f71`。
+上述结果仍不代表备份恢复、外部 PostgreSQL 或正式 publisher service account 已完成；
+这些项目通过前不得迁移或删除 zot。
+
+## 数据与恢复
+
+POC 的数据库、配置、blob、初始管理员状态都位于 `nexus-data` PVC。删除 Deployment
+不会删除 PVC；删除 PVC 会永久删除整个 POC。当前没有独立备份，不能将它用于唯一副本的
+正式制品。
+
+恢复验证至少包括：停止写入、取得一致备份、在独立 PVC/实例恢复、登录、列出 repository、
+拉取已缓存的 Ansible/Go 制品，并核对 OCI digest/referrers。正式化时再把数据库迁移至
+外部 PostgreSQL，并分别定义数据库与 blob 的备份、恢复顺序和 RPO。
+
+参考：
+
+- [Nexus OCI repositories](https://help.sonatype.com/en/oci-repositories.html)
+- [Nexus Ansible repositories](https://help.sonatype.com/en/ansible-repositories.html)
+- [Nexus Go repositories](https://help.sonatype.com/en/go-repositories.html)
+- [官方容器镜像](https://hub.docker.com/r/sonatype/nexus3)

apps/nexus/deployment.yaml

@@ -0,0 +1,82 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: nexus
+  namespace: nexus
+  labels:
+    app.kubernetes.io/name: nexus
+spec:
+  replicas: 1
+  strategy:
+    type: Recreate
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: nexus
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: nexus
+    spec:
+      automountServiceAccountToken: false
+      securityContext:
+        fsGroup: 200
+        fsGroupChangePolicy: OnRootMismatch
+        runAsGroup: 200
+        runAsNonRoot: true
+        runAsUser: 200
+        seccompProfile:
+          type: RuntimeDefault
+      terminationGracePeriodSeconds: 120
+      containers:
+        - name: nexus
+          image: docker.io/sonatype/nexus3:3.96.1@sha256:56142f13432cf072e017aebb2025f201e42ae36ff40bb82618c702504c61f7dd
+          imagePullPolicy: IfNotPresent
+          env:
+            - name: INSTALL4J_ADD_VM_PARAMS
+              value: >-
+                -Xms1024m -Xmx2048m -XX:MaxDirectMemorySize=1024m
+                -Djava.util.prefs.userRoot=/nexus-data/javaprefs
+          ports:
+            - name: http
+              containerPort: 8081
+              protocol: TCP
+          resources:
+            requests:
+              cpu: 250m
+              memory: 2Gi
+            limits:
+              cpu: "2"
+              memory: 4Gi
+          securityContext:
+            allowPrivilegeEscalation: false
+            capabilities:
+              drop:
+                - ALL
+          startupProbe:
+            httpGet:
+              path: /service/rest/v1/status
+              port: http
+            failureThreshold: 60
+            periodSeconds: 10
+            timeoutSeconds: 5
+          readinessProbe:
+            httpGet:
+              path: /service/rest/v1/status
+              port: http
+            failureThreshold: 6
+            periodSeconds: 10
+            timeoutSeconds: 5
+          livenessProbe:
+            httpGet:
+              path: /service/rest/v1/status
+              port: http
+            failureThreshold: 6
+            periodSeconds: 30
+            timeoutSeconds: 5
+          volumeMounts:
+            - name: data
+              mountPath: /nexus-data
+      volumes:
+        - name: data
+          persistentVolumeClaim:
+            claimName: nexus-data

apps/nexus/httproute.yaml

@@ -0,0 +1,16 @@
+apiVersion: gateway.networking.k8s.io/v1
+kind: HTTPRoute
+metadata:
+  name: nexus
+  namespace: nexus
+spec:
+  parentRefs:
+    - name: eg
+      namespace: envoy-gateway-system
+      sectionName: https
+  hostnames:
+    - nexus.ad.ddupan.top
+  rules:
+    - backendRefs:
+        - name: nexus
+          port: 8081

apps/nexus/kustomization.yaml

@@ -0,0 +1,9 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - namespace.yaml
+  - pvc.yaml
+  - deployment.yaml
+  - service.yaml
+  - httproute.yaml
+  - networkpolicy.yaml

apps/nexus/namespace.yaml

@@ -0,0 +1,8 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: nexus
+  labels:
+    pod-security.kubernetes.io/enforce: restricted
+    pod-security.kubernetes.io/audit: restricted
+    pod-security.kubernetes.io/warn: restricted

apps/nexus/networkpolicy.yaml

@@ -0,0 +1,23 @@
+apiVersion: networking.k8s.io/v1
+kind: NetworkPolicy
+metadata:
+  name: nexus-ingress
+  namespace: nexus
+spec:
+  podSelector:
+    matchLabels:
+      app.kubernetes.io/name: nexus
+  policyTypes:
+    - Ingress
+  ingress:
+    - from:
+        - namespaceSelector:
+            matchLabels:
+              kubernetes.io/metadata.name: envoy-gateway-system
+          podSelector:
+            matchLabels:
+              gateway.envoyproxy.io/owning-gateway-name: eg
+              gateway.envoyproxy.io/owning-gateway-namespace: envoy-gateway-system
+      ports:
+        - protocol: TCP
+          port: 8081

apps/nexus/pvc.yaml

@@ -0,0 +1,12 @@
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: nexus-data
+  namespace: nexus
+spec:
+  accessModes:
+    - ReadWriteOnce
+  resources:
+    requests:
+      storage: 50Gi
+  storageClassName: localpv-zfs-ceph

apps/nexus/service.yaml

@@ -0,0 +1,14 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: nexus
+  namespace: nexus
+spec:
+  type: ClusterIP
+  selector:
+    app.kubernetes.io/name: nexus
+  ports:
+    - name: http
+      port: 8081
+      protocol: TCP
+      targetPort: http

apps/nexus/terraform/.gitignore

@@ -0,0 +1,6 @@
+.terraform/
+*.tfstate
+*.tfstate.*
+*.tfplan
+crash.log
+crash.*.log

apps/nexus/terraform/.terraform.lock.hcl

@@ -0,0 +1,24 @@
+# This file is maintained automatically by "terraform init".
+# Manual edits may be lost in future updates.
+
+provider "registry.terraform.io/sonatype-nexus-community/sonatyperepo" {
+  version     = "1.17.0"
+  constraints = "1.17.0"
+  hashes = [
+    "h1:uKhvhhhI7B+HBsh0zq/ybqKt+EnOGyI6rjcRCtj79ZA=",
+    "zh:0dde99e7b343fa01f8eefc378171fb8621bedb20f59157d6cc8e3d46c738105f",
+    "zh:3315929df254a3a6ac27c8c846c2006f7d2a91fadc014351bc4d617f948e5bf9",
+    "zh:36be5a455af3ce4e187de26753de63e78c1ee9a32dba0135c6cf96a6c1fff25f",
+    "zh:3f73f7ff57b8c339a7c7ac37653e2dc0b2dd9dcc3f3a538788e7e3ac838337b2",
+    "zh:40286ecca4c22ab7ae90618ac6d2743f5055199dac81cf5204a4a397c784d439",
+    "zh:4d24e5c0195fb3155b1967583ee64cfeda402d7cc7f3c73369438f6c69f4245b",
+    "zh:828a9d7aceaac36af7f9c07af43ec8d20a89148780645d170ffb1c68b2da792d",
+    "zh:a5ab04de3fe626ec57c832618c6f990abd6610f81e132621651e0b180b970cff",
+    "zh:a959fa6090a8c0f53739879184e7346423494aee598003df0d1ab4a22b2eee91",
+    "zh:bdda26c2f03f918bbe59e75abea44868fafda019c3a543725331195df126350b",
+    "zh:d8048e149ee97ba62971e6a79355d59887bc6d10fcf72cc2feff3d0a2582670c",
+    "zh:dd36f9988af4e1ca5b1ca7b7bb6f658df9a220dfcda7fec7392fedfe9064f652",
+    "zh:dda2688d46c7e539fe97e8fe9d3ec81fb364170e018d9c6a681364c8955d4e9d",
+    "zh:e6b519afe2dea1c0434f766eb6bc9ba78cc5b6ef2c311c2ca3c65cb24744f31f",
+  ]
+}

apps/nexus/terraform/oci/oci-hosted.json

@@ -0,0 +1,17 @@
+{
+  "name": "oci-hosted",
+  "online": true,
+  "storage": {
+    "blobStoreName": "default",
+    "strictContentTypeValidation": true,
+    "writePolicy": "ALLOW",
+    "latestPolicy": false
+  },
+  "oci": {
+    "v1Enabled": false,
+    "forceBasicAuth": false,
+    "pathEnabled": true
+  },
+  "component": { "proprietaryComponents": false },
+  "cosign": { "enforcement": "NONE" }
+}

apps/nexus/terraform/oci/oci-proxy.json

@@ -0,0 +1,27 @@
+{
+  "name": "oci-proxy",
+  "online": true,
+  "storage": {
+    "blobStoreName": "default",
+    "strictContentTypeValidation": true
+  },
+  "oci": {
+    "v1Enabled": false,
+    "forceBasicAuth": false,
+    "pathEnabled": true
+  },
+  "ociProxy": {
+    "indexType": "HUB",
+    "cacheForeignLayers": false,
+    "foreignLayerUrlWhitelist": []
+  },
+  "proxy": {
+    "remoteUrl": "https://registry-1.docker.io",
+    "contentMaxAge": 1440,
+    "metadataMaxAge": 60,
+    "preserveEncodedCharacters": false
+  },
+  "negativeCache": { "enabled": true, "timeToLive": 60 },
+  "httpClient": { "blocked": false, "autoBlock": true },
+  "cosign": { "enforcement": "NONE" }
+}

apps/nexus/terraform/oci/oci-public.json

@@ -0,0 +1,15 @@
+{
+  "name": "oci-public",
+  "online": true,
+  "storage": {
+    "blobStoreName": "default",
+    "strictContentTypeValidation": true
+  },
+  "group": { "memberNames": ["oci-proxy"] },
+  "oci": {
+    "v1Enabled": false,
+    "forceBasicAuth": false,
+    "pathEnabled": true
+  },
+  "cosign": { "enforcement": "NONE" }
+}

apps/nexus/terraform/reconcile-oci.sh

@@ -0,0 +1,80 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+mode="${1:---check}"
+case "$mode" in
+  --check | --apply) ;;
+  *) echo "usage: $0 [--check|--apply]" >&2; exit 2 ;;
+esac
+
+: "${TF_VAR_nexus_url:?set TF_VAR_nexus_url}"
+: "${TF_VAR_nexus_username:?set TF_VAR_nexus_username}"
+: "${TF_VAR_nexus_password:?set TF_VAR_nexus_password}"
+
+script_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"
+auth_file="$(mktemp /tmp/nexus-oci-auth.XXXXXX)"
+trap 'rm -f -- "$auth_file"' EXIT
+chmod 0600 "$auth_file"
+printf 'machine %s\nlogin %s\npassword %s\n' \
+  "${TF_VAR_nexus_url#*://}" "$TF_VAR_nexus_username" \
+  "$TF_VAR_nexus_password" >"$auth_file"
+
+drift=0
+for entry in \
+  "hosted:$script_dir/oci/oci-hosted.json" \
+  "proxy:$script_dir/oci/oci-proxy.json" \
+  "group:$script_dir/oci/oci-public.json"; do
+  repository_type="${entry%%:*}"
+  desired_file="${entry#*:}"
+  repository_name="$(jq -er '.name' "$desired_file")"
+  endpoint="$TF_VAR_nexus_url/service/rest/v1/repositories/oci/$repository_type"
+  current_file="$(mktemp /tmp/nexus-oci-current.XXXXXX)"
+
+  status="$(curl --silent --show-error --netrc-file "$auth_file" \
+    --output "$current_file" --write-out '%{http_code}' \
+    "$endpoint/$repository_name")"
+
+  if [[ "$status" == 404 ]]; then
+    drift=1
+    if [[ "$mode" == --apply ]]; then
+      curl --fail --silent --show-error --netrc-file "$auth_file" \
+        --header 'Content-Type: application/json' \
+        --data-binary "@$desired_file" "$endpoint"
+      echo "created $repository_name"
+    else
+      echo "missing $repository_name" >&2
+    fi
+  elif [[ "$status" == 200 ]]; then
+    if jq -e --slurpfile desired "$desired_file" '
+      def subset($actual; $wanted):
+        if ($wanted | type) == "object" then
+          all($wanted | keys[];
+            ($actual[.] != null) and subset($actual[.]; $wanted[.]))
+        else
+          $actual == $wanted
+        end;
+      subset(.; $desired[0])
+    ' "$current_file" >/dev/null; then
+      echo "in sync $repository_name"
+    else
+      drift=1
+      if [[ "$mode" == --apply ]]; then
+        curl --fail --silent --show-error --netrc-file "$auth_file" \
+          --request PUT --header 'Content-Type: application/json' \
+          --data-binary "@$desired_file" "$endpoint/$repository_name"
+        echo "updated $repository_name"
+      else
+        echo "drifted $repository_name" >&2
+      fi
+    fi
+  else
+    cat "$current_file" >&2
+    echo "unexpected HTTP $status for $repository_name" >&2
+    exit 1
+  fi
+  rm -f -- "$current_file"
+done
+
+if [[ "$mode" == --check && "$drift" -ne 0 ]]; then
+  exit 1
+fi

apps/nexus/terraform/repositories.tf

@@ -0,0 +1,64 @@
+locals {
+  proxy_http_client = {
+    auto_block = true
+    blocked    = false
+  }
+
+  proxy_negative_cache = {
+    enabled      = true
+    time_to_live = 60
+  }
+
+  repository_storage = {
+    blob_store_name                = "default"
+    strict_content_type_validation = true
+  }
+}
+
+resource "sonatyperepo_repository_ansiblegalaxy_proxy" "galaxy" {
+  name   = "ansible-galaxy-proxy"
+  online = true
+
+  http_client    = local.proxy_http_client
+  negative_cache = local.proxy_negative_cache
+  proxy = {
+    remote_url       = "https://galaxy.ansible.com"
+    content_max_age  = 1440
+    metadata_max_age = 60
+  }
+  storage = local.repository_storage
+}
+
+resource "sonatyperepo_repository_ansiblegalaxy_group" "public" {
+  name   = "ansible-public"
+  online = true
+
+  group = {
+    member_names = [sonatyperepo_repository_ansiblegalaxy_proxy.galaxy.name]
+  }
+  storage = local.repository_storage
+}
+
+resource "sonatyperepo_repository_go_proxy" "public" {
+  name   = "go-proxy"
+  online = true
+
+  http_client    = local.proxy_http_client
+  negative_cache = local.proxy_negative_cache
+  proxy = {
+    remote_url       = "https://proxy.golang.org"
+    content_max_age  = 1440
+    metadata_max_age = 60
+  }
+  storage = local.repository_storage
+}
+
+resource "sonatyperepo_repository_go_group" "public" {
+  name   = "go-public"
+  online = true
+
+  group = {
+    member_names = [sonatyperepo_repository_go_proxy.public.name]
+  }
+  storage = local.repository_storage
+}

apps/nexus/terraform/security.tf

@@ -0,0 +1,75 @@
+resource "sonatyperepo_privilege_repository_view" "anonymous_ansible" {
+  name        = "ci-anonymous-ansible-read"
+  description = "Anonymous read access to the Ansible Galaxy group"
+  actions     = ["BROWSE", "READ"]
+  format      = "ansiblegalaxy"
+  repository  = sonatyperepo_repository_ansiblegalaxy_group.public.name
+}
+
+resource "sonatyperepo_privilege_repository_view" "anonymous_ansible_proxy" {
+  name        = "ci-anonymous-ansible-proxy-read"
+  description = "Anonymous artifact read access to the Ansible Galaxy proxy"
+  actions     = ["BROWSE", "READ"]
+  format      = "ansiblegalaxy"
+  repository  = sonatyperepo_repository_ansiblegalaxy_proxy.galaxy.name
+}
+
+resource "sonatyperepo_privilege_repository_view" "anonymous_go" {
+  name        = "ci-anonymous-go-read"
+  description = "Anonymous read access to the Go module group"
+  actions     = ["BROWSE", "READ"]
+  format      = "go"
+  repository  = sonatyperepo_repository_go_group.public.name
+}
+
+resource "sonatyperepo_privilege_repository_view" "anonymous_oci_public" {
+  name        = "ci-anonymous-oci-public-read"
+  description = "Anonymous read access to the public OCI group"
+  actions     = ["BROWSE", "READ"]
+  format      = "oci"
+  repository  = "oci-public"
+}
+
+resource "sonatyperepo_privilege_repository_view" "anonymous_oci_proxy" {
+  name        = "ci-anonymous-oci-proxy-read"
+  description = "Anonymous read access to the OCI proxy member"
+  actions     = ["BROWSE", "READ"]
+  format      = "oci"
+  repository  = "oci-proxy"
+}
+
+resource "sonatyperepo_role" "anonymous_ci" {
+  id          = "ci-anonymous-read"
+  name        = "CI anonymous read"
+  description = "Read-only access to public CI dependency proxy groups"
+  privileges = [
+    sonatyperepo_privilege_repository_view.anonymous_ansible.name,
+    sonatyperepo_privilege_repository_view.anonymous_ansible_proxy.name,
+    sonatyperepo_privilege_repository_view.anonymous_go.name,
+    sonatyperepo_privilege_repository_view.anonymous_oci_public.name,
+    sonatyperepo_privilege_repository_view.anonymous_oci_proxy.name,
+  ]
+  roles = []
+}
+
+resource "sonatyperepo_user" "anonymous" {
+  user_id       = "anonymous"
+  first_name    = "Anonymous"
+  last_name     = "User"
+  email_address = "[email protected]"
+  status        = "active"
+  roles         = [sonatyperepo_role.anonymous_ci.id]
+}
+
+resource "sonatyperepo_system_anonymous_access" "ci" {
+  enabled    = true
+  user_id    = sonatyperepo_user.anonymous.user_id
+  realm_name = "NexusAuthorizingRealm"
+}
+
+resource "sonatyperepo_security_realms" "active" {
+  active = [
+    "NexusAuthenticatingRealm",
+    "OciBearerToken",
+  ]
+}

apps/nexus/terraform/versions.tf

@@ -0,0 +1,33 @@
+terraform {
+  required_version = ">= 1.11.0"
+
+  required_providers {
+    sonatyperepo = {
+      source  = "sonatype-nexus-community/sonatyperepo"
+      version = "1.17.0"
+    }
+  }
+}
+
+provider "sonatyperepo" {
+  url      = var.nexus_url
+  username = var.nexus_username
+  password = var.nexus_password
+}
+
+variable "nexus_url" {
+  description = "Nexus Repository base URL"
+  type        = string
+}
+
+variable "nexus_username" {
+  description = "Nexus Terraform management username"
+  type        = string
+  sensitive   = true
+}
+
+variable "nexus_password" {
+  description = "Nexus Terraform management password"
+  type        = string
+  sensitive   = true
+}

apps/ps3netsrv/docker-compose.yml

@@ -12,3 +12,10 @@ services:
       - "38008:38008"
     volumes:
       - "/mnt/pool/games/ps3:/games:rw"
+
+# 避免与 DN42 的 172.20.0.0/14 重叠。
+networks:
+  default:
+    ipam:
+      config:
+        - subnet: 172.28.1.0/24

apps/victoriametrics/alertmanager.yaml

@@ -1,5 +0,0 @@
-route:
-  receiver: blackhole
-
-receivers:
-  - name: blackhole

apps/victoriametrics/compose.yaml

@@ -1,94 +0,0 @@
-services:
-  #  Metrics collector.
-  #  It scrapes targets defined in --promscrape.config
-  #  And forward them to --remoteWrite.url
-  vmagent:
-    image: victoriametrics/vmagent:v1.132.0
-    depends_on:
-      - "victoriametrics"
-    ports:
-      - 8429:8429
-    volumes:
-      - vmagentdata:/vmagentdata
-      - ./prometheus.yaml:/etc/prometheus/prometheus.yml
-    command:
-      - "--promscrape.config=/etc/prometheus/prometheus.yml"
-      - "--remoteWrite.url=http://victoriametrics:8428/api/v1/write"
-    restart: always
-  # VictoriaMetrics instance, a single process responsible for
-  # storing metrics and serve read requests.
-  victoriametrics:
-    image: victoriametrics/victoria-metrics:v1.132.0
-    ports:
-      - 8428:8428
-      - 8089:8089
-      - 8089:8089/udp
-      - 2003:2003
-      - 2003:2003/udp
-      - 4242:4242
-    volumes:
-      - vmdata:/storage
-    command:
-      - "--storageDataPath=/storage"
-      - "--graphiteListenAddr=:2003"
-      - "--opentsdbListenAddr=:4242"
-      - "--httpListenAddr=:8428"
-      - "--influxListenAddr=:8089"
-      - "--vmalert.proxyURL=http://vmalert:8880"
-    restart: always
-
-  grafana:
-    image: grafana/grafana:12.2.0
-    depends_on:
-      - "victoriametrics"
-    ports:
-      - 3000:3000
-    volumes:
-      - grafanadata:/var/lib/grafana
-      - ./provisioning/datasources/prometheus-datasource/single.yml:/etc/grafana/provisioning/datasources/single.yml
-      - ./provisioning/dashboards:/etc/grafana/provisioning/dashboards
-      - ./provisioning/dashboards/victoriametrics.json:/var/lib/grafana/dashboards/vm.json
-      - ./provisioning/dashboards/vmagent.json:/var/lib/grafana/dashboards/vmagent.json
-      - ./provisioning/dashboards/vmalert.json:/var/lib/grafana/dashboards/vmalert.json
-    restart: always
-
-  # vmalert executes alerting and recording rules
-  vmalert:
-    image: victoriametrics/vmalert:v1.132.0
-    depends_on:
-      - "victoriametrics"
-      - "alertmanager"
-    ports:
-      - 8880:8880
-    volumes:
-      - ./rules/alerts.yml:/etc/alerts/alerts.yml
-      - ./rules/alerts-health.yml:/etc/alerts/alerts-health.yml
-      - ./rules/alerts-vmagent.yml:/etc/alerts/alerts-vmagent.yml
-      - ./rules/alerts-vmalert.yml:/etc/alerts/alerts-vmalert.yml
-    command:
-      - "--datasource.url=http://victoriametrics:8428/"
-      - "--remoteRead.url=http://victoriametrics:8428/"
-      - "--remoteWrite.url=http://vmagent:8429/"
-      - "--notifier.url=http://alertmanager:9093/"
-      - "--rule=/etc/alerts/*.yml"
-      # display source of alerts in grafana
-      - "--external.url=http://127.0.0.1:3000" #grafana outside container
-      - '--external.alert.source=explore?orgId=1&left={"datasource":"VictoriaMetrics","queries":[{"expr":{{.Expr|jsonEscape|queryEscape}},"refId":"A"}],"range":{"from":"{{ .ActiveAt.UnixMilli }}","to":"now"}}'
-    restart: always
-
-  # alertmanager receives alerting notifications from vmalert
-  # and distributes them according to --config.file.
-  alertmanager:
-    image: prom/alertmanager:v0.28.1
-    volumes:
-      - ./alertmanager.yaml:/config/alertmanager.yml
-    command:
-      - "--config.file=/config/alertmanager.yml"
-    ports:
-      - 9093:9093
-    restart: always
-
-volumes:
-  vmagentdata: {}
-  vmdata: {}
-  grafanadata: {}

apps/victoriametrics/prometheus.yaml

@@ -1,16 +0,0 @@
-global:
-  scrape_interval: 10s
-
-scrape_configs:
-- job_name: vmagent
-  static_configs:
-  - targets:
-    - vmagent:8429
-- job_name: vmalert
-  static_configs:
-  - targets:
-    - vmalert:8880
-- job_name: victoriametrics
-  static_configs:
-  - targets:
-    - victoriametrics:8428

apps/victoriametrics/provisioning/dashboards/dashboard.yml

@@ -1,9 +0,0 @@
-apiVersion: 1
-
-providers:
-- name: Prometheus
-  orgId: 1
-  folder: ''
-  type: file
-  options:
-    path: /var/lib/grafana/dashboards

apps/victoriametrics/provisioning/dashboards/datasources/prometheus-datasource/single.yml

@@ -1,11 +0,0 @@
-apiVersion: 1
-
-datasources:
-    - name: VictoriaMetrics
-      type: prometheus
-      access: proxy
-      url: http://victoriametrics:8428
-      isDefault: true
-      jsonData:
-        prometheusType: Prometheus
-        prometheusVersion: 2.24.0

apps/victoriametrics/provisioning/datasources/prometheus-datasource/single.yml

@@ -1,11 +0,0 @@
-apiVersion: 1
-
-datasources:
-    - name: VictoriaMetrics
-      type: prometheus
-      access: proxy
-      url: http://victoriametrics:8428
-      isDefault: true
-      jsonData:
-        prometheusType: Prometheus
-        prometheusVersion: 2.24.0

apps/victoriametrics/rules/alerts-health.yml

@@ -1,149 +0,0 @@
-# File contains default list of alerts for various VM components.
-# The following alerts are recommended for use for any VM installation.
-# The alerts below are just recommendations and may require some updates
-# and threshold calibration according to every specific setup.
-groups:
-  - name: vm-health
-    # note the `job` filter and update accordingly to your setup
-    rules:
-      - alert: TooManyRestarts
-        expr: changes(process_start_time_seconds{job=~".*(victoriametrics|vmselect|vminsert|vmstorage|vmagent|vmalert|vmsingle|vmalertmanager|vmauth).*"}[15m]) > 2
-        labels:
-          severity: critical
-        annotations:
-          summary: "{{ $labels.job }} too many restarts (instance {{ $labels.instance }})"
-          description: >
-            Job {{ $labels.job }} (instance {{ $labels.instance }}) has restarted more than twice in the last 15 minutes.
-            It might be crashlooping.
-
-      - alert: ServiceDown
-        expr: up{job=~".*(victoriametrics|vmselect|vminsert|vmstorage|vmagent|vmalert|vmsingle|vmalertmanager|vmauth).*"} == 0
-        for: 2m
-        labels:
-          severity: critical
-        annotations:
-          summary: "Service {{ $labels.job }} is down on {{ $labels.instance }}"
-          description: "{{ $labels.instance }} of job {{ $labels.job }} has been down for more than 2 minutes."
-
-      - alert: ProcessNearFDLimits
-        expr: (process_max_fds - process_open_fds) < 100
-        for: 5m
-        labels:
-          severity: critical
-        annotations:
-          summary: "Number of free file descriptors is less than 100 for \"{{ $labels.job }}\"(\"{{ $labels.instance }}\") for the last 5m"
-          description: | 
-            Exhausting OS file descriptors limit can cause severe degradation of the process.
-            Consider to increase the limit as fast as possible.
-
-      - alert: TooHighMemoryUsage
-        expr: (min_over_time(process_resident_memory_anon_bytes[10m]) / vm_available_memory_bytes) > 0.8
-        for: 5m
-        labels:
-          severity: critical
-        annotations:
-          summary: "It is more than 80% of memory used by \"{{ $labels.job }}\"(\"{{ $labels.instance }}\")"
-          description: |
-            Too high memory usage may result into multiple issues such as OOMs or degraded performance.
-            Consider to either increase available memory or decrease the load on the process.
-
-      - alert: TooHighCPUUsage
-        expr: rate(process_cpu_seconds_total[5m]) / process_cpu_cores_available > 0.9
-        for: 5m
-        labels:
-          severity: critical
-        annotations:
-          summary: "More than 90% of CPU is used by \"{{ $labels.job }}\"(\"{{ $labels.instance }}\") during the last 5m"
-          description: >
-            Too high CPU usage may be a sign of insufficient resources and make process unstable.
-            Consider to either increase available CPU resources or decrease the load on the process.
-
-      - alert: TooHighGoroutineSchedulingLatency
-        expr: histogram_quantile(0.99, sum(rate(go_sched_latencies_seconds_bucket{job=~".*(victoriametrics|vmselect|vminsert|vmstorage|vmagent|vmalert|vmsingle|vmalertmanager|vmauth).*"}[5m])) by (le, job, instance)) > 0.1
-        for: 15m
-        labels:
-          severity: critical
-        annotations:
-          summary: "\"{{ $labels.job }}\"(\"{{ $labels.instance }}\") has insufficient CPU resources for >15m"
-          description: >
-            Go runtime is unable to schedule goroutines execution in acceptable time. This is usually a sign of
-            insufficient CPU resources or CPU throttling. Verify that service has enough CPU resources. Otherwise,
-            the service could work unreliably with delays in processing.
-
-      - alert: TooManyLogs
-        expr: sum(increase(vm_log_messages_total{level="error"}[5m])) without (app_version, location) > 0
-        for: 15m
-        labels:
-          severity: warning
-        annotations:
-          summary: "Too many logs printed for job \"{{ $labels.job }}\" ({{ $labels.instance }})"
-          description: >
-            Logging rate for job \"{{ $labels.job }}\" ({{ $labels.instance }}) is {{ $value }} for last 15m.
-            Worth to check logs for specific error messages.
-
-      - alert: TooManyTSIDMisses
-        expr: increase(vm_missing_tsids_for_metric_id_total[5m]) > 0
-        for: 15m
-        labels:
-          severity: critical
-        annotations:
-          summary: "Unexpected TSID misses for job \"{{ $labels.job }}\" ({{ $labels.instance }}) for the last 15 minutes"
-          description: |
-            Unexpected TSID misses for \"{{ $labels.job }}\" ({{ $labels.instance }}) for the last 15 minutes.
-            If this happens after unclean shutdown of VictoriaMetrics process (via \"kill -9\", OOM or power off),
-            then this is OK - the alert must go away in a few minutes after the restart.
-            Otherwise this may point to the corruption of index data.
-
-      - alert: ConcurrentInsertsHitTheLimit
-        expr: avg_over_time(vm_concurrent_insert_current[1m]) >= vm_concurrent_insert_capacity
-        for: 15m
-        labels:
-          severity: warning
-        annotations:
-          summary: "{{ $labels.job }} on instance {{ $labels.instance }} is constantly hitting concurrent inserts limit"
-          description: | 
-            The limit of concurrent inserts on instance {{ $labels.instance }} depends on the number of CPUs.
-            Usually, when component constantly hits the limit it is likely the component is overloaded and requires more CPU.
-            In some cases for components like vmagent or vminsert the alert might trigger if there are too many clients
-            making write attempts. If vmagent's or vminsert's CPU usage and network saturation are at normal level, then 
-            it might be worth adjusting `-maxConcurrentInserts` cmd-line flag.
-
-      - alert: IndexDBRecordsDrop
-        expr: increase(vm_indexdb_items_dropped_total[5m]) > 0
-        labels:
-          severity: critical
-        annotations:
-          summary: "IndexDB skipped registering items during data ingestion with reason={{ $labels.reason }}."
-          description: | 
-            VictoriaMetrics could skip registering new timeseries during ingestion if they fail the validation process. 
-            For example, `reason=too_long_item` means that time series cannot exceed 64KB. Please, reduce the number 
-            of labels or label values for such series. Or enforce these limits via `-maxLabelsPerTimeseries` and 
-            `-maxLabelValueLen` command-line flags.
-
-      - alert: RowsRejectedOnIngestion
-        expr: rate(vm_rows_ignored_total[5m]) > 0
-        for: 15m
-        labels:
-          severity: warning
-        annotations:
-          summary: "Some rows are rejected on \"{{ $labels.instance }}\" on ingestion attempt"
-          description: "Ingested rows on instance \"{{ $labels.instance }}\" are rejected due to the
-            following reason: \"{{ $labels.reason }}\""
-
-      - alert: TooHighQueryLoad
-        expr: increase(vm_concurrent_select_limit_timeout_total[5m]) > 0
-        for: 15m
-        labels:
-          severity: warning
-        annotations:
-          summary: "Read queries fail with timeout for {{ $labels.job }} on instance {{ $labels.instance }}"
-          description: |
-            Instance {{ $labels.instance }} ({{ $labels.job }}) is failing to serve read queries during last 15m.
-            Concurrency limit `-search.maxConcurrentRequests` was reached on this instance and extra queries were
-            put into the queue for `-search.maxQueueDuration` interval. But even after waiting in the queue these queries weren't served.
-            This happens if instance is overloaded with the current workload, or datasource is too slow to respond.
-            Possible solutions are the following:
-            * reduce the query load;
-            * increase compute resources or number of replicas;
-            * adjust limits `-search.maxConcurrentRequests` and `-search.maxQueueDuration`.
-            See more at https://docs.victoriametrics.com/victoriametrics/troubleshooting/#slow-queries

apps/victoriametrics/rules/alerts-vmagent.yml

@@ -1,172 +0,0 @@
-# File contains default list of alerts for vmagent service.
-# The alerts below are just recommendations and may require some updates
-# and threshold calibration according to every specific setup.
-groups:
-  # Alerts group for vmagent assumes that Grafana dashboard
-  # https://grafana.com/grafana/dashboards/12683 is installed.
-  # Pls update the `dashboard` annotation according to your setup.
-  - name: vmagent
-    interval: 30s
-    concurrency: 2
-    rules:
-      - alert: PersistentQueueIsDroppingData
-        expr: sum(increase(vm_persistentqueue_bytes_dropped_total[5m])) without (path) > 0
-        for: 10m
-        labels:
-          severity: critical
-        annotations:
-          dashboard: "http://localhost:3000/d/G7Z9GzMGz?viewPanel=49&var-instance={{ $labels.instance }}"
-          summary: "Instance {{ $labels.instance }} is dropping data from persistent queue"
-          description: "Vmagent dropped {{ $value | humanize1024 }} from persistent queue
-              on instance {{ $labels.instance }} for the last 10m."
-
-      - alert: RejectedRemoteWriteDataBlocksAreDropped
-        expr: sum(increase(vmagent_remotewrite_packets_dropped_total[5m])) without (url) > 0
-        for: 15m
-        labels:
-          severity: warning
-        annotations:
-          dashboard: "http://localhost:3000/d/G7Z9GzMGz?viewPanel=79&var-instance={{ $labels.instance }}"
-          summary: "Vmagent is dropping data blocks that are rejected by remote storage"
-          description: "Job \"{{ $labels.job }}\" on instance {{ $labels.instance }} drops the rejected by 
-            remote-write server data blocks. Check the logs to find the reason for rejects."
-
-      - alert: TooManyScrapeErrors
-        expr: increase(vm_promscrape_scrapes_failed_total[5m]) > 0
-        for: 15m
-        labels:
-          severity: warning
-        annotations:
-          dashboard: "http://localhost:3000/d/G7Z9GzMGz?viewPanel=31&var-instance={{ $labels.instance }}"
-          summary: "Vmagent fails to scrape one or more targets"
-          description: "Job \"{{ $labels.job }}\" on instance {{ $labels.instance }} fails to scrape targets for last 15m"
-
-      - alert: ScrapePoolHasNoTargets
-        expr: sum(vm_promscrape_scrape_pool_targets) without (status, instance, pod) == 0
-        for: 30m
-        labels:
-          severity: warning
-        annotations:
-          summary: "Vmagent has scrape_pool with 0 configured/discovered targets"
-          description: "Vmagent \"{{ $labels.job }}\" has scrape_pool \"{{ $labels.scrape_job }}\"
-            with 0 discovered targets. It is likely a misconfiguration. Please follow https://docs.victoriametrics.com/victoriametrics/vmagent/#debugging-scrape-targets
-            to troubleshoot the scraping config."
-
-      - alert: TooManyWriteErrors
-        expr: |
-          (sum(increase(vm_ingestserver_request_errors_total[5m])) without (name,net,type)
-          +
-          sum(increase(vmagent_http_request_errors_total[5m])) without (path,protocol)) > 0
-        for: 15m
-        labels:
-          severity: warning
-        annotations:
-          dashboard: "http://localhost:3000/d/G7Z9GzMGz?viewPanel=77&var-instance={{ $labels.instance }}"
-          summary: "Vmagent responds with too many errors on data ingestion protocols"
-          description: "Job \"{{ $labels.job }}\" on instance {{ $labels.instance }} responds with errors to write requests for last 15m."
-
-      - alert: TooManyRemoteWriteErrors
-        expr: rate(vmagent_remotewrite_retries_count_total[5m]) > 0
-        for: 15m
-        labels:
-          severity: warning
-        annotations:
-          dashboard: "http://localhost:3000/d/G7Z9GzMGz?viewPanel=61&var-instance={{ $labels.instance }}"
-          summary: "Job \"{{ $labels.job }}\" on instance {{ $labels.instance }} fails to push to remote storage"
-          description: "Vmagent fails to push data via remote write protocol to destination \"{{ $labels.url }}\"\n
-            Ensure that destination is up and reachable."
-
-      - alert: RemoteWriteConnectionIsSaturated
-        expr: |
-          (
-           rate(vmagent_remotewrite_send_duration_seconds_total[5m])
-           / 
-           vmagent_remotewrite_queues
-          ) > 0.9
-        for: 15m
-        labels:
-          severity: warning
-        annotations:
-          dashboard: "http://localhost:3000/d/G7Z9GzMGz?viewPanel=84&var-instance={{ $labels.instance }}"
-          summary: "Remote write connection from \"{{ $labels.job }}\" (instance {{ $labels.instance }}) to {{ $labels.url }} is saturated"
-          description: "The remote write connection between vmagent \"{{ $labels.job }}\" (instance {{ $labels.instance }}) and destination \"{{ $labels.url }}\"
-            is saturated by more than 90% and vmagent won't be able to keep up.\n
-            There could be the following reasons for this:\n
-             * vmagent can't send data fast enough through the existing network connections. Increase `-remoteWrite.queues` cmd-line flag value to establish more connections per destination.\n
-             * remote destination can't accept data fast enough. Check if remote destination has enough resources for processing."
-
-      - alert: PersistentQueueForWritesIsSaturated
-        expr: rate(vm_persistentqueue_write_duration_seconds_total[5m]) > 0.9
-        for: 15m
-        labels:
-          severity: warning
-        annotations:
-          dashboard: "http://localhost:3000/d/G7Z9GzMGz?viewPanel=98&var-instance={{ $labels.instance }}"
-          summary: "Persistent queue writes for instance {{ $labels.instance }} are saturated"
-          description: "Persistent queue writes for vmagent \"{{ $labels.job }}\" (instance {{ $labels.instance }})
-            are saturated by more than 90% and vmagent won't be able to keep up with flushing data on disk. 
-            In this case, consider to decrease load on the vmagent or improve the disk throughput."
-
-      - alert: PersistentQueueForReadsIsSaturated
-        expr: rate(vm_persistentqueue_read_duration_seconds_total[5m]) > 0.9
-        for: 15m
-        labels:
-          severity: warning
-        annotations:
-          dashboard: "http://localhost:3000/d/G7Z9GzMGz?viewPanel=99&var-instance={{ $labels.instance }}"
-          summary: "Persistent queue reads for instance {{ $labels.instance }} are saturated"
-          description: "Persistent queue reads for vmagent \"{{ $labels.job }}\" (instance {{ $labels.instance }})
-            are saturated by more than 90% and vmagent won't be able to keep up with reading data from the disk. 
-            In this case, consider to decrease load on the vmagent or improve the disk throughput."
-
-      - alert: SeriesLimitHourReached
-        expr: (vmagent_hourly_series_limit_current_series / vmagent_hourly_series_limit_max_series) > 0.9
-        labels:
-          severity: critical
-        annotations:
-          dashboard: "http://localhost:3000/d/G7Z9GzMGz?viewPanel=88&var-instance={{ $labels.instance }}"
-          summary: "Instance {{ $labels.instance }} reached 90% of the limit"
-          description: "Max series limit set via -remoteWrite.maxHourlySeries flag is close to reaching the max value. 
-            Then samples for new time series will be dropped instead of sending them to remote storage systems."
-
-      - alert: SeriesLimitDayReached
-        expr: (vmagent_daily_series_limit_current_series / vmagent_daily_series_limit_max_series) > 0.9
-        labels:
-          severity: critical
-        annotations:
-          dashboard: "http://localhost:3000/d/G7Z9GzMGz?viewPanel=90&var-instance={{ $labels.instance }}"
-          summary: "Instance {{ $labels.instance }} reached 90% of the limit"
-          description: "Max series limit set via -remoteWrite.maxDailySeries flag is close to reaching the max value. 
-            Then samples for new time series will be dropped instead of sending them to remote storage systems."
-
-      - alert: ConfigurationReloadFailure
-        expr: |
-          vm_promscrape_config_last_reload_successful != 1
-          or
-          vmagent_relabel_config_last_reload_successful != 1
-        labels:
-          severity: warning
-        annotations:
-          summary: "Configuration reload failed for vmagent instance {{ $labels.instance }}"
-          description: "Configuration hot-reload failed for vmagent on instance {{ $labels.instance }}.
-          Check vmagent's logs for detailed error message."
-
-      - alert: StreamAggrFlushTimeout
-        expr: |
-          increase(vm_streamaggr_flush_timeouts_total[5m]) > 0
-        labels:
-          severity: warning
-        annotations:
-          summary: "Streaming aggregation at \"{{ $labels.job }}\" (instance {{ $labels.instance }}) can't be finished within the configured aggregation interval."
-          description: "Stream aggregation process can't keep up with the load and might produce incorrect aggregation results. Check logs for more details.
-            Possible solutions: increase aggregation interval; aggregate smaller number of series; reduce samples' ingestion rate to stream aggregation."
-
-      - alert: StreamAggrDedupFlushTimeout
-        expr: |
-          increase(vm_streamaggr_dedup_flush_timeouts_total[5m]) > 0
-        labels:
-          severity: warning
-        annotations:
-          summary: "Deduplication \"{{ $labels.job }}\" (instance {{ $labels.instance }}) can't be finished within configured deduplication interval."
-          description: "Deduplication process can't keep up with the load and might produce incorrect results. Check docs https://docs.victoriametrics.com/victoriametrics/stream-aggregation/#deduplication and logs for more details.
-            Possible solutions: increase deduplication interval; deduplicate smaller number of series; reduce samples' ingestion rate."

apps/victoriametrics/rules/alerts-vmalert.yml

@@ -1,96 +0,0 @@
-# File contains default list of alerts for vmalert service.
-# The alerts below are just recommendations and may require some updates
-# and threshold calibration according to every specific setup.
-groups:
-  # Alerts group for vmalert assumes that Grafana dashboard
-  # https://grafana.com/grafana/dashboards/14950 is installed.
-  # Pls update the `dashboard` annotation according to your setup.
-  - name: vmalert
-    interval: 30s
-    rules:
-      - alert: ConfigurationReloadFailure
-        expr: vmalert_config_last_reload_successful != 1
-        labels:
-          severity: warning
-        annotations:
-          summary: "Configuration reload failed for vmalert instance {{ $labels.instance }}"
-          description: "Configuration hot-reload failed for vmalert on instance {{ $labels.instance }}.
-            Check vmalert's logs for detailed error message."
-
-      - alert: AlertingRulesError
-        expr: sum(increase(vmalert_alerting_rules_errors_total[5m])) without(id) > 0
-        for: 5m
-        labels:
-          severity: warning
-        annotations:
-          dashboard: "http://localhost:3000/d/LzldHAVnz?viewPanel=13&var-instance={{ $labels.instance }}&var-file={{ $labels.file }}&var-group={{ $labels.group }}"
-          summary: "Alerting rules are failing for vmalert instance {{ $labels.instance }}"
-          description: "Alerting rules execution is failing for \"{{ $labels.alertname }}\" from group \"{{ $labels.group }}\" in file \"{{ $labels.file }}\".
-            Check vmalert's logs for detailed error message."
-
-      - alert: RecordingRulesError
-        expr: sum(increase(vmalert_recording_rules_errors_total[5m])) without(id) > 0
-        for: 5m
-        labels:
-          severity: warning
-        annotations:
-          dashboard: "http://localhost:3000/d/LzldHAVnz?viewPanel=30&var-instance={{ $labels.instance }}&var-file={{ $labels.file }}&var-group={{ $labels.group }}"
-          summary: "Recording rules are failing for vmalert instance {{ $labels.instance }}"
-          description: "Recording rules execution is failing for \"{{ $labels.recording }}\" from group \"{{ $labels.group }}\" in file \"{{ $labels.file }}\".
-            Check vmalert's logs for detailed error message."
-
-      - alert: RecordingRulesNoData
-        expr: sum(vmalert_recording_rules_last_evaluation_samples) without(id) < 1
-        for: 30m
-        labels:
-          severity: info
-        annotations:
-          dashboard: "http://localhost:3000/d/LzldHAVnz?viewPanel=33&var-file={{ $labels.file }}&var-group={{ $labels.group }}"
-          summary: "Recording rule {{ $labels.recording }} ({{ $labels.group }}) produces no data"
-          description: "Recording rule \"{{ $labels.recording }}\" from group \"{{ $labels.group }}\ in file \"{{ $labels.file }}\"
-            produces 0 samples over the last 30min. It might be caused by a misconfiguration
-            or incorrect query expression."
-
-      - alert: TooManyMissedIterations
-        expr: increase(vmalert_iteration_missed_total[5m]) > 0
-        for: 15m
-        labels:
-          severity: warning
-        annotations:
-          summary: "vmalert instance {{ $labels.instance }} is missing rules evaluations"
-          description: "vmalert instance {{ $labels.instance }} is missing rules evaluations for group \"{{ $labels.group }}\" in file \"{{ $labels.file }}\".
-            The group evaluation time takes longer than the configured evaluation interval. This may result in missed
-            alerting notifications or recording rules samples. Try increasing evaluation interval or concurrency of
-            group \"{{ $labels.group }}\". See https://docs.victoriametrics.com/victoriametrics/vmalert/#groups.
-            If rule expressions are taking longer than expected, please see https://docs.victoriametrics.com/victoriametrics/troubleshooting/#slow-queries."
-
-      - alert: RemoteWriteErrors
-        expr: increase(vmalert_remotewrite_errors_total[5m]) > 0
-        for: 15m
-        labels:
-          severity: warning
-        annotations:
-          summary: "vmalert instance {{ $labels.instance }} is failing to push metrics to remote write URL"
-          description: "vmalert instance {{ $labels.instance }} is failing to push metrics generated via alerting
-            or recording rules to the configured remote write URL. Check vmalert's logs for detailed error message."
-
-      - alert: RemoteWriteDroppingData
-        expr: increase(vmalert_remotewrite_dropped_rows_total[5m]) > 0
-        for: 5m
-        labels:
-          severity: critical
-        annotations:
-          summary: "vmalert instance {{ $labels.instance }} is dropping data sent to remote write URL"
-          description: "vmalert instance {{ $labels.instance }} is failing to send results of alerting or recording rules
-            to the configured remote write URL. This may result into gaps in recording rules or alerts state.
-            Check vmalert's logs for detailed error message."
-
-      - alert: AlertmanagerErrors
-        expr: increase(vmalert_alerts_send_errors_total[5m]) > 0
-        for: 15m
-        labels:
-          severity: warning
-        annotations:
-          summary: "vmalert instance {{ $labels.instance }} is failing to send notifications to Alertmanager"
-          description: "vmalert instance {{ $labels.instance }} is failing to send alert notifications to \"{{ $labels.addr }}\".
-            Check vmalert's logs for detailed error message."

apps/victoriametrics/rules/alerts.yml

@@ -1,138 +0,0 @@
-# File contains default list of alerts for VictoriaMetrics single server.
-# The alerts below are just recommendations and may require some updates
-# and threshold calibration according to every specific setup.
-groups:
-  # Alerts group for VM single assumes that Grafana dashboard
-  # https://grafana.com/grafana/dashboards/10229 is installed.
-  # Pls update the `dashboard` annotation according to your setup.
-  - name: vmsingle
-    interval: 30s
-    concurrency: 2
-    rules:
-      - alert: DiskRunsOutOfSpaceIn3Days
-        expr: |
-            sum(vm_free_disk_space_bytes) without(path) /
-            (
-              (rate(vm_rows_added_to_storage_total[1d]) - sum(rate(vm_deduplicated_samples_total[1d])) without(type)) * (
-                sum(vm_data_size_bytes{type!~"indexdb.*"}) without(type) /
-                sum(vm_rows{type!~"indexdb.*"}) without(type)
-              )
-              +
-              rate(vm_new_timeseries_created_total[1d]) * (
-                sum(vm_data_size_bytes{type="indexdb/file"}) without(type)/
-                sum(vm_rows{type="indexdb/file"}) without(type)
-              )
-            ) < 3 * 24 * 3600 > 0
-        for: 30m
-        labels:
-          severity: critical
-        annotations:
-          dashboard: "http://localhost:3000/d/wNf0q_kZk?viewPanel=53&var-instance={{ $labels.instance }}"
-          summary: "Instance {{ $labels.instance }} will run out of disk space soon"
-          description: "Taking into account current ingestion rate, free disk space will be enough only
-            for {{ $value | humanizeDuration }} on instance {{ $labels.instance }}.\n
-            Consider to limit the ingestion rate, decrease retention or scale the disk space if possible."
-
-      - alert: NodeBecomesReadonlyIn3Days
-        expr: |
-          sum(vm_free_disk_space_bytes - vm_free_disk_space_limit_bytes) without(path) /
-          (
-              (rate(vm_rows_added_to_storage_total[1d]) - sum(rate(vm_deduplicated_samples_total[1d])) without(type)) * (
-                sum(vm_data_size_bytes{type!~"indexdb.*"}) without(type) /
-                sum(vm_rows{type!~"indexdb.*"}) without(type)
-              )
-              +
-              rate(vm_new_timeseries_created_total[1d]) * (
-                sum(vm_data_size_bytes{type="indexdb/file"}) without(type) /
-                sum(vm_rows{type="indexdb/file"}) without(type)
-              )
-          ) < 3 * 24 * 3600 > 0
-        for: 30m
-        labels:
-          severity: warning
-        annotations:
-          dashboard: "http://localhost:3000/d/oS7Bi_0Wz?viewPanel=53&var-instance={{ $labels.instance }}"
-          summary: "Instance {{ $labels.instance }} will become read-only in 3 days"
-          description: "Taking into account current ingestion rate and free disk space
-              instance {{ $labels.instance }} is writable for {{ $value | humanizeDuration }}.\n
-              Consider to limit the ingestion rate, decrease retention or scale the disk space up if possible."
-
-      - alert: DiskRunsOutOfSpace
-        expr: |
-          sum(vm_data_size_bytes) by(job, instance) /
-          (
-           sum(vm_free_disk_space_bytes) by(job, instance) +
-           sum(vm_data_size_bytes) by(job, instance)
-          ) > 0.8
-        for: 30m
-        labels:
-          severity: critical
-        annotations:
-          dashboard: "http://localhost:3000/d/wNf0q_kZk?viewPanel=53&var-instance={{ $labels.instance }}"
-          summary: "Instance {{ $labels.instance }} (job={{ $labels.job }}) will run out of disk space soon"
-          description: "Disk utilisation on instance {{ $labels.instance }} is more than 80%.\n
-            Having less than 20% of free disk space could cripple merge processes and overall performance.
-            Consider to limit the ingestion rate, decrease retention or scale the disk space if possible."
-
-      - alert: RequestErrorsToAPI
-        expr: increase(vm_http_request_errors_total[5m]) > 0
-        for: 15m
-        labels:
-          severity: warning
-        annotations:
-          dashboard: "http://localhost:3000/d/wNf0q_kZk?viewPanel=35&var-instance={{ $labels.instance }}"
-          summary: "Too many errors served for path {{ $labels.path }} (instance {{ $labels.instance }})"
-          description: "Requests to path {{ $labels.path }} are receiving errors.
-            Please verify if clients are sending correct requests."
-
-      - alert: TooHighChurnRate
-        expr: |
-          (
-             sum(rate(vm_new_timeseries_created_total[5m])) by(instance)
-             /
-             sum(rate(vm_rows_inserted_total[5m])) by(instance)
-           ) > 0.1
-        for: 15m
-        labels:
-          severity: warning
-        annotations:
-          dashboard: "http://localhost:3000/d/wNf0q_kZk?viewPanel=66&var-instance={{ $labels.instance }}"
-          summary: "Churn rate is more than 10% on \"{{ $labels.instance }}\" for the last 15m"
-          description: "VM constantly creates new time series on \"{{ $labels.instance }}\".\n
-            This effect is known as Churn Rate.\n
-            High Churn Rate is tightly connected with database performance and may
-            result in unexpected OOM's or slow queries."
-
-      - alert: TooHighChurnRate24h
-        expr: |
-          sum(increase(vm_new_timeseries_created_total[24h])) by(instance)
-          >
-          (sum(vm_cache_entries{type="storage/hour_metric_ids"}) by(instance) * 3)
-        for: 15m
-        labels:
-          severity: warning
-        annotations:
-          dashboard: "http://localhost:3000/d/wNf0q_kZk?viewPanel=66&var-instance={{ $labels.instance }}"
-          summary: "Too high number of new series on \"{{ $labels.instance }}\" created over last 24h"
-          description: "The number of created new time series over last 24h is 3x times higher than
-            current number of active series on \"{{ $labels.instance }}\".\n
-            This effect is known as Churn Rate.\n
-            High Churn Rate is tightly connected with database performance and may
-            result in unexpected OOM's or slow queries."
-
-      - alert: TooHighSlowInsertsRate
-        expr: |
-          (
-             sum(rate(vm_slow_row_inserts_total[5m])) by(instance)
-             /
-             sum(rate(vm_rows_inserted_total[5m])) by(instance)
-           ) > 0.05
-        for: 15m
-        labels:
-          severity: warning
-        annotations:
-          dashboard: "http://localhost:3000/d/wNf0q_kZk?viewPanel=68&var-instance={{ $labels.instance }}"
-          summary: "Percentage of slow inserts is more than 5% on \"{{ $labels.instance }}\" for the last 15m"
-          description: "High rate of slow inserts on \"{{ $labels.instance }}\" may be a sign of resource exhaustion
-            for the current load. It is likely more RAM is needed for optimal handling of the current number of active time series.
-            See also https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3976#issuecomment-1476883183"

apps/zot/README.md

@@ -1,15 +1,32 @@
 # zot OCI Registry
 
-内网入口为 `https://zot.ad.ddupan.top`。使用官方 Helm chart `0.1.124`，运行
+内网匿名拉取入口为 `https://zot.ad.ddupan.top`，SPIRE 鉴权推送入口为
+`https://zot-push.ad.ddupan.top`。使用官方 Helm chart `0.1.124`，运行
 zot `v2.1.21`，镜像固定到官方 linux/amd64 digest。
 
+## 当前工作状态（2026-09-16 核验）
+
+| 项目 | 状态 |
+|---|---|
+| 匿名拉取 | `zot.ad.ddupan.top` 已上线；空 `DOCKER_CONFIG` 的 crane pull 通过 |
+| SPIRE 鉴权入口 | `zot-push.ad.ddupan.top` 已上线；真实 JWT-SVID 推送后可匿名拉取同一 digest |
+| GitOps | 双入口配置已合并；Flux `zot` Kustomization 已应用 `d15733c`，状态 Ready |
+| 运行与凭据同步 | `zot`、`zot-reader` HelmRelease 均 Ready，Pod 均 1/1；ESO SecretSynced |
+| 临时配置清理 | 两个 HelmRelease 均无 `spec.values` 临时覆盖；暂停回写标记、测试身份和临时写权限已清理 |
+| 接管复验 | 匿名拉取成功；推送入口无凭据返回 401，token realm 指向推送域名；接管未触发 Pod 重启 |
+
+后续工作是给实际 CI 的 SPIFFE ID 配置具体仓库的 `create`/`update` 权限。
+SPIRE 认证链路已经验证，但当前没有常驻 publisher 或删除授权；认证成功本身不代表
+可以推送。S3 侧仍使用 Bao 管理的静态 AK/SK，尚未接入 SPIRE/STS。
+
 ## 存储与凭据
 
 制品、manifest 和 OCI layout 保存在现有 SeaweedFS 的 `zot` bucket，前缀为
 `registry/`，S3 endpoint 为 `https://s3.ad.ddupan.top`。**不创建 PVC**；chart 的
 `/var/lib/registry` 是 `emptyDir`，仅用于运行时本地工作数据。
 
-首期单副本，关闭跨仓库 dedupe，不额外部署 Redis/DynamoDB 缓存。保留 zot GC，
+两个单副本实例共用同一 bucket 和前缀：`zot` 负责鉴权写入，`zot-reader` 负责匿名
+读取。关闭跨仓库 dedupe，不额外部署 Redis/DynamoDB 缓存。只有写入实例启用 GC，
 暂不配置自动删除已发布版本的 retention policy。增加副本、启用 dedupe 或搜索等
 扩展前，需要重新检查共享元数据与缓存的持久化要求。
 
@@ -20,7 +37,7 @@ OpenBao kv/k8s/seaweedfs-s3
   → 原有 S3 身份及基础配置 ─┐
                          ├→ ESO 模板 → seaweedfs/seaweedfs-s3-config
 OpenBao kv/k8s/zot-s3 ────┘                 → 完整 s3.config
-  └→ ESO → zot/zot-s3 → zot 的 AWS_ACCESS_KEY_ID / AWS_SECRET_ACCESS_KEY
+  └→ ESO → zot/zot-s3 → zot 与 zot-reader 的 AWS_ACCESS_KEY_ID / AWS_SECRET_ACCESS_KEY
 
 ```
 
@@ -37,7 +54,7 @@ Terraform 的 `tfstate` bucket。`kv/k8s/zot-s3` 的字段是 `access_key` 和
 本次归一没有轮换密钥，生成的完整配置与归一前语义一致。当前模板只有一组 zot
 凭据，尚未实现新旧密钥重叠轮换。后续轮换只修改 `kv/k8s/zot-s3`，但仍需协调
 两个 ExternalSecret 同步：确认 SeaweedFS Secret volume 更新后向 filer 的
-`weed` 进程发送 SIGHUP，再确认 zot Secret 更新并重启 zot（环境变量不会热更新）。
+`weed` 进程发送 SIGHUP，再确认 zot Secret 更新并重启 `zot` 和 `zot-reader`（环境变量不会热更新）。
 两端异步更新期间可能短暂认证失败；需要无中断轮换时先扩展模板支持新旧凭据重叠。
 
 ## SPIRE 认证和授权
@@ -47,24 +64,40 @@ Terraform 的 `tfstate` bucket。`kv/k8s/zot-s3` 的字段是 `access_key` 和
 | issuer | `https://spire-oidc.ad.ddupan.top` |
 | JWT audience | `zot` |
 | subject | `spiffe://ddupan.top/` 下的 workload SPIFFE ID |
-| token endpoint | `https://zot.ad.ddupan.top/zot/auth/token` |
+| token endpoint | `https://zot-push.ad.ddupan.top/zot/auth/token` |
-| 当前权限 | 受信身份可以读取所有仓库；没有常驻写入或删除授权 |
+| 拉取入口 | 内网匿名读取所有仓库，不要求 SPIRE 身份 |
+| 推送入口当前权限 | 受信身份可以读取所有仓库；没有常驻写入或删除授权 |
 
 zot 通过已配置的 issuer discovery/JWKS 验证 JWT-SVID，再以 `sub` 作为授权身份。
 不接受任意 issuer，不关闭 TLS/issuer 验证。新的 Kata CI 负责取得并更新自己的
 JWT-SVID；确认其身份命名后，再添加针对具体 repository 的 `create`/`update`
 授权，不能把整个 trust domain 都授予写权限。
 
-现阶段拉取也需要 JWT-SVID。原定内网匿名拉取尚未启用：zot `v2.1.21` 的
+zot `v2.1.21` 的 OIDC Bearer middleware 会在授权阶段之前拒绝无 token 请求。
-OIDC Bearer middleware 会在授权阶段之前拒绝无 token 请求，单独增加
+因此使用两个官方 zot 实例与两个域名，避免修改上游镜像，也避免同域名下匿名
-`anonymousPolicy` 无法解决。匿名读取与 SPIRE 写入共存需后续单独验证方案。
+`/v2/` 返回 200 导致标准客户端跳过 token 交换的问题。
 
+- `zot-reader` 叠加 `reader-values.yaml`，没有认证 middleware，只有
+  `anonymousPolicy: [read]`。入口只转发 `/v2/` 的 GET/HEAD，并移除客户端遗留的
+  Authorization/Cookie；直接访问 reader Service 也不能写入。
+- `zot` 保留 SPIRE issuer/audience/subject 校验及仓库授权，`externalUrl`、
+  Bearer realm、service 与 HTTPRoute 均使用 `zot-push.ad.ddupan.top`。
+- reader 关闭 GC，没有同步或扫描扩展；读取同一份 S3 制品，不复制 bucket，
+  不新增 PVC 或 S3 密钥。镜像、安全上下文、资源和 Secret 引用由共用 values 继承。
+- 两个配置的 `storageDriver` 必须保持一致；修改 S3 endpoint/bucket/prefix 时
+  同时更新 `values.yaml` 与 `reader-values.yaml`。
+
+推送客户端应登录 `zot-push.ad.ddupan.top`；拉取客户端无需登录。
 已有 SPIRE 身份的进程可以通过 Workload API 获取 `aud=zot` 的 JWT-SVID，然后
 通过 `docker login` 或 `crane auth login` 的 `--password-stdin` 交给 Registry。
 用户名可以使用 `zot`，实际权限取自已验证 JWT 的身份。使用独立、权限为 `0700`
 的临时 `DOCKER_CONFIG`，结束后删除；不要开启 shell tracing，不要打印 token，
 不要把 token 放进命令参数。token 接口不会延长 SVID 有效期。
 
+同一仓库在两个入口使用相同路径和 tag/digest，例如 CI 推送到
+`zot-push.ad.ddupan.top/team/image:tag`，部署时使用
+`zot.ad.ddupan.top/team/image:tag`；无需在两个仓库间复制。
+
 ## 部署与网络
 
 - 官方 chart 管理 Deployment、Service、ConfigMap 和 HTTPRoute。
@@ -72,13 +105,17 @@ OIDC Bearer middleware 会在授权阶段之前拒绝无 token 请求，单独
   `https` listener 与内网通配符证书终止。
 - 仅配置 Samba AD 内网 DNS；不创建公网 DNS 或 Cloudflare Tunnel route。
 - NetworkPolicy 只允许现有 Envoy Gateway 数据面访问 zot 的 5000 端口。
-- HTTPRoute 只暴露 `/v2/` 和 `/zot/auth/token`，不暴露内部健康检查或管理端点。
+- 拉取域名仅暴露 `/v2/` 的 GET/HEAD；推送域名暴露 `/v2/` 和
+  `/zot/auth/token`，均不暴露内部健康检查或管理端点。
 - namespace 使用 restricted PodSecurity，容器非 root、只读根文件系统。
 
-首次已按用户授权从本地执行 `kubectl apply -k apps/zot`，由集群 Helm controller
+`clusters/homelab/apps/zot.yaml` 已将 `zot` 和 `zot-reader` 一并纳入 Flux 管理。
-安装。`clusters/homelab/apps/zot.yaml` 是 GitOps composition；对应文件合并进入
+两个 HelmRelease 通过共用 `zot-values` 继承基础配置，reader 再叠加
-Flux 跟踪分支后，才由根 Kustomization 持续管理，不能把未提交的本地部署写成
+`zot-reader-values`。当前由 main 分支持续管理，不依赖本地覆盖或暂停回写。
-已完成 Git 接管。
+
+后续若需临时验收，收尾时先确认 Git 管理的配置与目标运行配置一致，再移除
+`spec.values` 临时覆盖及 `kustomize.toolkit.fluxcd.io/reconcile=disabled` 标记，
+触发 zot Kustomization reconcile 并复验。临时测试身份和写权限不得留在持久配置中。
 
 检查与渲染：
 
@@ -114,6 +151,35 @@ sha256:b8d3b977a1235022759470903dab4a46b7cf8107958624f1f76a323eabe37c5e
 
 它是仅含验证文本的 OCI 测试镜像，没有可执行入口，不用于运行服务。
 
+双域名验收还使用 `verification/anonymous-spire:smoke`：标准 crane 从
+`zot-push.ad.ddupan.top` 登录、推送，再从 `zot.ad.ddupan.top` 使用空
+`DOCKER_CONFIG` 拉取，两个入口的 digest 必须一致。验证匿名 blob HEAD、tags、
+referrers，以及客户端保存旧凭据时的公共拉取。推送入口检查无凭据、错误签名、
+错误 audience、过期 SVID、跨仓库写入和删除拒绝；公共入口拒绝所有写方法，
+reader Service 直连也拒绝写入。测试完成后撤回临时单仓库写权限。
+
+2026-09-16 上述双域名验收通过；SVID 过期后推送入口返回 401，匿名拉取不受
+影响。临时写权限已撤销，两个 HelmRelease Ready；推送 DNS 第二次检查 changed=0。
+
+匿名拉取示例：
+
+```bash
+crane pull zot.ad.ddupan.top/verification/anonymous-spire:smoke image.tar --format oci
+```
+
+鉴权推送示例（先通过 Workload API 将短期 JWT-SVID 保存到当前进程的 `ZOT_JWT`，
+不要启用 shell tracing；示例中的仓库仍需提前给具体 SPIFFE ID 授权）：
+
+```bash
+export DOCKER_CONFIG="$(mktemp -d)"
+printf '%s' "$ZOT_JWT" | crane auth login zot-push.ad.ddupan.top \
+  --username zot --password-stdin
+crane push image.tar zot-push.ad.ddupan.top/team/image:tag
+rm -rf -- "$DOCKER_CONFIG"
+unset DOCKER_CONFIG ZOT_JWT
+```
+
+
 Registry 恢复需要完整的 SeaweedFS bucket 数据、Bao 专用凭据和此目录配置。
 zot 的临时目录不是制品备份。独立异机/离线备份尚未在本次部署中建立；不能把同一
 SeaweedFS 内的数据副本当作独立灾备。重装 zot 不得删除 `zot` bucket。

apps/zot/helmrelease-reader.yaml

@@ -0,0 +1,32 @@
+apiVersion: helm.toolkit.fluxcd.io/v2
+kind: HelmRelease
+metadata:
+  name: zot-reader
+  namespace: zot
+spec:
+  chart:
+    spec:
+      chart: zot
+      version: 0.1.124
+      interval: 1h
+      sourceRef:
+        kind: HelmRepository
+        name: zot
+  releaseName: zot-reader
+  interval: 30m
+  timeout: 5m
+  driftDetection:
+    mode: enabled
+  install:
+    strategy:
+      name: RetryOnFailure
+      retryInterval: 5m
+  upgrade:
+    strategy:
+      name: RetryOnFailure
+      retryInterval: 5m
+  valuesFrom:
+  - kind: ConfigMap
+    name: zot-values
+  - kind: ConfigMap
+    name: zot-reader-values

apps/zot/kustomization.yaml

@@ -6,6 +6,7 @@ resources:
   - external-secret.yaml
   - helmrepository.yaml
   - helmrelease.yaml
+  - helmrelease-reader.yaml
   - networkpolicy.yaml
 generatorOptions:
   disableNameSuffixHash: true
@@ -16,3 +17,7 @@ configMapGenerator:
     namespace: zot
     files:
       - values.yaml=values.yaml
+  - name: zot-reader-values
+    namespace: zot
+    files:
+      - values.yaml=reader-values.yaml

apps/zot/reader-values.yaml

@@ -0,0 +1,64 @@
+# 叠加于共用 values.yaml；同一镜像、S3、Secret、安全设置，无制品副本。
+# 无 Bearer middleware，仅 anonymousPolicy=read；关闭 GC 避免多个实例清理共享存储。
+configFiles:
+  config.json: |
+    {
+      "distSpecVersion": "1.1.1",
+      "storage": {
+        "rootDirectory": "/var/lib/registry",
+        "dedupe": false,
+        "gc": false,
+        "storageDriver": {
+          "name": "s3",
+          "region": "us-east-1",
+          "regionendpoint": "https://s3.ad.ddupan.top",
+          "bucket": "zot",
+          "rootdirectory": "/registry",
+          "secure": true,
+          "skipverify": false,
+          "forcepathstyle": true
+        }
+      },
+      "http": {
+        "address": "0.0.0.0",
+        "port": "5000",
+        "externalUrl": "https://zot.ad.ddupan.top",
+        "compat": [
+          "docker2s2"
+        ],
+        "accessControl": {
+          "repositories": {
+            "**": {
+              "anonymousPolicy": [
+                "read"
+              ]
+            }
+          }
+        }
+      },
+      "log": {
+        "level": "info"
+      }
+    }
+httproute:
+  hostnames:
+  - zot.ad.ddupan.top
+  rules:
+  - matches:
+    - path:
+        type: PathPrefix
+        value: /v2/
+      method: GET
+    - path:
+        type: PathPrefix
+        value: /v2/
+      method: HEAD
+    filters:
+    - type: RequestHeaderModifier
+      requestHeaderModifier:
+        remove:
+        - Cookie
+        - Authorization
+    timeouts:
+      request: 900s
+      backendRequest: 900s

apps/zot/values.yaml

@@ -41,14 +41,14 @@ configFiles:
       "http": {
         "address": "0.0.0.0",
         "port": "5000",
-        "externalUrl": "https://zot.ad.ddupan.top",
+        "externalUrl": "https://zot-push.ad.ddupan.top",
         "compat": [
           "docker2s2"
         ],
         "auth": {
           "bearer": {
-            "realm": "https://zot.ad.ddupan.top/zot/auth/token",
+            "realm": "https://zot-push.ad.ddupan.top/zot/auth/token",
-            "service": "zot.ad.ddupan.top",
+            "service": "zot-push.ad.ddupan.top",
             "oidc": [
               {
                 "issuer": "https://spire-oidc.ad.ddupan.top",
@@ -70,7 +70,80 @@ configFiles:
         },
         "accessControl": {
           "repositories": {
+            "panxiao81/backstage": {
+              "policies": [
+                {
+                  "users": [
+                    "spiffe://ddupan.top/ci/panxiao81/backstage/image",
+                    "spiffe://ddupan.top/dev/panxiao81"
+                  ],
+                  "actions": [
+                    "read",
+                    "create",
+                    "update"
+                  ]
+                }
+              ],
+              "defaultPolicy": [
+                "read"
+              ]
+            },
+            "panxiao81/gitea-dynamic-runner-controller": {
+              "policies": [
+                {
+                  "users": [
+                    "spiffe://ddupan.top/ci/panxiao81/gitea-dynamic-runner/publish-images"
+                  ],
+                  "actions": [
+                    "read",
+                    "create",
+                    "update"
+                  ]
+                },
+                {
+                  "users": ["spiffe://ddupan.top/dev/panxiao81"],
+                  "actions": ["read", "create", "update", "delete"]
+                }
+              ],
+              "defaultPolicy": [
+                "read"
+              ]
+            },
+            "panxiao81/gitea-dynamic-runner-runner": {
+              "policies": [
+                {
+                  "users": [
+                    "spiffe://ddupan.top/ci/panxiao81/gitea-dynamic-runner/publish-images"
+                  ],
+                  "actions": [
+                    "read",
+                    "create",
+                    "update"
+                  ]
+                },
+                {
+                  "users": ["spiffe://ddupan.top/dev/panxiao81"],
+                  "actions": ["read", "create", "update", "delete"]
+                }
+              ],
+              "defaultPolicy": [
+                "read"
+              ]
+            },
             "**": {
+              "policies": [
+                {
+                  "users": [
+                    "spiffe://ddupan.top/dev/panxiao81"
+                  ],
+                  "actions": [
+                    "read",
+                    "create",
+                    "update",
+                    "delete"
+                  ]
+                }
+              ],
               "defaultPolicy": [
                 "read"
               ]
@@ -133,7 +206,7 @@ httproute:
     namespace: envoy-gateway-system
     sectionName: https
   hostnames:
-  - zot.ad.ddupan.top
+  - zot-push.ad.ddupan.top
   rules:
   - matches:
     - path:

clusters/homelab/README.md

@@ -50,6 +50,9 @@ sudo k3s kubectl -n flux-system get gitrepositories,kustomizations
 - VictoriaMetrics Operator 已固定现有 chart `0.66.2` 并完成分阶段 Flux HelmRelease
   接管；Metrics、Logs、Traces 与 Grafana 也已统一完成 Flux 接管；
 - External Secrets Operator 已固定 chart `2.8.0` 并完成分阶段接管；
-- SPIRE 已按官方 hardened chart `0.30.2`（SPIRE `1.15.3`）声明，使用共享
+- SPIRE 已按 hardened chart 内部 fork `0.30.2-ddupan.1`（基于上游 `0.30.2`，SPIRE
+  `1.15.3`）声明，使用共享
   PostgreSQL 与独立 signing-key PVC；首次上线和 OpenBao JWT-SVID PoC 尚待合并后验证；
+- Nexus Repository CE POC 已加入 GitOps 声明，计划验证 Ansible Galaxy、Go Modules 与
+  OCI/BuildKit 缓存；尚未部署或完成现场验收，现有 zot 保持不变；
 - root Kustomization 与所有 brownfield 子 Kustomization 继续保持 `prune: false`。

clusters/homelab/apps/backstage.yaml

@@ -0,0 +1,22 @@
+apiVersion: kustomize.toolkit.fluxcd.io/v1
+kind: Kustomization
+metadata:
+  name: backstage
+  namespace: flux-system
+spec:
+  dependsOn:
+    - name: envoy-gateway
+    - name: external-secrets
+  healthChecks:
+    - apiVersion: apps/v1
+      kind: Deployment
+      name: backstage
+      namespace: backstage
+  interval: 10m
+  path: ./apps/backstage
+  prune: true
+  sourceRef:
+    kind: GitRepository
+    name: flux-system
+  timeout: 5m
+  wait: true

clusters/homelab/apps/dynamic-runner.yaml

@@ -0,0 +1,19 @@
+apiVersion: kustomize.toolkit.fluxcd.io/v1
+kind: Kustomization
+metadata:
+  name: dynamic-runner
+  namespace: flux-system
+spec:
+  dependsOn:
+    - name: external-secrets
+    - name: spire
+  interval: 10m
+  path: ./platform/dynamic-runner
+  # The runner backends are replaceable. Prune is required when a retired
+  # worker is removed from the component; otherwise it keeps consuming work.
+  prune: true
+  sourceRef:
+    kind: GitRepository
+    name: flux-system
+  timeout: 5m
+  wait: true

clusters/homelab/apps/hydra.yaml

@@ -0,0 +1,17 @@
+apiVersion: kustomize.toolkit.fluxcd.io/v1
+kind: Kustomization
+metadata:
+  name: hydra
+  namespace: flux-system
+spec:
+  dependsOn:
+  - name: envoy-gateway
+  - name: external-secrets
+  interval: 10m
+  path: ./apps/hydra
+  prune: false
+  sourceRef:
+    kind: GitRepository
+    name: flux-system
+  timeout: 5m
+  wait: true

clusters/homelab/apps/nats.yaml

@@ -0,0 +1,18 @@
+apiVersion: kustomize.toolkit.fluxcd.io/v1
+kind: Kustomization
+metadata:
+  name: nats
+  namespace: flux-system
+spec:
+  dependsOn:
+    - name: cert-manager
+    - name: external-secrets
+    - name: openebs
+  interval: 10m
+  path: ./platform/nats
+  prune: false
+  sourceRef:
+    kind: GitRepository
+    name: flux-system
+  timeout: 10m
+  wait: true

clusters/homelab/apps/nexus.yaml

@@ -0,0 +1,22 @@
+apiVersion: kustomize.toolkit.fluxcd.io/v1
+kind: Kustomization
+metadata:
+  name: nexus
+  namespace: flux-system
+spec:
+  dependsOn:
+    - name: envoy-gateway
+    - name: openebs
+  healthChecks:
+    - apiVersion: apps/v1
+      kind: Deployment
+      name: nexus
+      namespace: nexus
+  interval: 10m
+  path: ./apps/nexus
+  prune: false
+  sourceRef:
+    kind: GitRepository
+    name: flux-system
+  timeout: 15m
+  wait: true

clusters/homelab/kustomization.yaml

@@ -7,9 +7,13 @@ resources:
   - apps/envoy-gateway.yaml
   - apps/external-secrets.yaml
   - apps/gitea.yaml
-  - apps/gitea-actions.yaml
+  - apps/backstage.yaml
   - apps/http-echo.yaml
   - apps/openebs.yaml
+  - apps/nats.yaml
+  - apps/dynamic-runner.yaml
   - apps/spire.yaml
   - apps/observability.yaml
   - apps/zot.yaml
+  - apps/nexus.yaml
+  - apps/hydra.yaml

clusters/sandbox/README.md

@@ -0,0 +1,72 @@
+# Sandbox 集群
+
+这里是 OpenSandbox、CI 和 AI Agent workload 所在双节点 k3s 集群的 Flux
+reconciliation 入口。LXC、PostgreSQL、K3s、固定版本的 Flux controllers 与 root
+sync 由 `infrastructure/sandbox-cluster/` 中的 Ansible 管理；本目录只组合集群内
+workload。
+
+Flux 通过 `https://git.ddupan.top/panxiao81/homelab-infra.git` 读取公开仓库。
+Ansible 将 homelab CA 注入 `GitRepository/flux-system` 引用的同名 Secret，不使用
+长期 Git 凭据。root Kustomization 从 `./clusters/sandbox` 开始 reconciliation，
+初始保持 `prune: false`。
+
+Root bootstrap 已完成。后续按依赖顺序分别引入：
+
+1. 监控 CRD、kube-state-metrics 以及 kubelet/cAdvisor 抓取配置；
+2. SPIRE Agent、SPIFFE CSI Driver 与 workload registration；
+3. Kata Containers、`block-plain` RuntimeClass；
+4. 独立 External Secrets Operator 与 sandbox 专用 OpenBao auth backend；
+5. OpenSandbox controller/server；CI Pool 与 runner 调度器随后独立接入。
+
+每一阶段单独合并并等待对应 Flux Kustomization Ready，不在 bootstrap 时一次性部署。
+第一阶段监控拆为 `monitoring-operator` 与依赖它的 `monitoring`，防止 VM CR 在
+VictoriaMetrics Operator CRD Ready 前进入 reconciliation。
+
+SPIRE 阶段先由 `spire-bootstrap` 安装 CRD，并声明按上游 k8s_psat Server plugin
+要求收窄的 reviewer：它可以调用 TokenReview，并只读查询用于证明的 Pod 与 Node。
+Agent ServiceAccount 留给后续 HelmRelease 创建，避免两个声明方争夺同一资源。随后运行
+`infrastructure/sandbox-cluster/ansible/spire-bootstrap.yml`：playbook 从 sandbox
+读取 reviewer token，在内存中组成受限 kubeconfig，再通过 stdin reconcile 到 central
+集群的 `spire-server/spire-external-kubeconfigs` Secret。凭据不写入仓库、日志或控制机
+文件；该 Secret 准备完成后，才能启用 central external PSAT/controller-manager 和
+sandbox Agent/CSI。
+
+External controller-manager 使用独立的 `spire-controller-manager` ServiceAccount；其
+RBAC 与上游 controller-manager 所需权限一致，用于读取 workload selectors、维护
+SPIFFE CR status/finalizer 和 leader election。它不复用只允许 TokenReview 的 Server
+reviewer。Ansible 将两份 kubeconfig 写入同一个 central Secret 的不同 key，便于 central
+chart 分别绑定 `sandbox` 与 `sandbox-controller`。
+
+Central SPIRE Server 通过内网 `spire-server.ad.ddupan.top:8081` 接收 sandbox Agent
+attestation。Server 使用 external bundle publisher 持续维护 sandbox
+`spire-system/spire-bundle`，Agent 不固定或复制 trust bundle。Sandbox HelmRelease
+显式关闭 Server 与 OIDC Provider，只部署 Agent DaemonSet 和 SPIFFE CSI Driver；因此
+不会产生第二个 trust root。
+
+`spire-smoke` namespace、ServiceAccount 和 `sandbox-spire-smoke` ClusterSPIFFEID 只用于
+普通 Pod 的 CSI 回归夹具，稳定身份为 `spiffe://ddupan.top/sandbox/smoke`。Kata guest
+不能复用 node Agent 暴露的 Unix socket；virtio-fs 只能呈现 socket 路径，不能把连接
+跨过 VM 边界。Kata workload 必须使用 guest 内 Agent，具体约束见
+`platform/sandbox-kata/README.md`。测试 Pod 临时创建并在验收后删除，普通 Pod 的身份
+声明保留。
+
+Kata 阶段使用官方 4.1.0 `kata-deploy` chart 的短生命周期 `job` 模式，逐节点安装并
+重启 K3s。只启用 `kata-clh-runtime-rs`，不创建默认 `kata` 别名；该 handler 的
+`emptyDir` 固定使用 `block-plain`，为 Docker/BuildKit overlay2 与 kind 提供 guest
+内块设备文件系统。详细限制与上线验收见 `platform/sandbox-kata/README.md`。
+
+Sandbox 的 ESO 通过独立 `auth/kubernetes-sandbox` 向 OpenBao 证明 ServiceAccount
+身份，只能读取共享的 `kv/k8s/opensandbox-api`。它不保存 reviewer JWT 或长期 Bao token；相关
+Terraform 与 Flux 边界见 `platform/sandbox-external-secrets/README.md`。
+
+OpenSandbox 阶段固定官方源码 commit 与 umbrella chart `0.2.2`，只部署 controller、
+ClusterIP server 和 CRD。API key 由 ESO 从 OpenBao 投影，明文不进入 Git。
+
+## 监控边界
+
+这里只管理 sandbox LXC 内的 Kubernetes 监控，不负责 PVE 宿主监控。LXC 与宿主共享
+内核，即使 lxcfs 虚拟化了内存和 uptime，容器内 `/proc/stat` 仍是宿主 CPU 视图；
+在 LXC 内运行 node_exporter 会生成混合语义并重复采集宿主指标，因此禁止部署。
+
+Sandbox 节点与 workload 指标来自 kubelet/cAdvisor 和 kube-state-metrics；K3s 或 LXC
+特有但上述接口未覆盖的指标，应使用目标明确的 collector，不以 node_exporter 补齐。

clusters/sandbox/apps/ci-runners.yaml

@@ -0,0 +1,18 @@
+---
+apiVersion: kustomize.toolkit.fluxcd.io/v1
+kind: Kustomization
+metadata:
+  name: ci-runners
+  namespace: flux-system
+spec:
+  dependsOn:
+    - name: opensandbox
+    - name: spire-agents
+  interval: 10m
+  path: ./platform/sandbox-ci-runners
+  prune: true
+  sourceRef:
+    kind: GitRepository
+    name: flux-system
+  timeout: 20m
+  wait: true

clusters/sandbox/apps/external-secrets-operator.yaml

@@ -0,0 +1,17 @@
+---
+apiVersion: kustomize.toolkit.fluxcd.io/v1
+kind: Kustomization
+metadata:
+  name: external-secrets-operator
+  namespace: flux-system
+spec:
+  dependsOn:
+    - name: monitoring-operator
+  interval: 10m
+  path: ./platform/sandbox-external-secrets/operator
+  prune: true
+  sourceRef:
+    kind: GitRepository
+    name: flux-system
+  timeout: 10m
+  wait: true

clusters/sandbox/apps/external-secrets.yaml

@@ -0,0 +1,17 @@
+---
+apiVersion: kustomize.toolkit.fluxcd.io/v1
+kind: Kustomization
+metadata:
+  name: external-secrets
+  namespace: flux-system
+spec:
+  dependsOn:
+    - name: external-secrets-operator
+  interval: 10m
+  path: ./platform/sandbox-external-secrets/config
+  prune: true
+  sourceRef:
+    kind: GitRepository
+    name: flux-system
+  timeout: 10m
+  wait: true

clusters/sandbox/apps/kata.yaml

@@ -0,0 +1,18 @@
+---
+apiVersion: kustomize.toolkit.fluxcd.io/v1
+kind: Kustomization
+metadata:
+  name: kata
+  namespace: flux-system
+spec:
+  dependsOn:
+    - name: monitoring-operator
+    - name: spire-agents
+  interval: 10m
+  path: ./platform/sandbox-kata
+  prune: true
+  sourceRef:
+    kind: GitRepository
+    name: flux-system
+  timeout: 35m
+  wait: true

clusters/sandbox/apps/monitoring-operator.yaml

@@ -0,0 +1,15 @@
+---
+apiVersion: kustomize.toolkit.fluxcd.io/v1
+kind: Kustomization
+metadata:
+  name: monitoring-operator
+  namespace: flux-system
+spec:
+  interval: 10m
+  path: ./platform/sandbox-monitoring/operator
+  prune: true
+  sourceRef:
+    kind: GitRepository
+    name: flux-system
+  timeout: 10m
+  wait: true

clusters/sandbox/apps/monitoring.yaml

@@ -0,0 +1,17 @@
+---
+apiVersion: kustomize.toolkit.fluxcd.io/v1
+kind: Kustomization
+metadata:
+  name: monitoring
+  namespace: flux-system
+spec:
+  dependsOn:
+    - name: monitoring-operator
+  interval: 10m
+  path: ./platform/sandbox-monitoring/workloads
+  prune: true
+  sourceRef:
+    kind: GitRepository
+    name: flux-system
+  timeout: 10m
+  wait: true

clusters/sandbox/apps/opensandbox.yaml

@@ -0,0 +1,19 @@
+---
+apiVersion: kustomize.toolkit.fluxcd.io/v1
+kind: Kustomization
+metadata:
+  name: opensandbox
+  namespace: flux-system
+spec:
+  dependsOn:
+    - name: external-secrets
+    - name: kata
+    - name: monitoring-operator
+  interval: 10m
+  path: ./platform/sandbox-opensandbox
+  prune: true
+  sourceRef:
+    kind: GitRepository
+    name: flux-system
+  timeout: 15m
+  wait: true

clusters/sandbox/apps/spire-agents.yaml

@@ -0,0 +1,18 @@
+---
+apiVersion: kustomize.toolkit.fluxcd.io/v1
+kind: Kustomization
+metadata:
+  name: spire-agents
+  namespace: flux-system
+spec:
+  dependsOn:
+    - name: spire-bootstrap
+    - name: monitoring-operator
+  interval: 10m
+  path: ./platform/sandbox-spire/agents
+  prune: true
+  sourceRef:
+    kind: GitRepository
+    name: flux-system
+  timeout: 15m
+  wait: true

clusters/sandbox/apps/spire-bootstrap.yaml

@@ -1,14 +1,15 @@
+---
 apiVersion: kustomize.toolkit.fluxcd.io/v1
 kind: Kustomization
 metadata:
-  name: gitea-actions
+  name: spire-bootstrap
   namespace: flux-system
 spec:
   interval: 10m
-  path: ./platform/gitea-runner
+  path: ./platform/sandbox-spire/bootstrap
-  prune: false
+  prune: true
   sourceRef:
     kind: GitRepository
     name: flux-system
-  timeout: 3m
+  timeout: 10m
-  wait: false
+  wait: true

clusters/sandbox/kustomization.yaml

@@ -0,0 +1,13 @@
+---
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - apps/monitoring-operator.yaml
+  - apps/monitoring.yaml
+  - apps/spire-bootstrap.yaml
+  - apps/spire-agents.yaml
+  - apps/kata.yaml
+  - apps/external-secrets-operator.yaml
+  - apps/external-secrets.yaml
+  - apps/opensandbox.yaml
+  - apps/ci-runners.yaml

infrastructure/cloudflared/terraform/dns.generated.tf

@@ -0,0 +1,9 @@
+# Generated by infrastructure/dns/generate.py. Do not edit directly.
+resource "cloudflare_dns_record" "auth" {
+  zone_id = var.zone_id
+  name    = "auth.ddupan.top"
+  type    = "CNAME"
+  content = "ff392451-b0b1-45bb-964e-6d9372c3a9e3.cfargotunnel.com"
+  proxied = true
+  ttl     = 1
+}

infrastructure/cloudflared/terraform/main.tf

@@ -38,17 +38,6 @@ resource "cloudflare_zero_trust_tunnel_cloudflared_config" "main" {
   }
 }
 
-# Public DNS: proxied CNAME -> the tunnel. (auth was bootstrapped with
-# `cloudflared tunnel route dns`; import it into state — see README.)
-resource "cloudflare_dns_record" "auth" {
-  zone_id = var.zone_id
-  name    = "auth.ddupan.top"
-  type    = "CNAME"
-  content = "${var.tunnel_id}.cfargotunnel.com"
-  proxied = true
-  ttl     = 1 # 1 = automatic (required when proxied)
-}
-
 # DKIM for Microsoft 365 mail sent as *@ddupan.top (via the smtp-relay). CNAMEs point
 # at the tenant's DKIM keys; must be DNS-only (unproxied). Enable signing in Exchange
 # after these resolve: smtp-relay/scripts/enable-dkim.ps1.

infrastructure/dns/README.md

@@ -5,10 +5,32 @@
 
 | 视图 | 权威或递归服务 | 配置方式 |
 |---|---|---|
-| 公网 `ddupan.top` | Cloudflare | Terraform；尚待完整导入已有记录 |
+| 公网 `ddupan.top` | Cloudflare | 由生成器输出 Terraform；尚待完整导入已有记录 |
 | AD `ad.ddupan.top` | Samba internal DNS | `samba_dns_record` Ansible module |
-| LAN split horizon | Blocky | 尚待从 inventory 渲染或校验 |
+| LAN split horizon | Blocky | 由生成器维护 `customDNS.mapping` 标记块 |
-| Kubernetes Pod split horizon | CoreDNS | 尚待从 inventory 渲染或校验 |
+| Kubernetes Pod split horizon | CoreDNS | 由生成器维护 `.server` 标记块 |
+
+## 生成配置
+
+安装了 `uv` 后，在仓库根目录运行：
+
+```bash
+uv run infrastructure/dns/generate.py
+uv run infrastructure/dns/generate.py --check
+```
+
+脚本使用内嵌锁定版本的 PyYAML 和 Jinja2，从 `records.yml` 渲染三个目标：
+
+- `apps/blocky/config.yml` 中带 marker 的 LAN split-horizon mapping；
+- `platform/k3s/coredns-custom.yaml` 中带 marker 的 Pod split-horizon server blocks；
+- `infrastructure/cloudflared/terraform/dns.generated.tf` 中已经完成 Terraform 接管的公网记录。
+
+生成文件需要提交进 Git，以便 PR 直接审阅最终配置。CI 执行 `--check`，任何手工修改生成块、
+漏跑生成器或非确定性输出都会失败。Jinja 使用 `[[ ... ]]` 作为变量定界符，避免与 CoreDNS
+模板表达式 `{{ .Name }}` 冲突。
+
+`backends` 和 `terraform.managed` 是分阶段接管开关，而不是第二份记录数据：只有已经完成
+零变更接管的后端才会生成。把记录加入新的后端前，应先完成相应的 live/state 对账。
 
 ## 安全边界
 
@@ -21,9 +43,13 @@
 ## 分阶段接管
 
 1. 用 Samba module 接管现有静态 A RRset，首次 check mode 应为零变更。
-2. 将 Cloudflare 已有 tunnel DNS 记录导入 Terraform state。
+2. 将 Cloudflare 已有 tunnel DNS 记录逐条导入 Terraform state，再启用 `terraform.managed`。
-3. 让 Blocky 与 CoreDNS 从 `split_horizon.records` 生成配置或执行 CI 一致性检查。
+3. Blocky 与 CoreDNS 已从 `split_horizon.records` 生成；通过 `backends` 分阶段扩展。
 4. 验证公网、LAN、Pod、AD 四个视图后，再单独修改 DHCP。
 
-当前 inventory 已明确暴露一个既有差异：`obj.ddupan.top` 在 Blocky 中存在，但 CoreDNS
+当前 inventory 明确保留一个既有差异：`obj.ddupan.top` 的 `backends` 只有 Blocky，CoreDNS
-尚无对应覆盖。本阶段不会偷偷修复它；后续在两个 resolver 同时接管时统一修复。
+尚无对应覆盖。本阶段不改变线上语义；后续验证 Pod 侧入口后再加入 `coredns`。
+
+CoreDNS split-horizon 的原因是避免集群内请求经 Cloudflare 公网绕回同一个集群。尤其 Gitea
+启动时会访问 Authelia discovery URL，公网路径故障曾令其启动失败；生成块仍返回相同 LAN A
+记录，并对 AAAA 返回 NOERROR/no-data。

infrastructure/dns/generate.py

@@ -0,0 +1,146 @@
+#!/usr/bin/env -S uv run --script
+# /// script
+# requires-python = ">=3.12"
+# dependencies = ["Jinja2==3.1.6", "PyYAML==6.0.3"]
+# ///
+"""Render backend DNS configuration from records.yml."""
+
+from __future__ import annotations
+
+import argparse
+import difflib
+from pathlib import Path
+import sys
+
+import yaml
+from jinja2 import Environment, FileSystemLoader, StrictUndefined
+
+
+ROOT = Path(__file__).resolve().parents[2]
+DNS_DIR = ROOT / "infrastructure/dns"
+BEGIN = "# BEGIN GENERATED: homelab DNS ([[ target ]])"
+END = "# END GENERATED: homelab DNS ([[ target ]])"
+
+
+def load_inventory() -> dict:
+    data = yaml.safe_load((DNS_DIR / "records.yml").read_text())
+    try:
+        inventory = data["homelab_dns"]
+        split_records = inventory["split_horizon"]["records"]
+        public_records = inventory["public"]["records"]
+    except (KeyError, TypeError) as exc:
+        raise ValueError(f"invalid DNS inventory: missing {exc}") from exc
+
+    for record in split_records:
+        require_fields(record, "name", "type", "values", "backends")
+        if record["type"] != "A" or len(record["values"]) != 1:
+            raise ValueError(f"split record must be a single A value: {record!r}")
+        unknown = set(record["backends"]) - {"blocky", "coredns"}
+        if unknown:
+            raise ValueError(f"unknown split DNS backends {sorted(unknown)}")
+
+    for record in public_records:
+        require_fields(record, "name", "type", "values", "proxied", "terraform")
+        terraform = record["terraform"]
+        if terraform.get("managed") and not terraform.get("resource_name"):
+            raise ValueError(f"managed Terraform record needs resource_name: {record['name']}")
+        if len(record["values"]) != 1:
+            raise ValueError(f"Cloudflare Terraform supports one value per record: {record['name']}")
+    return inventory
+
+
+def require_fields(record: dict, *fields: str) -> None:
+    missing = [field for field in fields if field not in record]
+    if missing:
+        raise ValueError(f"record missing {', '.join(missing)}: {record!r}")
+
+
+def environment() -> Environment:
+    return Environment(
+        loader=FileSystemLoader(DNS_DIR / "templates"),
+        undefined=StrictUndefined,
+        autoescape=False,
+        keep_trailing_newline=True,
+        trim_blocks=True,
+        lstrip_blocks=True,
+        variable_start_string="[[",
+        variable_end_string="]]",
+        block_start_string="[%",
+        block_end_string="%]",
+    )
+
+
+def marker(target: str, end: bool = False) -> str:
+    return (END if end else BEGIN).replace("[[ target ]]", target)
+
+
+def replace_block(original: str, target: str, rendered: str) -> str:
+    begin = marker(target)
+    end = marker(target, end=True)
+    if original.count(begin) != 1 or original.count(end) != 1:
+        raise ValueError(f"expected exactly one generated block for {target}")
+    prefix, remainder = original.split(begin, 1)
+    _, suffix = remainder.split(end, 1)
+    indent = prefix.rsplit("\n", 1)[-1]
+    body = rendered.rstrip("\n")
+    return f"{prefix}{begin}\n{body}\n{indent}{end}{suffix}"
+
+
+def outputs(inventory: dict) -> dict[Path, str]:
+    env = environment()
+    split_records = inventory["split_horizon"]["records"]
+    public_records = inventory["public"]["records"]
+    result = {}
+
+    blocky_path = ROOT / "apps/blocky/config.yml"
+    blocky = env.get_template("blocky.yml.j2").render(
+        records=[record for record in split_records if "blocky" in record["backends"]]
+    )
+    result[blocky_path] = replace_block(blocky_path.read_text(), "blocky", blocky)
+
+    coredns_path = ROOT / "platform/k3s/coredns-custom.yaml"
+    coredns = env.get_template("coredns.yaml.j2").render(
+        records=[record for record in split_records if "coredns" in record["backends"]]
+    )
+    result[coredns_path] = replace_block(coredns_path.read_text(), "coredns", coredns)
+
+    terraform_path = ROOT / "infrastructure/cloudflared/terraform/dns.generated.tf"
+    terraform = env.get_template("cloudflare.tf.j2").render(
+        records=[record for record in public_records if record["terraform"]["managed"]]
+    )
+    result[terraform_path] = terraform
+    return result
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--check", action="store_true", help="fail when generated files differ")
+    args = parser.parse_args()
+    try:
+        rendered_outputs = outputs(load_inventory())
+    except (OSError, ValueError, yaml.YAMLError) as exc:
+        print(f"dns generation failed: {exc}", file=sys.stderr)
+        return 2
+
+    changed = False
+    for path, expected in rendered_outputs.items():
+        actual = path.read_text() if path.exists() else ""
+        if actual == expected:
+            continue
+        changed = True
+        if args.check:
+            print("".join(difflib.unified_diff(
+                actual.splitlines(keepends=True),
+                expected.splitlines(keepends=True),
+                fromfile=str(path.relative_to(ROOT)),
+                tofile=f"{path.relative_to(ROOT)} (generated)",
+            )))
+        else:
+            path.parent.mkdir(parents=True, exist_ok=True)
+            path.write_text(expected)
+            print(f"rendered {path.relative_to(ROOT)}")
+    return 1 if args.check and changed else 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

infrastructure/dns/records.yml

@@ -10,19 +10,30 @@ homelab_dns:
       - { zone: ad.ddupan.top, name: pve1, type: A, values: [192.168.10.4] }
       - { zone: ad.ddupan.top, name: pve2, type: A, values: [192.168.10.7] }
       - { zone: ad.ddupan.top, name: pve3, type: A, values: [192.168.10.9] }
+      - { zone: ad.ddupan.top, name: sandbox1, type: A, values: [10.60.0.11] }
+      - { zone: ad.ddupan.top, name: sandbox2, type: A, values: [10.60.0.12] }
+      - { zone: ad.ddupan.top, name: sandbox-k8s, type: A, values: [10.60.0.13] }
       - { zone: ad.ddupan.top, name: retrolab, type: A, values: [10.60.0.10] }
+      - { zone: ad.ddupan.top, name: grafana, type: A, values: [192.168.10.127] }
+      - { zone: ad.ddupan.top, name: metrics-write, type: A, values: [192.168.10.127] }
       - { zone: ad.ddupan.top, name: netbox, type: A, values: [192.168.10.127] }
+      - { zone: ad.ddupan.top, name: nats, type: A, values: [192.168.10.127] }
+      - { zone: ad.ddupan.top, name: nexus, type: A, values: [192.168.10.127] }
       - { zone: ad.ddupan.top, name: s3, type: A, values: [192.168.10.127] }
+      - { zone: ad.ddupan.top, name: hydra, type: A, values: [192.168.10.127] }
+      - { zone: ad.ddupan.top, name: hydra-login, type: A, values: [192.168.10.127] }
       - { zone: ad.ddupan.top, name: spire-oidc, type: A, values: [192.168.10.127] }
+      - { zone: ad.ddupan.top, name: spire-server, type: A, values: [192.168.10.127] }
       - { zone: ad.ddupan.top, name: zot, type: A, values: [192.168.10.127] }
+      - { zone: ad.ddupan.top, name: zot-push, type: A, values: [192.168.10.127] }
 
   split_horizon:
-    # LAN and pod resolvers should eventually render the same set from here.
+    # backends records the current adoption boundary. obj is deliberately not
-    # Adoption of Blocky/CoreDNS is deliberately a separate change.
+    # emitted to CoreDNS yet, preserving the current pod resolver behaviour.
     records:
-      - { name: git.ddupan.top, type: A, values: [192.168.10.127] }
+      - { name: git.ddupan.top, type: A, values: [192.168.10.127], backends: [blocky, coredns] }
-      - { name: auth.ddupan.top, type: A, values: [192.168.10.127] }
+      - { name: auth.ddupan.top, type: A, values: [192.168.10.127], backends: [blocky, coredns] }
-      - { name: obj.ddupan.top, type: A, values: [192.168.10.127] }
+      - { name: obj.ddupan.top, type: A, values: [192.168.10.127], backends: [blocky] }
 
   public:
     # Names expected at Cloudflare. Terraform adoption is a separate change;
@@ -32,15 +43,36 @@ homelab_dns:
         type: CNAME
         values: [ff392451-b0b1-45bb-964e-6d9372c3a9e3.cfargotunnel.com]
         proxied: true
+        terraform:
+          managed: true
+          resource_name: auth
       - name: git.ddupan.top
         type: CNAME
         values: [ff392451-b0b1-45bb-964e-6d9372c3a9e3.cfargotunnel.com]
         proxied: true
+        terraform: { managed: false }
       - name: obj.ddupan.top
         type: CNAME
         values: [ff392451-b0b1-45bb-964e-6d9372c3a9e3.cfargotunnel.com]
         proxied: true
+        terraform: { managed: false }
       - name: e5renew.ddupan.top
         type: CNAME
         values: [ff392451-b0b1-45bb-964e-6d9372c3a9e3.cfargotunnel.com]
         proxied: true
+        terraform: { managed: false }
+      # OCI 主机直接解析公网 IP，SSH 不经过 Cloudflare 代理。
+      - name: oci-arm.ddupan.top
+        type: A
+        values:
+          - 129.225.138.179
+        proxied: false
+        ttl: 300
+        terraform: { managed: false }
+      - name: oci-amd.ddupan.top
+        type: A
+        values:
+          - 129.225.176.134
+        proxied: false
+        ttl: 300
+        terraform: { managed: false }

infrastructure/dns/templates/blocky.yml.j2

@@ -0,0 +1,3 @@
+[% for record in records %]
+    [[ record.name ]]: [[ record['values'][0] ]]
+[% endfor %]