接入 OpenBao 受鉴权监控并修复快照续期 (#162)

Co-authored-by: panxiao81 <[email protected]>

infrastructure/openbao/ansible/roles/openbao/defaults/main.yml

@@ -47,3 +47,6 @@ openbao_transit_mount_path: "transit/"
 openbao_manage_firewall: false
 openbao_allowed_cidrs:
   - "100.64.0.0/10"
+
+# 30 秒采集间隔下保留五分钟，指标端点继续要求鉴权。
+openbao_prometheus_retention_time: "5m"

infrastructure/openbao/ansible/roles/openbao/templates/config.hcl.j2

@@ -25,3 +25,9 @@ seal "transit" {
   mount_path = "{{ openbao_transit_mount_path }}"
 }
 {% endif %}
+
+# 显式固定 Prometheus 行为；应用此配置会经 handler 重启，须人工解封。
+telemetry {
+  prometheus_retention_time = "{{ openbao_prometheus_retention_time }}"
+  disable_hostname = true
+}

infrastructure/openbao/ansible/roles/openbao_bootstrap/defaults/main.yml

@@ -96,3 +96,6 @@ openbao_snapshot_oncalendar: "*-*-* 02:00:00"
 #   3. terraform apply                          # mounts, roles, policies
 #   4. ansible-playbook bootstrap-openbao.yml   # CA material, OIDC secret, snapshots
 openbao_config_managed_by_terraform: true
+
+# 仅在维护窗口显式启用，用于替换已失效的快照 token；不读取/打印 token。
+openbao_snapshot_rotate_token: false

infrastructure/openbao/ansible/roles/openbao_bootstrap/tasks/snapshots.yml

@@ -17,11 +17,11 @@
     path: /etc/openbao/snapshot.token
   register: snap_tok_stat
 
-- name: Create a periodic snapshot token (once)
+- name: Create or explicitly rotate the periodic snapshot token
   ansible.builtin.command: "bao token create -policy=snapshot -period=768h -orphan -field=token"
   environment: "{{ openbao_cli_env }}"
   register: snap_tok_new
-  when: not snap_tok_stat.stat.exists
+  when: (not snap_tok_stat.stat.exists) or (openbao_snapshot_rotate_token | bool)
   changed_when: snap_tok_new.rc == 0
   no_log: true
 
@@ -32,7 +32,7 @@
     owner: root
     group: root
     mode: "0600"
-  when: not snap_tok_stat.stat.exists
+  when: (not snap_tok_stat.stat.exists) or (openbao_snapshot_rotate_token | bool)
   no_log: true
 
 - name: Ensure the snapshot output directory exists

infrastructure/openbao/ansible/roles/openbao_bootstrap/templates/bao-snapshot.sh.j2

@@ -3,18 +3,25 @@
 # Take a Raft snapshot and prune old ones. Ship {{ openbao_snapshot_dir }} off-box
 # separately (rsync/restic/scp) — a snapshot on the same host is not a backup.
 set -euo pipefail
+umask 077
 
 export BAO_ADDR="{{ openbao_addr }}"
 export BAO_CACERT="{{ openbao_tls_dir }}/cert.pem"
 BAO_TOKEN="$(cat /etc/openbao/snapshot.token)"
 export BAO_TOKEN
 
+# periodic token 不会自动续期；在每日快照前续期，避免 768h 后永久失败。
+bao token renew >/dev/null
+
 dir="{{ openbao_snapshot_dir }}"
 stamp="$(date +%Y%m%d-%H%M%S)"
 out="${dir}/openbao-${stamp}.snap"
 
-bao operator raft snapshot save "${out}"
+partial="${out}.partial"
-chmod 600 "${out}"
+trap 'rm -f -- "$partial"' EXIT
+bao operator raft snapshot save "${partial}"
+chmod 600 "${partial}"
+mv -- "${partial}" "${out}"
 
 # Retention: keep the newest {{ openbao_snapshot_keep }}.
 ls -1t "${dir}"/openbao-*.snap 2>/dev/null | tail -n +{{ openbao_snapshot_keep + 1 }} | xargs -r rm -f

infrastructure/openbao/terraform/imports.tf

@@ -68,3 +68,14 @@ import {
   to = vault_pki_secret_backend_config_urls.this
   id = "pki/config/urls"
 }
+
+# 本次维护预检先通过同名 API 配置验证，后续 Terraform 纳入既有主 state。
+import {
+  to = vault_policy.metrics
+  id = "metrics"
+}
+
+import {
+  to = vault_kubernetes_auth_backend_role.metrics
+  id = "auth/kubernetes/role/metrics"
+}

infrastructure/openbao/terraform/monitoring.tf

@@ -0,0 +1,15 @@
+# 指标只读身份；vmagent 的 Bao Agent sidecar 使用 Pod SA 自动登录与续期。
+resource "vault_policy" "metrics" {
+  name   = "metrics"
+  policy = file("${path.module}/policies/metrics.hcl")
+}
+
+resource "vault_kubernetes_auth_backend_role" "metrics" {
+  backend                          = "kubernetes"
+  role_name                        = "metrics"
+  bound_service_account_names      = ["vmagent-main"]
+  bound_service_account_namespaces = ["monitoring"]
+  token_policies                   = [vault_policy.metrics.name]
+  token_ttl                        = 900
+  token_max_ttl                    = 3600
+}

infrastructure/openbao/terraform/policies/metrics.hcl

@@ -0,0 +1,4 @@
+# 不含秘密读取或管理权限；默认 policy 只用于自身 token 续期等通用能力。
+path "sys/metrics" {
+  capabilities = ["read"]
+}

platform/observability/metrics/kustomization.yaml

@@ -30,3 +30,5 @@ resources:
   - scrapes/cnpg.yaml
   - scrapes/seaweedfs.yaml
   - rules/data-services.yaml
+  - scrapes/openbao.yaml
+  - rules/openbao.yaml

platform/observability/metrics/rules/openbao.yaml

@@ -0,0 +1,18 @@
+apiVersion: monitoring.coreos.com/v1
+kind: PrometheusRule
+metadata:
+  name: openbao
+  namespace: monitoring
+spec:
+  groups:
+  - name: openbao
+    rules:
+    - alert: OpenBaoMetricsUnavailable
+      expr: up{job="openbao"} == 0 or absent(up{job="openbao"})
+      for: 3m
+      labels:
+        severity: critical
+      annotations:
+        summary: OpenBao 受鉴权指标采集不可用
+        description: 检查 Bao 健康与 sealed 状态、采集 Agent 登录/续期、TLS 和目标发现；此告警不区分停机与鉴权失败。
+        runbook_url: https://git.ddupan.top/panxiao81/homelab-wiki/src/branch/main/guides/openbao-monitoring-maintenance.md

platform/observability/metrics/scrapes/openbao.yaml

@@ -0,0 +1,63 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: bao-metrics-agent
+  namespace: monitoring
+data:
+  config.hcl: "disable_mlock = true\nexit_after_auth = false\nvault {\n  address =\
+    \ \"https://bao.ad.ddupan.top:8200\"\n}\nauto_auth {\n  method \"kubernetes\"\
+    \ {\n    mount_path = \"auth/kubernetes\"\n    config = {\n      role = \"metrics\"\
+    \n      token_path = \"/var/run/bao-jwt/token\"\n    }\n  }\n  sink \"file\" {\n\
+    \    config = {\n      path = \"/var/run/bao-metrics/token\"\n      mode = 0644\n\
+    \    }\n  }\n}\n"
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: openbao-metrics
+  namespace: monitoring
+  labels:
+    app.kubernetes.io/name: openbao-metrics
+spec:
+  ports:
+  - name: https
+    port: 8200
+    targetPort: 8200
+---
+apiVersion: v1
+kind: Endpoints
+metadata:
+  name: openbao-metrics
+  namespace: monitoring
+subsets:
+- addresses:
+  - ip: 192.168.10.8
+  ports:
+  - name: https
+    port: 8200
+    protocol: TCP
+---
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  name: openbao
+  namespace: monitoring
+spec:
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: openbao-metrics
+  endpoints:
+  - port: https
+    scheme: https
+    path: /v1/sys/metrics
+    params:
+      format:
+      - prometheus
+    interval: 30s
+    scrapeTimeout: 10s
+    bearerTokenFile: /var/run/bao-metrics/token
+    tlsConfig:
+      serverName: bao.ad.ddupan.top
+    relabelings:
+    - targetLabel: job
+      replacement: openbao

platform/observability/metrics/vmagent.yaml

@@ -1,6 +1,3 @@
-# Scrape agent. selectAllByDefault picks up every VMServiceScrape / VMPodScrape /
-# VMNodeScrape / VMStaticScrape in ALL namespaces, so app teams add their own
-# scrape CRs and vmagent discovers them automatically. Writes to VMSingle.
 apiVersion: operator.victoriametrics.com/v1beta1
 kind: VMAgent
 metadata:
@@ -11,11 +8,85 @@ spec:
   selectAllByDefault: true
   scrapeInterval: 30s
   remoteWrite:
-    - url: http://vmsingle-main.monitoring.svc:8428/api/v1/write
+  - url: http://vmsingle-main.monitoring.svc:8428/api/v1/write
   resources:
     requests:
       cpu: 100m
       memory: 256Mi
     limits:
-      cpu: "1"
+      cpu: '1'
       memory: 1Gi
+  volumes:
+  - name: bao-agent-config
+    configMap:
+      name: bao-metrics-agent
+  - name: bao-metrics-token
+    emptyDir:
+      medium: Memory
+  - name: bao-metrics-jwt
+    projected:
+      sources:
+      - serviceAccountToken:
+          path: token
+          expirationSeconds: 3600
+  volumeMounts:
+  - name: bao-metrics-token
+    mountPath: /var/run/bao-metrics
+    readOnly: true
+  containers:
+  - name: bao-metrics-agent
+    image: quay.io/openbao/openbao:2.6.1
+    command:
+    - bao
+    args:
+    - agent
+    - -config=/etc/bao-agent/config.hcl
+    env:
+    - name: HOME
+      value: /tmp
+    securityContext:
+      runAsNonRoot: true
+      runAsUser: 65534
+      runAsGroup: 65534
+      allowPrivilegeEscalation: false
+      readOnlyRootFilesystem: true
+      capabilities:
+        drop:
+        - ALL
+    resources:
+      requests:
+        cpu: 10m
+        memory: 32Mi
+      limits:
+        cpu: 100m
+        memory: 128Mi
+    volumeMounts:
+    - name: bao-agent-config
+      mountPath: /etc/bao-agent
+      readOnly: true
+    - name: bao-metrics-token
+      mountPath: /var/run/bao-metrics
+    - name: bao-metrics-jwt
+      mountPath: /var/run/bao-jwt
+      readOnly: true
+  securityContext:
+    fsGroup: 65534
+  initContainers:
+  - name: prepare-bao-token-file
+    image: quay.io/openbao/openbao:2.6.1
+    command:
+    - sh
+    - -ec
+    - 'umask 0022; : > /var/run/bao-metrics/token'
+    securityContext:
+      runAsNonRoot: true
+      runAsUser: 65534
+      runAsGroup: 65534
+      allowPrivilegeEscalation: false
+      readOnlyRootFilesystem: true
+      capabilities:
+        drop:
+        - ALL
+    volumeMounts:
+    - name: bao-metrics-token
+      mountPath: /var/run/bao-metrics