接入 OpenBao 受鉴权监控并修复快照续期 (#162)

Co-authored-by: panxiao81 <[email protected]>

platform/observability/metrics/kustomization.yaml

@@ -30,3 +30,5 @@ resources:
   - scrapes/cnpg.yaml
   - scrapes/seaweedfs.yaml
   - rules/data-services.yaml
+  - scrapes/openbao.yaml
+  - rules/openbao.yaml

platform/observability/metrics/rules/openbao.yaml

@@ -0,0 +1,18 @@
+apiVersion: monitoring.coreos.com/v1
+kind: PrometheusRule
+metadata:
+  name: openbao
+  namespace: monitoring
+spec:
+  groups:
+  - name: openbao
+    rules:
+    - alert: OpenBaoMetricsUnavailable
+      expr: up{job="openbao"} == 0 or absent(up{job="openbao"})
+      for: 3m
+      labels:
+        severity: critical
+      annotations:
+        summary: OpenBao 受鉴权指标采集不可用
+        description: 检查 Bao 健康与 sealed 状态、采集 Agent 登录/续期、TLS 和目标发现；此告警不区分停机与鉴权失败。
+        runbook_url: https://git.ddupan.top/panxiao81/homelab-wiki/src/branch/main/guides/openbao-monitoring-maintenance.md

platform/observability/metrics/scrapes/openbao.yaml

@@ -0,0 +1,63 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: bao-metrics-agent
+  namespace: monitoring
+data:
+  config.hcl: "disable_mlock = true\nexit_after_auth = false\nvault {\n  address =\
+    \ \"https://bao.ad.ddupan.top:8200\"\n}\nauto_auth {\n  method \"kubernetes\"\
+    \ {\n    mount_path = \"auth/kubernetes\"\n    config = {\n      role = \"metrics\"\
+    \n      token_path = \"/var/run/bao-jwt/token\"\n    }\n  }\n  sink \"file\" {\n\
+    \    config = {\n      path = \"/var/run/bao-metrics/token\"\n      mode = 0644\n\
+    \    }\n  }\n}\n"
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: openbao-metrics
+  namespace: monitoring
+  labels:
+    app.kubernetes.io/name: openbao-metrics
+spec:
+  ports:
+  - name: https
+    port: 8200
+    targetPort: 8200
+---
+apiVersion: v1
+kind: Endpoints
+metadata:
+  name: openbao-metrics
+  namespace: monitoring
+subsets:
+- addresses:
+  - ip: 192.168.10.8
+  ports:
+  - name: https
+    port: 8200
+    protocol: TCP
+---
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  name: openbao
+  namespace: monitoring
+spec:
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: openbao-metrics
+  endpoints:
+  - port: https
+    scheme: https
+    path: /v1/sys/metrics
+    params:
+      format:
+      - prometheus
+    interval: 30s
+    scrapeTimeout: 10s
+    bearerTokenFile: /var/run/bao-metrics/token
+    tlsConfig:
+      serverName: bao.ad.ddupan.top
+    relabelings:
+    - targetLabel: job
+      replacement: openbao

platform/observability/metrics/vmagent.yaml

@@ -1,6 +1,3 @@
-# Scrape agent. selectAllByDefault picks up every VMServiceScrape / VMPodScrape /
-# VMNodeScrape / VMStaticScrape in ALL namespaces, so app teams add their own
-# scrape CRs and vmagent discovers them automatically. Writes to VMSingle.
 apiVersion: operator.victoriametrics.com/v1beta1
 kind: VMAgent
 metadata:
@@ -11,11 +8,85 @@ spec:
   selectAllByDefault: true
   scrapeInterval: 30s
   remoteWrite:
-    - url: http://vmsingle-main.monitoring.svc:8428/api/v1/write
+  - url: http://vmsingle-main.monitoring.svc:8428/api/v1/write
   resources:
     requests:
       cpu: 100m
       memory: 256Mi
     limits:
-      cpu: "1"
+      cpu: '1'
       memory: 1Gi
+  volumes:
+  - name: bao-agent-config
+    configMap:
+      name: bao-metrics-agent
+  - name: bao-metrics-token
+    emptyDir:
+      medium: Memory
+  - name: bao-metrics-jwt
+    projected:
+      sources:
+      - serviceAccountToken:
+          path: token
+          expirationSeconds: 3600
+  volumeMounts:
+  - name: bao-metrics-token
+    mountPath: /var/run/bao-metrics
+    readOnly: true
+  containers:
+  - name: bao-metrics-agent
+    image: quay.io/openbao/openbao:2.6.1
+    command:
+    - bao
+    args:
+    - agent
+    - -config=/etc/bao-agent/config.hcl
+    env:
+    - name: HOME
+      value: /tmp
+    securityContext:
+      runAsNonRoot: true
+      runAsUser: 65534
+      runAsGroup: 65534
+      allowPrivilegeEscalation: false
+      readOnlyRootFilesystem: true
+      capabilities:
+        drop:
+        - ALL
+    resources:
+      requests:
+        cpu: 10m
+        memory: 32Mi
+      limits:
+        cpu: 100m
+        memory: 128Mi
+    volumeMounts:
+    - name: bao-agent-config
+      mountPath: /etc/bao-agent
+      readOnly: true
+    - name: bao-metrics-token
+      mountPath: /var/run/bao-metrics
+    - name: bao-metrics-jwt
+      mountPath: /var/run/bao-jwt
+      readOnly: true
+  securityContext:
+    fsGroup: 65534
+  initContainers:
+  - name: prepare-bao-token-file
+    image: quay.io/openbao/openbao:2.6.1
+    command:
+    - sh
+    - -ec
+    - 'umask 0022; : > /var/run/bao-metrics/token'
+    securityContext:
+      runAsNonRoot: true
+      runAsUser: 65534
+      runAsGroup: 65534
+      allowPrivilegeEscalation: false
+      readOnlyRootFilesystem: true
+      capabilities:
+        drop:
+        - ALL
+    volumeMounts:
+    - name: bao-metrics-token
+      mountPath: /var/run/bao-metrics