From 005d8341ed3fddb30a871c450b83a2dc1e8592db Mon Sep 17 00:00:00 2001 From: panxiao81 Date: Fri, 25 Sep 2026 20:02:22 +0000 Subject: [PATCH] =?UTF-8?q?=E5=87=86=E5=A4=87=20OpenBao=20=E5=8F=97?= =?UTF-8?q?=E9=89=B4=E6=9D=83=E7=9B=91=E6=8E=A7=E4=B8=8E=E5=BF=AB=E7=85=A7?= =?UTF-8?q?=E7=BB=AD=E6=9C=9F=E4=BF=AE=E5=A4=8D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../ansible/roles/openbao/defaults/main.yml | 3 + .../roles/openbao/templates/config.hcl.j2 | 6 ++ .../roles/openbao_bootstrap/defaults/main.yml | 3 + .../openbao_bootstrap/tasks/snapshots.yml | 6 +- .../templates/bao-snapshot.sh.j2 | 11 ++- .../openbao/terraform/monitoring.tf | 15 ++++ .../openbao/terraform/policies/metrics.hcl | 4 + .../observability/metrics/kustomization.yaml | 2 + .../observability/metrics/rules/openbao.yaml | 18 +++++ .../metrics/scrapes/openbao.yaml | 63 +++++++++++++++ platform/observability/metrics/vmagent.yaml | 81 +++++++++++++++++-- 11 files changed, 202 insertions(+), 10 deletions(-) create mode 100644 infrastructure/openbao/terraform/monitoring.tf create mode 100644 infrastructure/openbao/terraform/policies/metrics.hcl create mode 100644 platform/observability/metrics/rules/openbao.yaml create mode 100644 platform/observability/metrics/scrapes/openbao.yaml diff --git a/infrastructure/openbao/ansible/roles/openbao/defaults/main.yml b/infrastructure/openbao/ansible/roles/openbao/defaults/main.yml index 6bcfb6b..9758900 100644 --- a/infrastructure/openbao/ansible/roles/openbao/defaults/main.yml +++ b/infrastructure/openbao/ansible/roles/openbao/defaults/main.yml @@ -47,3 +47,6 @@ openbao_transit_mount_path: "transit/" openbao_manage_firewall: false openbao_allowed_cidrs: - "100.64.0.0/10" + +# 30 秒采集间隔下保留五分钟，指标端点继续要求鉴权。 +openbao_prometheus_retention_time: "5m" diff --git a/infrastructure/openbao/ansible/roles/openbao/templates/config.hcl.j2 b/infrastructure/openbao/ansible/roles/openbao/templates/config.hcl.j2 index 7c880b6..5beabee 100644 --- a/infrastructure/openbao/ansible/roles/openbao/templates/config.hcl.j2 +++ b/infrastructure/openbao/ansible/roles/openbao/templates/config.hcl.j2 @@ -25,3 +25,9 @@ seal "transit" { mount_path = "{{ openbao_transit_mount_path }}" } {% endif %} + +# 显式固定 Prometheus 行为；应用此配置会经 handler 重启，须人工解封。 +telemetry { + prometheus_retention_time = "{{ openbao_prometheus_retention_time }}" + disable_hostname = true +} diff --git a/infrastructure/openbao/ansible/roles/openbao_bootstrap/defaults/main.yml b/infrastructure/openbao/ansible/roles/openbao_bootstrap/defaults/main.yml index b3a4f0f..ea91f50 100644 --- a/infrastructure/openbao/ansible/roles/openbao_bootstrap/defaults/main.yml +++ b/infrastructure/openbao/ansible/roles/openbao_bootstrap/defaults/main.yml @@ -96,3 +96,6 @@ openbao_snapshot_oncalendar: "*-*-* 02:00:00" # 3. terraform apply # mounts, roles, policies # 4. ansible-playbook bootstrap-openbao.yml # CA material, OIDC secret, snapshots openbao_config_managed_by_terraform: true + +# 仅在维护窗口显式启用，用于替换已失效的快照 token；不读取/打印 token。 +openbao_snapshot_rotate_token: false diff --git a/infrastructure/openbao/ansible/roles/openbao_bootstrap/tasks/snapshots.yml b/infrastructure/openbao/ansible/roles/openbao_bootstrap/tasks/snapshots.yml index 2624927..a6cf034 100644 --- a/infrastructure/openbao/ansible/roles/openbao_bootstrap/tasks/snapshots.yml +++ b/infrastructure/openbao/ansible/roles/openbao_bootstrap/tasks/snapshots.yml @@ -17,11 +17,11 @@ path: /etc/openbao/snapshot.token register: snap_tok_stat -- name: Create a periodic snapshot token (once) +- name: Create or explicitly rotate the periodic snapshot token ansible.builtin.command: "bao token create -policy=snapshot -period=768h -orphan -field=token" environment: "{{ openbao_cli_env }}" register: snap_tok_new - when: not snap_tok_stat.stat.exists + when: (not snap_tok_stat.stat.exists) or (openbao_snapshot_rotate_token | bool) changed_when: snap_tok_new.rc == 0 no_log: true @@ -32,7 +32,7 @@ owner: root group: root mode: "0600" - when: not snap_tok_stat.stat.exists + when: (not snap_tok_stat.stat.exists) or (openbao_snapshot_rotate_token | bool) no_log: true - name: Ensure the snapshot output directory exists diff --git a/infrastructure/openbao/ansible/roles/openbao_bootstrap/templates/bao-snapshot.sh.j2 b/infrastructure/openbao/ansible/roles/openbao_bootstrap/templates/bao-snapshot.sh.j2 index a399289..e7294ac 100644 --- a/infrastructure/openbao/ansible/roles/openbao_bootstrap/templates/bao-snapshot.sh.j2 +++ b/infrastructure/openbao/ansible/roles/openbao_bootstrap/templates/bao-snapshot.sh.j2 @@ -3,18 +3,25 @@ # Take a Raft snapshot and prune old ones. Ship {{ openbao_snapshot_dir }} off-box # separately (rsync/restic/scp) — a snapshot on the same host is not a backup. set -euo pipefail +umask 077 export BAO_ADDR="{{ openbao_addr }}" export BAO_CACERT="{{ openbao_tls_dir }}/cert.pem" BAO_TOKEN="$(cat /etc/openbao/snapshot.token)" export BAO_TOKEN +# periodic token 不会自动续期；在每日快照前续期，避免 768h 后永久失败。 +bao token renew -self >/dev/null + dir="{{ openbao_snapshot_dir }}" stamp="$(date +%Y%m%d-%H%M%S)" out="${dir}/openbao-${stamp}.snap" -bao operator raft snapshot save "${out}" -chmod 600 "${out}" +partial="${out}.partial" +trap 'rm -f -- "$partial"' EXIT +bao operator raft snapshot save "${partial}" +chmod 600 "${partial}" +mv -- "${partial}" "${out}" # Retention: keep the newest {{ openbao_snapshot_keep }}. ls -1t "${dir}"/openbao-*.snap 2>/dev/null | tail -n +{{ openbao_snapshot_keep + 1 }} | xargs -r rm -f diff --git a/infrastructure/openbao/terraform/monitoring.tf b/infrastructure/openbao/terraform/monitoring.tf new file mode 100644 index 0000000..4a99e5f --- /dev/null +++ b/infrastructure/openbao/terraform/monitoring.tf @@ -0,0 +1,15 @@ +# 指标只读身份；vmagent 的 Bao Agent sidecar 使用 Pod SA 自动登录与续期。 +resource "vault_policy" "metrics" { + name = "metrics" + policy = file("${path.module}/policies/metrics.hcl") +} + +resource "vault_kubernetes_auth_backend_role" "metrics" { + backend = "kubernetes" + role_name = "metrics" + bound_service_account_names = ["vmagent-main"] + bound_service_account_namespaces = ["monitoring"] + token_policies = [vault_policy.metrics.name] + token_ttl = 900 + token_max_ttl = 3600 +} diff --git a/infrastructure/openbao/terraform/policies/metrics.hcl b/infrastructure/openbao/terraform/policies/metrics.hcl new file mode 100644 index 0000000..24d4b30 --- /dev/null +++ b/infrastructure/openbao/terraform/policies/metrics.hcl @@ -0,0 +1,4 @@ +# 不含秘密读取或管理权限；默认 policy 只用于自身 token 续期等通用能力。 +path "sys/metrics" { + capabilities = ["read"] +} diff --git a/platform/observability/metrics/kustomization.yaml b/platform/observability/metrics/kustomization.yaml index 2ad314a..55b8d52 100644 --- a/platform/observability/metrics/kustomization.yaml +++ b/platform/observability/metrics/kustomization.yaml @@ -30,3 +30,5 @@ resources: - scrapes/cnpg.yaml - scrapes/seaweedfs.yaml - rules/data-services.yaml + - scrapes/openbao.yaml + - rules/openbao.yaml diff --git a/platform/observability/metrics/rules/openbao.yaml b/platform/observability/metrics/rules/openbao.yaml new file mode 100644 index 0000000..c1b0d29 --- /dev/null +++ b/platform/observability/metrics/rules/openbao.yaml @@ -0,0 +1,18 @@ +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: openbao + namespace: monitoring +spec: + groups: + - name: openbao + rules: + - alert: OpenBaoMetricsUnavailable + expr: up{job="openbao"} == 0 or absent(up{job="openbao"}) + for: 3m + labels: + severity: critical + annotations: + summary: OpenBao 受鉴权指标采集不可用 + description: 检查 Bao 健康与 sealed 状态、采集 Agent 登录/续期、TLS 和目标发现；此告警不区分停机与鉴权失败。 + runbook_url: https://git.ddupan.top/panxiao81/homelab-wiki/src/branch/main/guides/openbao-monitoring-maintenance.md diff --git a/platform/observability/metrics/scrapes/openbao.yaml b/platform/observability/metrics/scrapes/openbao.yaml new file mode 100644 index 0000000..3b6e5ba --- /dev/null +++ b/platform/observability/metrics/scrapes/openbao.yaml @@ -0,0 +1,63 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: bao-metrics-agent + namespace: monitoring +data: + config.hcl: "disable_mlock = true\nexit_after_auth = false\nvault {\n address =\ + \ \"https://bao.ad.ddupan.top:8200\"\n}\nauto_auth {\n method \"kubernetes\"\ + \ {\n mount_path = \"auth/kubernetes\"\n config = {\n role = \"metrics\"\ + \n token_path = \"/var/run/bao-jwt/token\"\n }\n }\n sink \"file\" {\n\ + \ config = {\n path = \"/var/run/bao-metrics/token\"\n mode = 0644\n\ + \ }\n }\n}\n" +--- +apiVersion: v1 +kind: Service +metadata: + name: openbao-metrics + namespace: monitoring + labels: + app.kubernetes.io/name: openbao-metrics +spec: + ports: + - name: https + port: 8200 + targetPort: 8200 +--- +apiVersion: v1 +kind: Endpoints +metadata: + name: openbao-metrics + namespace: monitoring +subsets: +- addresses: + - ip: 192.168.10.8 + ports: + - name: https + port: 8200 + protocol: TCP +--- +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: openbao + namespace: monitoring +spec: + selector: + matchLabels: + app.kubernetes.io/name: openbao-metrics + endpoints: + - port: https + scheme: https + path: /v1/sys/metrics + params: + format: + - prometheus + interval: 30s + scrapeTimeout: 10s + bearerTokenFile: /var/run/bao-metrics/token + tlsConfig: + serverName: bao.ad.ddupan.top + relabelings: + - targetLabel: job + replacement: openbao diff --git a/platform/observability/metrics/vmagent.yaml b/platform/observability/metrics/vmagent.yaml index 5bd3bde..b9f8e21 100644 --- a/platform/observability/metrics/vmagent.yaml +++ b/platform/observability/metrics/vmagent.yaml @@ -1,6 +1,3 @@ -# Scrape agent. selectAllByDefault picks up every VMServiceScrape / VMPodScrape / -# VMNodeScrape / VMStaticScrape in ALL namespaces, so app teams add their own -# scrape CRs and vmagent discovers them automatically. Writes to VMSingle. apiVersion: operator.victoriametrics.com/v1beta1 kind: VMAgent metadata: @@ -11,11 +8,85 @@ spec: selectAllByDefault: true scrapeInterval: 30s remoteWrite: - - url: http://vmsingle-main.monitoring.svc:8428/api/v1/write + - url: http://vmsingle-main.monitoring.svc:8428/api/v1/write resources: requests: cpu: 100m memory: 256Mi limits: - cpu: "1" + cpu: '1' memory: 1Gi + volumes: + - name: bao-agent-config + configMap: + name: bao-metrics-agent + - name: bao-metrics-token + emptyDir: + medium: Memory + - name: bao-metrics-jwt + projected: + sources: + - serviceAccountToken: + path: token + expirationSeconds: 3600 + volumeMounts: + - name: bao-metrics-token + mountPath: /var/run/bao-metrics + readOnly: true + containers: + - name: bao-metrics-agent + image: quay.io/openbao/openbao:2.6.1 + command: + - bao + args: + - agent + - -config=/etc/bao-agent/config.hcl + env: + - name: HOME + value: /tmp + securityContext: + runAsNonRoot: true + runAsUser: 65534 + runAsGroup: 65534 + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: + - ALL + resources: + requests: + cpu: 10m + memory: 32Mi + limits: + cpu: 100m + memory: 128Mi + volumeMounts: + - name: bao-agent-config + mountPath: /etc/bao-agent + readOnly: true + - name: bao-metrics-token + mountPath: /var/run/bao-metrics + - name: bao-metrics-jwt + mountPath: /var/run/bao-jwt + readOnly: true + securityContext: + fsGroup: 65534 + initContainers: + - name: prepare-bao-token-file + image: quay.io/openbao/openbao:2.6.1 + command: + - sh + - -ec + - 'umask 0022; : > /var/run/bao-metrics/token' + securityContext: + runAsNonRoot: true + runAsUser: 65534 + runAsGroup: 65534 + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: + - ALL + volumeMounts: + - name: bao-metrics-token + mountPath: /var/run/bao-metrics