接入 OpenBao 受鉴权监控并修复快照续期 #162

Merged
panxiao81 merged 2 commits from feat/openbao-monitoring into main 2026-09-25 20:39:32 +00:00
11 changed files with 202 additions and 10 deletions
Showing only changes of commit 005d8341ed - Show all commits
@@ -47,3 +47,6 @@ openbao_transit_mount_path: "transit/"
openbao_manage_firewall: false
openbao_allowed_cidrs:
- "100.64.0.0/10"
# 30 秒采集间隔下保留五分钟,指标端点继续要求鉴权。
openbao_prometheus_retention_time: "5m"
@@ -25,3 +25,9 @@ seal "transit" {
mount_path = "{{ openbao_transit_mount_path }}"
}
{% endif %}
# 显式固定 Prometheus 行为;应用此配置会经 handler 重启,须人工解封。
telemetry {
prometheus_retention_time = "{{ openbao_prometheus_retention_time }}"
disable_hostname = true
}
@@ -96,3 +96,6 @@ openbao_snapshot_oncalendar: "*-*-* 02:00:00"
# 3. terraform apply # mounts, roles, policies
# 4. ansible-playbook bootstrap-openbao.yml # CA material, OIDC secret, snapshots
openbao_config_managed_by_terraform: true
# 仅在维护窗口显式启用,用于替换已失效的快照 token;不读取/打印 token。
openbao_snapshot_rotate_token: false
@@ -17,11 +17,11 @@
path: /etc/openbao/snapshot.token
register: snap_tok_stat
- name: Create a periodic snapshot token (once)
- name: Create or explicitly rotate the periodic snapshot token
ansible.builtin.command: "bao token create -policy=snapshot -period=768h -orphan -field=token"
environment: "{{ openbao_cli_env }}"
register: snap_tok_new
when: not snap_tok_stat.stat.exists
when: (not snap_tok_stat.stat.exists) or (openbao_snapshot_rotate_token | bool)
changed_when: snap_tok_new.rc == 0
no_log: true
@@ -32,7 +32,7 @@
owner: root
group: root
mode: "0600"
when: not snap_tok_stat.stat.exists
when: (not snap_tok_stat.stat.exists) or (openbao_snapshot_rotate_token | bool)
no_log: true
- name: Ensure the snapshot output directory exists
@@ -3,18 +3,25 @@
# Take a Raft snapshot and prune old ones. Ship {{ openbao_snapshot_dir }} off-box
# separately (rsync/restic/scp) — a snapshot on the same host is not a backup.
set -euo pipefail
umask 077
export BAO_ADDR="{{ openbao_addr }}"
export BAO_CACERT="{{ openbao_tls_dir }}/cert.pem"
BAO_TOKEN="$(cat /etc/openbao/snapshot.token)"
export BAO_TOKEN
# periodic token 不会自动续期;在每日快照前续期,避免 768h 后永久失败。
bao token renew -self >/dev/null
dir="{{ openbao_snapshot_dir }}"
stamp="$(date +%Y%m%d-%H%M%S)"
out="${dir}/openbao-${stamp}.snap"
bao operator raft snapshot save "${out}"
chmod 600 "${out}"
partial="${out}.partial"
trap 'rm -f -- "$partial"' EXIT
bao operator raft snapshot save "${partial}"
chmod 600 "${partial}"
mv -- "${partial}" "${out}"
# Retention: keep the newest {{ openbao_snapshot_keep }}.
ls -1t "${dir}"/openbao-*.snap 2>/dev/null | tail -n +{{ openbao_snapshot_keep + 1 }} | xargs -r rm -f
@@ -0,0 +1,15 @@
# 指标只读身份;vmagent 的 Bao Agent sidecar 使用 Pod SA 自动登录与续期。
resource "vault_policy" "metrics" {
name = "metrics"
policy = file("${path.module}/policies/metrics.hcl")
}
resource "vault_kubernetes_auth_backend_role" "metrics" {
backend = "kubernetes"
role_name = "metrics"
bound_service_account_names = ["vmagent-main"]
bound_service_account_namespaces = ["monitoring"]
token_policies = [vault_policy.metrics.name]
token_ttl = 900
token_max_ttl = 3600
}
@@ -0,0 +1,4 @@
# 不含秘密读取或管理权限;默认 policy 只用于自身 token 续期等通用能力。
path "sys/metrics" {
capabilities = ["read"]
}
@@ -30,3 +30,5 @@ resources:
- scrapes/cnpg.yaml
- scrapes/seaweedfs.yaml
- rules/data-services.yaml
- scrapes/openbao.yaml
- rules/openbao.yaml
@@ -0,0 +1,18 @@
apiVersion: monitoring.coreos.com/v1
kind: PrometheusRule
metadata:
name: openbao
namespace: monitoring
spec:
groups:
- name: openbao
rules:
- alert: OpenBaoMetricsUnavailable
expr: up{job="openbao"} == 0 or absent(up{job="openbao"})
for: 3m
labels:
severity: critical
annotations:
summary: OpenBao 受鉴权指标采集不可用
description: 检查 Bao 健康与 sealed 状态、采集 Agent 登录/续期、TLS 和目标发现;此告警不区分停机与鉴权失败。
runbook_url: https://git.ddupan.top/panxiao81/homelab-wiki/src/branch/main/guides/openbao-monitoring-maintenance.md
@@ -0,0 +1,63 @@
apiVersion: v1
kind: ConfigMap
metadata:
name: bao-metrics-agent
namespace: monitoring
data:
config.hcl: "disable_mlock = true\nexit_after_auth = false\nvault {\n address =\
\ \"https://bao.ad.ddupan.top:8200\"\n}\nauto_auth {\n method \"kubernetes\"\
\ {\n mount_path = \"auth/kubernetes\"\n config = {\n role = \"metrics\"\
\n token_path = \"/var/run/bao-jwt/token\"\n }\n }\n sink \"file\" {\n\
\ config = {\n path = \"/var/run/bao-metrics/token\"\n mode = 0644\n\
\ }\n }\n}\n"
---
apiVersion: v1
kind: Service
metadata:
name: openbao-metrics
namespace: monitoring
labels:
app.kubernetes.io/name: openbao-metrics
spec:
ports:
- name: https
port: 8200
targetPort: 8200
---
apiVersion: v1
kind: Endpoints
metadata:
name: openbao-metrics
namespace: monitoring
subsets:
- addresses:
- ip: 192.168.10.8
ports:
- name: https
port: 8200
protocol: TCP
---
apiVersion: monitoring.coreos.com/v1
kind: ServiceMonitor
metadata:
name: openbao
namespace: monitoring
spec:
selector:
matchLabels:
app.kubernetes.io/name: openbao-metrics
endpoints:
- port: https
scheme: https
path: /v1/sys/metrics
params:
format:
- prometheus
interval: 30s
scrapeTimeout: 10s
bearerTokenFile: /var/run/bao-metrics/token
tlsConfig:
serverName: bao.ad.ddupan.top
relabelings:
- targetLabel: job
replacement: openbao
+75 -4
View File
@@ -1,6 +1,3 @@
# Scrape agent. selectAllByDefault picks up every VMServiceScrape / VMPodScrape /
# VMNodeScrape / VMStaticScrape in ALL namespaces, so app teams add their own
# scrape CRs and vmagent discovers them automatically. Writes to VMSingle.
apiVersion: operator.victoriametrics.com/v1beta1
kind: VMAgent
metadata:
@@ -17,5 +14,79 @@ spec:
cpu: 100m
memory: 256Mi
limits:
cpu: "1"
cpu: '1'
memory: 1Gi
volumes:
- name: bao-agent-config
configMap:
name: bao-metrics-agent
- name: bao-metrics-token
emptyDir:
medium: Memory
- name: bao-metrics-jwt
projected:
sources:
- serviceAccountToken:
path: token
expirationSeconds: 3600
volumeMounts:
- name: bao-metrics-token
mountPath: /var/run/bao-metrics
readOnly: true
containers:
- name: bao-metrics-agent
image: quay.io/openbao/openbao:2.6.1
command:
- bao
args:
- agent
- -config=/etc/bao-agent/config.hcl
env:
- name: HOME
value: /tmp
securityContext:
runAsNonRoot: true
runAsUser: 65534
runAsGroup: 65534
allowPrivilegeEscalation: false
readOnlyRootFilesystem: true
capabilities:
drop:
- ALL
resources:
requests:
cpu: 10m
memory: 32Mi
limits:
cpu: 100m
memory: 128Mi
volumeMounts:
- name: bao-agent-config
mountPath: /etc/bao-agent
readOnly: true
- name: bao-metrics-token
mountPath: /var/run/bao-metrics
- name: bao-metrics-jwt
mountPath: /var/run/bao-jwt
readOnly: true
securityContext:
fsGroup: 65534
initContainers:
- name: prepare-bao-token-file
image: quay.io/openbao/openbao:2.6.1
command:
- sh
- -ec
- 'umask 0022; : > /var/run/bao-metrics/token'
securityContext:
runAsNonRoot: true
runAsUser: 65534
runAsGroup: 65534
allowPrivilegeEscalation: false
readOnlyRootFilesystem: true
capabilities:
drop:
- ALL
volumeMounts:
- name: bao-metrics-token
mountPath: /var/run/bao-metrics