接入 OpenBao 受鉴权监控并修复快照续期 #162
@@ -47,3 +47,6 @@ openbao_transit_mount_path: "transit/"
|
||||
openbao_manage_firewall: false
|
||||
openbao_allowed_cidrs:
|
||||
- "100.64.0.0/10"
|
||||
|
||||
# 30 秒采集间隔下保留五分钟,指标端点继续要求鉴权。
|
||||
openbao_prometheus_retention_time: "5m"
|
||||
|
||||
@@ -25,3 +25,9 @@ seal "transit" {
|
||||
mount_path = "{{ openbao_transit_mount_path }}"
|
||||
}
|
||||
{% endif %}
|
||||
|
||||
# 显式固定 Prometheus 行为;应用此配置会经 handler 重启,须人工解封。
|
||||
telemetry {
|
||||
prometheus_retention_time = "{{ openbao_prometheus_retention_time }}"
|
||||
disable_hostname = true
|
||||
}
|
||||
|
||||
@@ -96,3 +96,6 @@ openbao_snapshot_oncalendar: "*-*-* 02:00:00"
|
||||
# 3. terraform apply # mounts, roles, policies
|
||||
# 4. ansible-playbook bootstrap-openbao.yml # CA material, OIDC secret, snapshots
|
||||
openbao_config_managed_by_terraform: true
|
||||
|
||||
# 仅在维护窗口显式启用,用于替换已失效的快照 token;不读取/打印 token。
|
||||
openbao_snapshot_rotate_token: false
|
||||
|
||||
@@ -17,11 +17,11 @@
|
||||
path: /etc/openbao/snapshot.token
|
||||
register: snap_tok_stat
|
||||
|
||||
- name: Create a periodic snapshot token (once)
|
||||
- name: Create or explicitly rotate the periodic snapshot token
|
||||
ansible.builtin.command: "bao token create -policy=snapshot -period=768h -orphan -field=token"
|
||||
environment: "{{ openbao_cli_env }}"
|
||||
register: snap_tok_new
|
||||
when: not snap_tok_stat.stat.exists
|
||||
when: (not snap_tok_stat.stat.exists) or (openbao_snapshot_rotate_token | bool)
|
||||
changed_when: snap_tok_new.rc == 0
|
||||
no_log: true
|
||||
|
||||
@@ -32,7 +32,7 @@
|
||||
owner: root
|
||||
group: root
|
||||
mode: "0600"
|
||||
when: not snap_tok_stat.stat.exists
|
||||
when: (not snap_tok_stat.stat.exists) or (openbao_snapshot_rotate_token | bool)
|
||||
no_log: true
|
||||
|
||||
- name: Ensure the snapshot output directory exists
|
||||
|
||||
@@ -3,18 +3,25 @@
|
||||
# Take a Raft snapshot and prune old ones. Ship {{ openbao_snapshot_dir }} off-box
|
||||
# separately (rsync/restic/scp) — a snapshot on the same host is not a backup.
|
||||
set -euo pipefail
|
||||
umask 077
|
||||
|
||||
export BAO_ADDR="{{ openbao_addr }}"
|
||||
export BAO_CACERT="{{ openbao_tls_dir }}/cert.pem"
|
||||
BAO_TOKEN="$(cat /etc/openbao/snapshot.token)"
|
||||
export BAO_TOKEN
|
||||
|
||||
# periodic token 不会自动续期;在每日快照前续期,避免 768h 后永久失败。
|
||||
bao token renew -self >/dev/null
|
||||
|
||||
dir="{{ openbao_snapshot_dir }}"
|
||||
stamp="$(date +%Y%m%d-%H%M%S)"
|
||||
out="${dir}/openbao-${stamp}.snap"
|
||||
|
||||
bao operator raft snapshot save "${out}"
|
||||
chmod 600 "${out}"
|
||||
partial="${out}.partial"
|
||||
trap 'rm -f -- "$partial"' EXIT
|
||||
bao operator raft snapshot save "${partial}"
|
||||
chmod 600 "${partial}"
|
||||
mv -- "${partial}" "${out}"
|
||||
|
||||
# Retention: keep the newest {{ openbao_snapshot_keep }}.
|
||||
ls -1t "${dir}"/openbao-*.snap 2>/dev/null | tail -n +{{ openbao_snapshot_keep + 1 }} | xargs -r rm -f
|
||||
|
||||
@@ -0,0 +1,15 @@
|
||||
# 指标只读身份;vmagent 的 Bao Agent sidecar 使用 Pod SA 自动登录与续期。
|
||||
resource "vault_policy" "metrics" {
|
||||
name = "metrics"
|
||||
policy = file("${path.module}/policies/metrics.hcl")
|
||||
}
|
||||
|
||||
resource "vault_kubernetes_auth_backend_role" "metrics" {
|
||||
backend = "kubernetes"
|
||||
role_name = "metrics"
|
||||
bound_service_account_names = ["vmagent-main"]
|
||||
bound_service_account_namespaces = ["monitoring"]
|
||||
token_policies = [vault_policy.metrics.name]
|
||||
token_ttl = 900
|
||||
token_max_ttl = 3600
|
||||
}
|
||||
@@ -0,0 +1,4 @@
|
||||
# 不含秘密读取或管理权限;默认 policy 只用于自身 token 续期等通用能力。
|
||||
path "sys/metrics" {
|
||||
capabilities = ["read"]
|
||||
}
|
||||
@@ -30,3 +30,5 @@ resources:
|
||||
- scrapes/cnpg.yaml
|
||||
- scrapes/seaweedfs.yaml
|
||||
- rules/data-services.yaml
|
||||
- scrapes/openbao.yaml
|
||||
- rules/openbao.yaml
|
||||
|
||||
@@ -0,0 +1,18 @@
|
||||
apiVersion: monitoring.coreos.com/v1
|
||||
kind: PrometheusRule
|
||||
metadata:
|
||||
name: openbao
|
||||
namespace: monitoring
|
||||
spec:
|
||||
groups:
|
||||
- name: openbao
|
||||
rules:
|
||||
- alert: OpenBaoMetricsUnavailable
|
||||
expr: up{job="openbao"} == 0 or absent(up{job="openbao"})
|
||||
for: 3m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: OpenBao 受鉴权指标采集不可用
|
||||
description: 检查 Bao 健康与 sealed 状态、采集 Agent 登录/续期、TLS 和目标发现;此告警不区分停机与鉴权失败。
|
||||
runbook_url: https://git.ddupan.top/panxiao81/homelab-wiki/src/branch/main/guides/openbao-monitoring-maintenance.md
|
||||
@@ -0,0 +1,63 @@
|
||||
apiVersion: v1
|
||||
kind: ConfigMap
|
||||
metadata:
|
||||
name: bao-metrics-agent
|
||||
namespace: monitoring
|
||||
data:
|
||||
config.hcl: "disable_mlock = true\nexit_after_auth = false\nvault {\n address =\
|
||||
\ \"https://bao.ad.ddupan.top:8200\"\n}\nauto_auth {\n method \"kubernetes\"\
|
||||
\ {\n mount_path = \"auth/kubernetes\"\n config = {\n role = \"metrics\"\
|
||||
\n token_path = \"/var/run/bao-jwt/token\"\n }\n }\n sink \"file\" {\n\
|
||||
\ config = {\n path = \"/var/run/bao-metrics/token\"\n mode = 0644\n\
|
||||
\ }\n }\n}\n"
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: openbao-metrics
|
||||
namespace: monitoring
|
||||
labels:
|
||||
app.kubernetes.io/name: openbao-metrics
|
||||
spec:
|
||||
ports:
|
||||
- name: https
|
||||
port: 8200
|
||||
targetPort: 8200
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: Endpoints
|
||||
metadata:
|
||||
name: openbao-metrics
|
||||
namespace: monitoring
|
||||
subsets:
|
||||
- addresses:
|
||||
- ip: 192.168.10.8
|
||||
ports:
|
||||
- name: https
|
||||
port: 8200
|
||||
protocol: TCP
|
||||
---
|
||||
apiVersion: monitoring.coreos.com/v1
|
||||
kind: ServiceMonitor
|
||||
metadata:
|
||||
name: openbao
|
||||
namespace: monitoring
|
||||
spec:
|
||||
selector:
|
||||
matchLabels:
|
||||
app.kubernetes.io/name: openbao-metrics
|
||||
endpoints:
|
||||
- port: https
|
||||
scheme: https
|
||||
path: /v1/sys/metrics
|
||||
params:
|
||||
format:
|
||||
- prometheus
|
||||
interval: 30s
|
||||
scrapeTimeout: 10s
|
||||
bearerTokenFile: /var/run/bao-metrics/token
|
||||
tlsConfig:
|
||||
serverName: bao.ad.ddupan.top
|
||||
relabelings:
|
||||
- targetLabel: job
|
||||
replacement: openbao
|
||||
@@ -1,6 +1,3 @@
|
||||
# Scrape agent. selectAllByDefault picks up every VMServiceScrape / VMPodScrape /
|
||||
# VMNodeScrape / VMStaticScrape in ALL namespaces, so app teams add their own
|
||||
# scrape CRs and vmagent discovers them automatically. Writes to VMSingle.
|
||||
apiVersion: operator.victoriametrics.com/v1beta1
|
||||
kind: VMAgent
|
||||
metadata:
|
||||
@@ -17,5 +14,79 @@ spec:
|
||||
cpu: 100m
|
||||
memory: 256Mi
|
||||
limits:
|
||||
cpu: "1"
|
||||
cpu: '1'
|
||||
memory: 1Gi
|
||||
volumes:
|
||||
- name: bao-agent-config
|
||||
configMap:
|
||||
name: bao-metrics-agent
|
||||
- name: bao-metrics-token
|
||||
emptyDir:
|
||||
medium: Memory
|
||||
- name: bao-metrics-jwt
|
||||
projected:
|
||||
sources:
|
||||
- serviceAccountToken:
|
||||
path: token
|
||||
expirationSeconds: 3600
|
||||
volumeMounts:
|
||||
- name: bao-metrics-token
|
||||
mountPath: /var/run/bao-metrics
|
||||
readOnly: true
|
||||
containers:
|
||||
- name: bao-metrics-agent
|
||||
image: quay.io/openbao/openbao:2.6.1
|
||||
command:
|
||||
- bao
|
||||
args:
|
||||
- agent
|
||||
- -config=/etc/bao-agent/config.hcl
|
||||
env:
|
||||
- name: HOME
|
||||
value: /tmp
|
||||
securityContext:
|
||||
runAsNonRoot: true
|
||||
runAsUser: 65534
|
||||
runAsGroup: 65534
|
||||
allowPrivilegeEscalation: false
|
||||
readOnlyRootFilesystem: true
|
||||
capabilities:
|
||||
drop:
|
||||
- ALL
|
||||
resources:
|
||||
requests:
|
||||
cpu: 10m
|
||||
memory: 32Mi
|
||||
limits:
|
||||
cpu: 100m
|
||||
memory: 128Mi
|
||||
volumeMounts:
|
||||
- name: bao-agent-config
|
||||
mountPath: /etc/bao-agent
|
||||
readOnly: true
|
||||
- name: bao-metrics-token
|
||||
mountPath: /var/run/bao-metrics
|
||||
- name: bao-metrics-jwt
|
||||
mountPath: /var/run/bao-jwt
|
||||
readOnly: true
|
||||
securityContext:
|
||||
fsGroup: 65534
|
||||
initContainers:
|
||||
- name: prepare-bao-token-file
|
||||
image: quay.io/openbao/openbao:2.6.1
|
||||
command:
|
||||
- sh
|
||||
- -ec
|
||||
- 'umask 0022; : > /var/run/bao-metrics/token'
|
||||
securityContext:
|
||||
runAsNonRoot: true
|
||||
runAsUser: 65534
|
||||
runAsGroup: 65534
|
||||
allowPrivilegeEscalation: false
|
||||
readOnlyRootFilesystem: true
|
||||
capabilities:
|
||||
drop:
|
||||
- ALL
|
||||
volumeMounts:
|
||||
- name: bao-metrics-token
|
||||
mountPath: /var/run/bao-metrics
|
||||
|
||||
Reference in New Issue
Block a user