From 0309d034287c05fb9515b18373e03fd96fab6376 Mon Sep 17 00:00:00 2001 From: panxiao81 Date: Fri, 25 Sep 2026 20:56:09 +0000 Subject: [PATCH] =?UTF-8?q?=E8=A1=A5=E9=BD=90=20OpenBao=20=E5=86=85?= =?UTF-8?q?=E9=83=A8=E5=81=A5=E5=BA=B7=E4=B8=8E=E6=9C=AC=E5=9C=B0=E5=BF=AB?= =?UTF-8?q?=E7=85=A7=E7=9B=91=E6=8E=A7?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- infrastructure/openbao/MONITORING.md | 57 +++++++ .../openbao/ansible/monitor-openbao.yml | 13 ++ .../roles/openbao_bootstrap/defaults/main.yml | 2 + .../tasks/snapshot-runtime.yml | 34 ++++ .../openbao_bootstrap/tasks/snapshots.yml | 37 +---- .../templates/bao-snapshot.sh.j2 | 53 ++++-- .../openbao_monitoring/defaults/main.yml | 3 + .../openbao_monitoring/handlers/main.yml | 6 + .../roles/openbao_monitoring/tasks/main.yml | 35 ++++ .../openbao/ansible/tests/test_snapshot.py | 81 ++++++++++ .../observability/metrics/rules/openbao.yaml | 50 ++++++ .../metrics/scrapes/openbao.yaml | 45 ++++++ .../metrics/tests/check-rules.sh | 2 +- .../metrics/tests/openbao.test.yaml | 151 ++++++++++++++++++ 14 files changed, 525 insertions(+), 44 deletions(-) create mode 100644 infrastructure/openbao/MONITORING.md create mode 100644 infrastructure/openbao/ansible/monitor-openbao.yml create mode 100644 infrastructure/openbao/ansible/roles/openbao_bootstrap/tasks/snapshot-runtime.yml create mode 100644 infrastructure/openbao/ansible/roles/openbao_monitoring/defaults/main.yml create mode 100644 infrastructure/openbao/ansible/roles/openbao_monitoring/handlers/main.yml create mode 100644 infrastructure/openbao/ansible/roles/openbao_monitoring/tasks/main.yml create mode 100644 infrastructure/openbao/ansible/tests/test_snapshot.py create mode 100644 platform/observability/metrics/tests/openbao.test.yaml diff --git a/infrastructure/openbao/MONITORING.md b/infrastructure/openbao/MONITORING.md new file mode 100644 index 0000000..860ea55 --- /dev/null +++ b/infrastructure/openbao/MONITORING.md @@ -0,0 +1,57 @@ +# OpenBao 监控运维 + +受鉴权 `/v1/sys/metrics` 通过 vmagent 内的 Bao Agent 使用 Kubernetes 身份采集。 +`metrics` policy 仅允许读取该路径；policy 和 role 由 `terraform/monitoring.tf` 管理。 + +Bao VM 的 Ubuntu `prometheus-node-exporter` 软件包监听 `192.168.10.8:9100`， +由 `ansible/roles/openbao_monitoring` 管理。仅供内网监控，不配置公共入口； +现场 UFW 未启用，故当前边界是 LAN 地址绑定，并非逐来源访问控制。 +标准主机指标复用 `job=node-exporter` 的主机告警，额外标记 `node=bao1`。 + +首次部署前需要 bootstrap 已创建 `/etc/openbao/snapshot.token` 和快照目录。 +只更新监控与快照运行文件（不会重启 Bao 或轮换 token）： + +```bash +cd ansible +ansible-playbook monitor-openbao.yml --check +ansible-playbook monitor-openbao.yml +# 在 bao1 上执行一次真实快照，建立成功基线： +sudo systemctl start openbao-snapshot.service +``` + +快照脚本在 `/var/lib/prometheus/node-exporter/` 原子更新两个 `.prom` 文件： + +- `openbao_snapshot_result.prom`：最近一次运行结果（0/1）及完成时间。 +- `openbao_snapshot_success.prom`：最后一次成功生成完整本地快照的时间。 + +失败保留上次成功时间，先完成快照再删除超出保留数量的文件。 +指标文件 root 写、exporter 只读；不包含 token、路径标签或快照内容。 +首次成功前成功指标缺失，不用既有文件 mtime 冒充已验证成功。 +Bao sealed 或 token 失效时，独立 exporter 仍能报告失败及超时。 + +`PrometheusRule/openbao` 包含：采集不可用（3 分钟）、内部健康异常（3 分钟）、 +健康 gauge 缺失（5 分钟）、快照失败（5 分钟）、快照超过 36 小时或无成功记录 +（持续 15 分钟）、快照监控缺失/损坏（5 分钟）。内部健康规则按当前**单节点**设计， +以后增加 standby 必须先调整 active 判定。快照指标仅证明本地任务成功，不能证明异地备份或可恢复性。 + +检查与恢复入口： + +```bash +sudo systemctl status openbao-snapshot.timer openbao-snapshot.service prometheus-node-exporter +sudo journalctl -u openbao-snapshot.service -n 50 --no-pager +curl -fsS http://192.168.10.8:9100/metrics | grep '^openbao_snapshot_' +``` + +token 续期失败按[维护 runbook](https://git.ddupan.top/panxiao81/homelab-wiki/src/branch/main/guides/openbao-monitoring-maintenance.md) +受控轮换；不要通过反复重启 Bao 排障。回滚 exporter 或快照脚本无需重启 Bao。 +回滚时同步撤回相应 ServiceMonitor/规则，否则缺指标告警会继续触发。 + +本地验证： + +```bash +python3 -m unittest discover -s infrastructure/openbao/ansible/tests -v +# 需 PyYAML、Jinja2 和 promtool；已用 promtool 3.5.0 验证。 +PATH=/path/to/promtool:$PATH bash platform/observability/metrics/tests/check-rules.sh +``` + +textfile 的原子发布方式遵循 [node_exporter 文档](https://github.com/prometheus/node_exporter#textfile-collector)。 diff --git a/infrastructure/openbao/ansible/monitor-openbao.yml b/infrastructure/openbao/ansible/monitor-openbao.yml new file mode 100644 index 0000000..044ac85 --- /dev/null +++ b/infrastructure/openbao/ansible/monitor-openbao.yml @@ -0,0 +1,13 @@ +--- +# 只部署 exporter、快照脚本和 timer，不改 Bao 配置、不重启 Bao、不创建 token。 +# 前提：bootstrap 已安装有效的 /etc/openbao/snapshot.token。 +- name: 补齐 OpenBao 主机和快照监控 + hosts: openbao + become: true + roles: + - openbao_monitoring + tasks: + - name: 更新快照运行文件 + ansible.builtin.include_role: + name: openbao_bootstrap + tasks_from: snapshot-runtime diff --git a/infrastructure/openbao/ansible/roles/openbao_bootstrap/defaults/main.yml b/infrastructure/openbao/ansible/roles/openbao_bootstrap/defaults/main.yml index ea91f50..f92248f 100644 --- a/infrastructure/openbao/ansible/roles/openbao_bootstrap/defaults/main.yml +++ b/infrastructure/openbao/ansible/roles/openbao_bootstrap/defaults/main.yml @@ -99,3 +99,5 @@ openbao_config_managed_by_terraform: true # 仅在维护窗口显式启用，用于替换已失效的快照 token；不读取/打印 token。 openbao_snapshot_rotate_token: false + +openbao_snapshot_metrics_dir: /var/lib/prometheus/node-exporter diff --git a/infrastructure/openbao/ansible/roles/openbao_bootstrap/tasks/snapshot-runtime.yml b/infrastructure/openbao/ansible/roles/openbao_bootstrap/tasks/snapshot-runtime.yml new file mode 100644 index 0000000..ebcf153 --- /dev/null +++ b/infrastructure/openbao/ansible/roles/openbao_bootstrap/tasks/snapshot-runtime.yml @@ -0,0 +1,34 @@ +--- +- name: Ensure the snapshot output directory exists + ansible.builtin.file: + path: "{{ openbao_snapshot_dir }}" + state: directory + owner: root + group: root + mode: "0700" + +- name: Install the snapshot script + ansible.builtin.template: + src: bao-snapshot.sh.j2 + dest: /usr/local/bin/bao-snapshot.sh + owner: root + group: root + mode: "0755" + +- name: Install the snapshot systemd service + timer + ansible.builtin.template: + src: "{{ item }}.j2" + dest: "/etc/systemd/system/{{ item }}" + owner: root + group: root + mode: "0644" + loop: + - openbao-snapshot.service + - openbao-snapshot.timer + +- name: Enable and start the snapshot timer + ansible.builtin.systemd: + name: openbao-snapshot.timer + state: started + enabled: true + daemon_reload: true diff --git a/infrastructure/openbao/ansible/roles/openbao_bootstrap/tasks/snapshots.yml b/infrastructure/openbao/ansible/roles/openbao_bootstrap/tasks/snapshots.yml index a6cf034..03547f3 100644 --- a/infrastructure/openbao/ansible/roles/openbao_bootstrap/tasks/snapshots.yml +++ b/infrastructure/openbao/ansible/roles/openbao_bootstrap/tasks/snapshots.yml @@ -35,36 +35,9 @@ when: (not snap_tok_stat.stat.exists) or (openbao_snapshot_rotate_token | bool) no_log: true -- name: Ensure the snapshot output directory exists - ansible.builtin.file: - path: "{{ openbao_snapshot_dir }}" - state: directory - owner: root - group: root - mode: "0700" +- name: 安装独立的主机与快照指标 exporter + ansible.builtin.include_role: + name: openbao_monitoring -- name: Install the snapshot script - ansible.builtin.template: - src: bao-snapshot.sh.j2 - dest: /usr/local/bin/bao-snapshot.sh - owner: root - group: root - mode: "0755" - -- name: Install the snapshot systemd service + timer - ansible.builtin.template: - src: "{{ item }}.j2" - dest: "/etc/systemd/system/{{ item }}" - owner: root - group: root - mode: "0644" - loop: - - openbao-snapshot.service - - openbao-snapshot.timer - -- name: Enable and start the snapshot timer - ansible.builtin.systemd: - name: openbao-snapshot.timer - state: started - enabled: true - daemon_reload: true +- name: 部署快照脚本和定时器 + ansible.builtin.import_tasks: snapshot-runtime.yml diff --git a/infrastructure/openbao/ansible/roles/openbao_bootstrap/templates/bao-snapshot.sh.j2 b/infrastructure/openbao/ansible/roles/openbao_bootstrap/templates/bao-snapshot.sh.j2 index a8a48ff..05bd85a 100644 --- a/infrastructure/openbao/ansible/roles/openbao_bootstrap/templates/bao-snapshot.sh.j2 +++ b/infrastructure/openbao/ansible/roles/openbao_bootstrap/templates/bao-snapshot.sh.j2 @@ -1,27 +1,58 @@ #!/usr/bin/env bash # {{ ansible_managed }} -# Take a Raft snapshot and prune old ones. Ship {{ openbao_snapshot_dir }} off-box -# separately (rsync/restic/scp) — a snapshot on the same host is not a backup. +# 本机快照不等于异地备份。结果通过独立 node_exporter 上报。 set -euo pipefail umask 077 +dir="{{ openbao_snapshot_dir }}" +metrics_dir="{{ openbao_snapshot_metrics_dir }}" +# systemd oneshot 不会并发；flock 同时阻止手动执行与 timer 竞争。 +exec 9>"${dir}/.snapshot.lock" +flock -n 9 || exit 0 +partial="" +metrics_tmp="" +finish() { + rc=$? + trap - EXIT + [ -z "$partial" ] || rm -f -- "$partial" + # success 文件只在实际快照完成后更新；失败不能抹掉上次成功时间。 + metrics_tmp="$(mktemp "${metrics_dir}/.openbao-result.XXXXXX")" || exit 1 + { + echo '# HELP openbao_snapshot_last_run_success Whether the last snapshot run succeeded.' + echo '# TYPE openbao_snapshot_last_run_success gauge' + if [ "$rc" -eq 0 ]; then echo 'openbao_snapshot_last_run_success 1'; else echo 'openbao_snapshot_last_run_success 0'; fi + echo '# HELP openbao_snapshot_last_run_timestamp_seconds Completion time of the last snapshot attempt.' + echo '# TYPE openbao_snapshot_last_run_timestamp_seconds gauge' + echo "openbao_snapshot_last_run_timestamp_seconds $(date +%s)" + } > "$metrics_tmp" + chmod 644 "$metrics_tmp" + mv -f -- "$metrics_tmp" "${metrics_dir}/openbao_snapshot_result.prom" + exit "$rc" +} +trap finish EXIT + export BAO_ADDR="{{ openbao_addr }}" export BAO_CACERT="{{ openbao_tls_dir }}/cert.pem" BAO_TOKEN="$(cat /etc/openbao/snapshot.token)" export BAO_TOKEN - -# periodic token 不会自动续期；在每日快照前续期，避免 768h 后永久失败。 bao token renew >/dev/null -dir="{{ openbao_snapshot_dir }}" stamp="$(date +%Y%m%d-%H%M%S)" out="${dir}/openbao-${stamp}.snap" - partial="${out}.partial" -trap 'rm -f -- "$partial"' EXIT -bao operator raft snapshot save "${partial}" -chmod 600 "${partial}" -mv -- "${partial}" "${out}" +bao operator raft snapshot save "$partial" +chmod 600 "$partial" +mv -- "$partial" "$out" +partial="" -# Retention: keep the newest {{ openbao_snapshot_keep }}. +metrics_tmp="$(mktemp "${metrics_dir}/.openbao-success.XXXXXX")" +{ + echo '# HELP openbao_snapshot_last_success_timestamp_seconds Completion time of the last successful local Raft snapshot.' + echo '# TYPE openbao_snapshot_last_success_timestamp_seconds gauge' + echo "openbao_snapshot_last_success_timestamp_seconds $(date +%s)" +} > "$metrics_tmp" +chmod 644 "$metrics_tmp" +mv -f -- "$metrics_tmp" "${metrics_dir}/openbao_snapshot_success.prom" + +# 只有产生完整快照后才做保留清理。 ls -1t "${dir}"/openbao-*.snap 2>/dev/null | tail -n +{{ openbao_snapshot_keep + 1 }} | xargs -r rm -f diff --git a/infrastructure/openbao/ansible/roles/openbao_monitoring/defaults/main.yml b/infrastructure/openbao/ansible/roles/openbao_monitoring/defaults/main.yml new file mode 100644 index 0000000..1fb6afb --- /dev/null +++ b/infrastructure/openbao/ansible/roles/openbao_monitoring/defaults/main.yml @@ -0,0 +1,3 @@ +--- +openbao_metrics_listen_address: "{{ ansible_host }}:9100" +openbao_snapshot_metrics_dir: /var/lib/prometheus/node-exporter diff --git a/infrastructure/openbao/ansible/roles/openbao_monitoring/handlers/main.yml b/infrastructure/openbao/ansible/roles/openbao_monitoring/handlers/main.yml new file mode 100644 index 0000000..9634310 --- /dev/null +++ b/infrastructure/openbao/ansible/roles/openbao_monitoring/handlers/main.yml @@ -0,0 +1,6 @@ +--- +- name: restart openbao node exporter + ansible.builtin.systemd: + name: prometheus-node-exporter + state: restarted + when: not ansible_check_mode diff --git a/infrastructure/openbao/ansible/roles/openbao_monitoring/tasks/main.yml b/infrastructure/openbao/ansible/roles/openbao_monitoring/tasks/main.yml new file mode 100644 index 0000000..9c337c0 --- /dev/null +++ b/infrastructure/openbao/ansible/roles/openbao_monitoring/tasks/main.yml @@ -0,0 +1,35 @@ +--- +- name: 安装 Ubuntu node_exporter 软件包 + ansible.builtin.apt: + name: prometheus-node-exporter + state: present + update_cache: true + cache_valid_time: 3600 + install_recommends: false + policy_rc_d: 101 + +- name: 创建只允许 root 写入的 textfile 目录 + ansible.builtin.file: + path: "{{ openbao_snapshot_metrics_dir }}" + state: directory + owner: root + group: root + mode: "0755" + +- name: 将 exporter 绑定到内网地址并启用 textfile + ansible.builtin.copy: + dest: /etc/default/prometheus-node-exporter + owner: root + group: root + mode: "0644" + content: | + # Ansible managed; no Bao credentials needed. + ARGS="--web.listen-address={{ openbao_metrics_listen_address }} --collector.textfile.directory={{ openbao_snapshot_metrics_dir }}" + notify: restart openbao node exporter + +- name: 启用主机指标服务 + ansible.builtin.systemd: + name: prometheus-node-exporter + enabled: true + state: started + when: not ansible_check_mode diff --git a/infrastructure/openbao/ansible/tests/test_snapshot.py b/infrastructure/openbao/ansible/tests/test_snapshot.py new file mode 100644 index 0000000..a579833 --- /dev/null +++ b/infrastructure/openbao/ansible/tests/test_snapshot.py @@ -0,0 +1,81 @@ +"""用假 bao 验证失败不会推进成功时间或删除已有快照。""" +import os +from pathlib import Path +import subprocess +import tempfile +import unittest +from jinja2 import Template + +TEMPLATE = Path(__file__).resolve().parents[1] / 'roles/openbao_bootstrap/templates/bao-snapshot.sh.j2' + + +class SnapshotTest(unittest.TestCase): + def setUp(self): + self.tmp = tempfile.TemporaryDirectory() + self.addCleanup(self.tmp.cleanup) + self.root = Path(self.tmp.name) + self.snap = self.root / 'snapshots' + self.metrics = self.root / 'metrics' + self.snap.mkdir() + self.metrics.mkdir() + self.token = self.root / 'token' + self.token.write_text('test-only') + self.script = self.root / 'snapshot.sh' + self.script.write_text(Template(TEMPLATE.read_text()).render( + ansible_managed='test', openbao_snapshot_dir=str(self.snap), + openbao_snapshot_metrics_dir=str(self.metrics), openbao_addr='https://invalid', + openbao_tls_dir='/unused', openbao_snapshot_keep=2, + ).replace('/etc/openbao/snapshot.token', str(self.token))) + bao = self.root / 'bao' + bao.write_text('''#!/usr/bin/env bash +if [ "$1" = token ]; then + [ "${FAIL_AT:-}" != renew ]; exit $? +fi +printf snapshot > "$5" +[ "${FAIL_AT:-}" != save ] +''') + bao.chmod(0o755) + for n in range(3): + p = self.snap / f'openbao-old{n}.snap' + p.write_text('old snapshot') + os.utime(p, (100+n, 100+n)) + self.success = self.metrics / 'openbao_snapshot_success.prom' + self.success.write_text('openbao_snapshot_last_success_timestamp_seconds 123\n') + + def run_snapshot(self, failure=''): + return subprocess.run(['bash', str(self.script)], env=dict(os.environ, + PATH=str(self.root)+':'+os.environ['PATH'], FAIL_AT=failure), capture_output=True) + + def test_renew_failure_preserves_success_and_snapshots(self): + self.assertNotEqual(self.run_snapshot('renew').returncode, 0) + self.check_failure() + + def test_partial_snapshot_is_removed(self): + self.assertNotEqual(self.run_snapshot('save').returncode, 0) + self.check_failure() + self.assertEqual(list(self.snap.glob('*.partial')), []) + + def test_missing_token_is_reported(self): + self.token.unlink() + self.assertNotEqual(self.run_snapshot().returncode, 0) + self.check_failure() + + def check_failure(self): + self.assertEqual(self.success.read_text(), 'openbao_snapshot_last_success_timestamp_seconds 123\n') + self.assertIn('openbao_snapshot_last_run_success 0', (self.metrics/'openbao_snapshot_result.prom').read_text()) + self.assertEqual(len(list(self.snap.glob('*.snap'))), 3) + + def test_success_retention_and_permissions(self): + p = self.run_snapshot() + self.assertEqual(p.returncode, 0, p.stderr) + self.assertNotIn('seconds 123\n', self.success.read_text()) + self.assertIn('openbao_snapshot_last_run_success 1', (self.metrics/'openbao_snapshot_result.prom').read_text()) + snapshots = list(self.snap.glob('*.snap')) + self.assertEqual(len(snapshots), 2) + new = next(p for p in snapshots if 'old' not in p.name) + self.assertEqual(new.stat().st_mode & 0o777, 0o600) + self.assertEqual(self.success.stat().st_mode & 0o777, 0o644) + + +if __name__ == '__main__': + unittest.main() diff --git a/platform/observability/metrics/rules/openbao.yaml b/platform/observability/metrics/rules/openbao.yaml index c1b0d29..e0b6d64 100644 --- a/platform/observability/metrics/rules/openbao.yaml +++ b/platform/observability/metrics/rules/openbao.yaml @@ -16,3 +16,53 @@ spec: summary: OpenBao 受鉴权指标采集不可用 description: 检查 Bao 健康与 sealed 状态、采集 Agent 登录/续期、TLS 和目标发现；此告警不区分停机与鉴权失败。 runbook_url: https://git.ddupan.top/panxiao81/homelab-wiki/src/branch/main/guides/openbao-monitoring-maintenance.md + - alert: OpenBaoInternalUnhealthy + expr: (min by(job) (vault_core_active{job="openbao"}) < 1 or min by(job) (vault_core_unsealed{job="openbao"}) + < 1 or min by(job) (vault_autopilot_node_healthy{job="openbao"}) < 1) and on(job) (up{job="openbao"} == + 1) + for: 3m + labels: + severity: critical + annotations: + summary: OpenBao 内部健康异常 + description: 单节点部署应保持 active、unsealed 和 autopilot node healthy；检查 Raft 状态和服务日志。 + runbook_url: https://git.ddupan.top/panxiao81/homelab-wiki/src/branch/main/guides/openbao-monitoring-maintenance.md + - alert: OpenBaoHealthMetricsMissing + expr: (absent(vault_core_active{job="openbao"}) or absent(vault_core_unsealed{job="openbao"}) or absent(vault_autopilot_node_healthy{job="openbao"})) + and on(job) (up{job="openbao"} == 1) + for: 5m + labels: + severity: warning + annotations: + summary: OpenBao 内部健康指标缺失 + description: 接口可采集但核心 gauge 缺失；检查 telemetry、版本和指标过滤，不将缺数据视为健康。 + runbook_url: https://git.ddupan.top/panxiao81/homelab-wiki/src/branch/main/guides/openbao-monitoring-maintenance.md + - alert: OpenBaoSnapshotFailed + expr: openbao_snapshot_last_run_success{job="node-exporter",node="bao1"} == 0 + for: 5m + labels: + severity: warning + annotations: + summary: OpenBao 最近一次本地快照失败 + description: 检查 openbao-snapshot.service 日志、sealed 状态、快照 token 续期与磁盘；修复后重跑任务。 + runbook_url: https://git.ddupan.top/panxiao81/homelab-wiki/src/branch/main/guides/openbao-monitoring-maintenance.md + - alert: OpenBaoSnapshotStale + expr: time() - openbao_snapshot_last_success_timestamp_seconds{job="node-exporter",node="bao1"} > 36 * 3600 + or absent(openbao_snapshot_last_success_timestamp_seconds{job="node-exporter",node="bao1"}) + for: 15m + labels: + severity: critical + annotations: + summary: OpenBao 本地快照已超时或没有成功记录 + description: 每日快照超过 36 小时未成功，或成功指标缺失；检查定时器与快照任务。此指标不证明异地备份或恢复可用。 + runbook_url: https://git.ddupan.top/panxiao81/homelab-wiki/src/branch/main/guides/openbao-monitoring-maintenance.md + - alert: OpenBaoSnapshotMetricsUnavailable + expr: up{job="node-exporter",node="bao1"} == 0 or absent(up{job="node-exporter",node="bao1"}) or node_textfile_scrape_error{job="node-exporter",node="bao1"} + == 1 or absent(openbao_snapshot_last_run_success{job="node-exporter",node="bao1"}) or absent(openbao_snapshot_last_run_timestamp_seconds{job="node-exporter",node="bao1"}) + for: 5m + labels: + severity: warning + annotations: + summary: OpenBao 快照监控不可用 + description: 检查 bao1:9100 的 node_exporter、textfile 目录及快照脚本；采集独立于 Bao 的鉴权接口。 + runbook_url: https://git.ddupan.top/panxiao81/homelab-wiki/src/branch/main/guides/openbao-monitoring-maintenance.md diff --git a/platform/observability/metrics/scrapes/openbao.yaml b/platform/observability/metrics/scrapes/openbao.yaml index 3b6e5ba..94d8957 100644 --- a/platform/observability/metrics/scrapes/openbao.yaml +++ b/platform/observability/metrics/scrapes/openbao.yaml @@ -61,3 +61,48 @@ spec: relabelings: - targetLabel: job replacement: openbao +--- +apiVersion: v1 +kind: Service +metadata: + name: openbao-node-metrics + namespace: monitoring + labels: + app.kubernetes.io/name: openbao-node-metrics +spec: + ports: + - name: http + port: 9100 + targetPort: 9100 +--- +apiVersion: v1 +kind: Endpoints +metadata: + name: openbao-node-metrics + namespace: monitoring +subsets: +- addresses: + - ip: 192.168.10.8 + ports: + - name: http + port: 9100 + protocol: TCP +--- +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: openbao-node + namespace: monitoring +spec: + selector: + matchLabels: + app.kubernetes.io/name: openbao-node-metrics + endpoints: + - port: http + interval: 30s + scrapeTimeout: 10s + relabelings: + - targetLabel: job + replacement: node-exporter + - targetLabel: node + replacement: bao1 diff --git a/platform/observability/metrics/tests/check-rules.sh b/platform/observability/metrics/tests/check-rules.sh index 8bac009..a093dce 100644 --- a/platform/observability/metrics/tests/check-rules.sh +++ b/platform/observability/metrics/tests/check-rules.sh @@ -10,7 +10,7 @@ import sys import yaml groups = [] -for name in ("kubernetes-health", "host-health", "monitoring-delivery", "platform-controllers", "data-services"): +for name in ("kubernetes-health", "host-health", "monitoring-delivery", "platform-controllers", "data-services", "openbao"): path = pathlib.Path(sys.argv[1]) / f"{name}.yaml" groups.extend(yaml.safe_load(path.read_text())["spec"]["groups"]) pathlib.Path(sys.argv[2]).write_text(yaml.safe_dump({"groups": groups}, allow_unicode=True)) diff --git a/platform/observability/metrics/tests/openbao.test.yaml b/platform/observability/metrics/tests/openbao.test.yaml new file mode 100644 index 0000000..d8d5ce6 --- /dev/null +++ b/platform/observability/metrics/tests/openbao.test.yaml @@ -0,0 +1,151 @@ +rule_files: +- rules.yaml +evaluation_interval: 1m +tests: +- name: 健康节点不触发内部告警 + interval: 1m + input_series: + - series: up{job="openbao"} + values: 1x30 + - series: vault_core_active{job="openbao"} + values: 1x30 + - series: vault_core_unsealed{job="openbao"} + values: 1x30 + - series: vault_autopilot_node_healthy{job="openbao",node_id="bao1"} + values: 1x30 + alert_rule_test: + - eval_time: 10m + alertname: OpenBaoInternalUnhealthy + exp_alerts: [] +- name: autopilot 不健康触发内部告警 + interval: 1m + input_series: + - series: up{job="openbao"} + values: 1x30 + - series: vault_core_active{job="openbao"} + values: 1x30 + - series: vault_core_unsealed{job="openbao"} + values: 1x30 + - series: vault_autopilot_node_healthy{job="openbao",node_id="bao1"} + values: '0x30' + alert_rule_test: + - eval_time: 4m + alertname: OpenBaoInternalUnhealthy + exp_alerts: + - exp_labels: + job: openbao + severity: critical + exp_annotations: + summary: OpenBao 内部健康异常 + description: 单节点部署应保持 active、unsealed 和 autopilot node healthy；检查 Raft 状态和服务日志。 + runbook_url: https://git.ddupan.top/panxiao81/homelab-wiki/src/branch/main/guides/openbao-monitoring-maintenance.md +- name: 采集失败由不可用告警覆盖 + interval: 1m + input_series: + - series: up{job="openbao"} + values: '0x30' + - series: vault_core_active{job="openbao"} + values: '0x30' + - series: vault_core_unsealed{job="openbao"} + values: 1x30 + - series: vault_autopilot_node_healthy{job="openbao",node_id="bao1"} + values: 1x30 + alert_rule_test: + - eval_time: 10m + alertname: OpenBaoInternalUnhealthy + exp_alerts: [] +- name: 缺少健康指标不能静默 + interval: 1m + input_series: + - series: up{job="openbao"} + values: 1x30 + alert_rule_test: + - eval_time: 6m + alertname: OpenBaoHealthMetricsMissing + exp_alerts: + - exp_labels: + job: openbao + severity: warning + exp_annotations: + summary: OpenBao 内部健康指标缺失 + description: 接口可采集但核心 gauge 缺失；检查 telemetry、版本和指标过滤，不将缺数据视为健康。 + runbook_url: https://git.ddupan.top/panxiao81/homelab-wiki/src/branch/main/guides/openbao-monitoring-maintenance.md +- name: 没有成功快照记录 + interval: 1m + input_series: [] + alert_rule_test: + - eval_time: 16m + alertname: OpenBaoSnapshotStale + exp_alerts: + - exp_labels: + job: node-exporter + node: bao1 + severity: critical + exp_annotations: &id001 + summary: OpenBao 本地快照已超时或没有成功记录 + description: 每日快照超过 36 小时未成功，或成功指标缺失；检查定时器与快照任务。此指标不证明异地备份或恢复可用。 + runbook_url: https://git.ddupan.top/panxiao81/homelab-wiki/src/branch/main/guides/openbao-monitoring-maintenance.md +- name: 快照失败不需要 Bao 可用 + interval: 1m + input_series: + - series: openbao_snapshot_last_run_success{job="node-exporter",node="bao1"} + values: '0x30' + alert_rule_test: + - eval_time: 6m + alertname: OpenBaoSnapshotFailed + exp_alerts: + - exp_labels: + job: node-exporter + node: bao1 + severity: warning + exp_annotations: + summary: OpenBao 最近一次本地快照失败 + description: 检查 openbao-snapshot.service 日志、sealed 状态、快照 token 续期与磁盘；修复后重跑任务。 + runbook_url: https://git.ddupan.top/panxiao81/homelab-wiki/src/branch/main/guides/openbao-monitoring-maintenance.md +- name: 刚完成的快照不超时 + interval: 1m + input_series: + - series: openbao_snapshot_last_success_timestamp_seconds{job="node-exporter",node="bao1"} + values: '0x30' + alert_rule_test: + - eval_time: 20m + alertname: OpenBaoSnapshotStale + exp_alerts: [] +- name: exporter 失联 + interval: 1m + input_series: + - series: up{job="node-exporter",node="bao1"} + values: '0x30' + - series: openbao_snapshot_last_run_success{job="node-exporter",node="bao1"} + values: 1x30 + - series: openbao_snapshot_last_run_timestamp_seconds{job="node-exporter",node="bao1"} + values: '0x30' + alert_rule_test: + - eval_time: 6m + alertname: OpenBaoSnapshotMetricsUnavailable + exp_alerts: + - exp_labels: + job: node-exporter + node: bao1 + severity: warning + exp_annotations: + summary: OpenBao 快照监控不可用 + description: 检查 bao1:9100 的 node_exporter、textfile 目录及快照脚本；采集独立于 Bao 的鉴权接口。 + runbook_url: https://git.ddupan.top/panxiao81/homelab-wiki/src/branch/main/guides/openbao-monitoring-maintenance.md +- name: 36 小时后且持续 15 分钟才报告超时 + interval: 1h + input_series: + - series: openbao_snapshot_last_success_timestamp_seconds{job="node-exporter",node="bao1"} + values: '0x40' + alert_rule_test: + - eval_time: 36h + alertname: OpenBaoSnapshotStale + exp_alerts: [] + - eval_time: 37h20m + alertname: OpenBaoSnapshotStale + exp_alerts: + - exp_labels: + job: node-exporter + node: bao1 + severity: critical + exp_annotations: *id001