补齐 OpenBao 内部健康与快照新鲜度告警

Co-authored-by: panxiao81 <[email protected]>
2026-09-25 20:59:36 +00:00
parent 834f654941
commit 7137426e8f
14 changed files with 525 additions and 44 deletions
@@ -0,0 +1,13 @@
+---
+# 只部署 exporter、快照脚本和 timer，不改 Bao 配置、不重启 Bao、不创建 token。
+# 前提：bootstrap 已安装有效的 /etc/openbao/snapshot.token。
+- name: 补齐 OpenBao 主机和快照监控
+  hosts: openbao
+  become: true
+  roles:
+    - openbao_monitoring
+  tasks:
+    - name: 更新快照运行文件
+      ansible.builtin.include_role:
+        name: openbao_bootstrap
+        tasks_from: snapshot-runtime
@@ -99,3 +99,5 @@ openbao_config_managed_by_terraform: true

 # 仅在维护窗口显式启用，用于替换已失效的快照 token；不读取/打印 token。
 openbao_snapshot_rotate_token: false
+
+openbao_snapshot_metrics_dir: /var/lib/prometheus/node-exporter
@@ -0,0 +1,34 @@
+---
+- name: Ensure the snapshot output directory exists
+  ansible.builtin.file:
+    path: "{{ openbao_snapshot_dir }}"
+    state: directory
+    owner: root
+    group: root
+    mode: "0700"
+
+- name: Install the snapshot script
+  ansible.builtin.template:
+    src: bao-snapshot.sh.j2
+    dest: /usr/local/bin/bao-snapshot.sh
+    owner: root
+    group: root
+    mode: "0755"
+
+- name: Install the snapshot systemd service + timer
+  ansible.builtin.template:
+    src: "{{ item }}.j2"
+    dest: "/etc/systemd/system/{{ item }}"
+    owner: root
+    group: root
+    mode: "0644"
+  loop:
+    - openbao-snapshot.service
+    - openbao-snapshot.timer
+
+- name: Enable and start the snapshot timer
+  ansible.builtin.systemd:
+    name: openbao-snapshot.timer
+    state: started
+    enabled: true
+    daemon_reload: true
@@ -35,36 +35,9 @@
  when: (not snap_tok_stat.stat.exists) or (openbao_snapshot_rotate_token | bool)
  no_log: true

- name: Ensure the snapshot output directory exists
-  ansible.builtin.file:
-    path: "{{ openbao_snapshot_dir }}"
-    state: directory
-    owner: root
-    group: root
-    mode: "0700"
+- name: 安装独立的主机与快照指标 exporter
+  ansible.builtin.include_role:
+    name: openbao_monitoring

- name: Install the snapshot script
-  ansible.builtin.template:
-    src: bao-snapshot.sh.j2
-    dest: /usr/local/bin/bao-snapshot.sh
-    owner: root
-    group: root
-    mode: "0755"
-
- name: Install the snapshot systemd service + timer
-  ansible.builtin.template:
-    src: "{{ item }}.j2"
-    dest: "/etc/systemd/system/{{ item }}"
-    owner: root
-    group: root
-    mode: "0644"
-  loop:
-    - openbao-snapshot.service
-    - openbao-snapshot.timer
-
- name: Enable and start the snapshot timer
-  ansible.builtin.systemd:
-    name: openbao-snapshot.timer
-    state: started
-    enabled: true
-    daemon_reload: true
+- name: 部署快照脚本和定时器
+  ansible.builtin.import_tasks: snapshot-runtime.yml
@@ -1,27 +1,58 @@
 #!/usr/bin/env bash
 # {{ ansible_managed }}
-# Take a Raft snapshot and prune old ones. Ship {{ openbao_snapshot_dir }} off-box
-# separately (rsync/restic/scp) — a snapshot on the same host is not a backup.
+# 本机快照不等于异地备份。结果通过独立 node_exporter 上报。
 set -euo pipefail
 umask 077

+dir="{{ openbao_snapshot_dir }}"
+metrics_dir="{{ openbao_snapshot_metrics_dir }}"
+# systemd oneshot 不会并发；flock 同时阻止手动执行与 timer 竞争。
+exec 9>"${dir}/.snapshot.lock"
+flock -n 9 || exit 0
+partial=""
+metrics_tmp=""
+finish() {
+  rc=$?
+  trap - EXIT
+  [ -z "$partial" ] || rm -f -- "$partial"
+  # success 文件只在实际快照完成后更新；失败不能抹掉上次成功时间。
+  metrics_tmp="$(mktemp "${metrics_dir}/.openbao-result.XXXXXX")" || exit 1
+  {
+    echo '# HELP openbao_snapshot_last_run_success Whether the last snapshot run succeeded.'
+    echo '# TYPE openbao_snapshot_last_run_success gauge'
+    if [ "$rc" -eq 0 ]; then echo 'openbao_snapshot_last_run_success 1'; else echo 'openbao_snapshot_last_run_success 0'; fi
+    echo '# HELP openbao_snapshot_last_run_timestamp_seconds Completion time of the last snapshot attempt.'
+    echo '# TYPE openbao_snapshot_last_run_timestamp_seconds gauge'
+    echo "openbao_snapshot_last_run_timestamp_seconds $(date +%s)"
+  } > "$metrics_tmp"
+  chmod 644 "$metrics_tmp"
+  mv -f -- "$metrics_tmp" "${metrics_dir}/openbao_snapshot_result.prom"
+  exit "$rc"
+}
+trap finish EXIT
+
 export BAO_ADDR="{{ openbao_addr }}"
 export BAO_CACERT="{{ openbao_tls_dir }}/cert.pem"
 BAO_TOKEN="$(cat /etc/openbao/snapshot.token)"
 export BAO_TOKEN
-
-# periodic token 不会自动续期；在每日快照前续期，避免 768h 后永久失败。
 bao token renew >/dev/null

-dir="{{ openbao_snapshot_dir }}"
 stamp="$(date +%Y%m%d-%H%M%S)"
 out="${dir}/openbao-${stamp}.snap"
-
 partial="${out}.partial"
-trap 'rm -f -- "$partial"' EXIT
-bao operator raft snapshot save "${partial}"
-chmod 600 "${partial}"
-mv -- "${partial}" "${out}"
+bao operator raft snapshot save "$partial"
+chmod 600 "$partial"
+mv -- "$partial" "$out"
+partial=""

-# Retention: keep the newest {{ openbao_snapshot_keep }}.
+metrics_tmp="$(mktemp "${metrics_dir}/.openbao-success.XXXXXX")"
+{
+  echo '# HELP openbao_snapshot_last_success_timestamp_seconds Completion time of the last successful local Raft snapshot.'
+  echo '# TYPE openbao_snapshot_last_success_timestamp_seconds gauge'
+  echo "openbao_snapshot_last_success_timestamp_seconds $(date +%s)"
+} > "$metrics_tmp"
+chmod 644 "$metrics_tmp"
+mv -f -- "$metrics_tmp" "${metrics_dir}/openbao_snapshot_success.prom"
+
+# 只有产生完整快照后才做保留清理。
 ls -1t "${dir}"/openbao-*.snap 2>/dev/null | tail -n +{{ openbao_snapshot_keep + 1 }} | xargs -r rm -f
@@ -0,0 +1,3 @@
+---
+openbao_metrics_listen_address: "{{ ansible_host }}:9100"
+openbao_snapshot_metrics_dir: /var/lib/prometheus/node-exporter
@@ -0,0 +1,6 @@
+---
+- name: restart openbao node exporter
+  ansible.builtin.systemd:
+    name: prometheus-node-exporter
+    state: restarted
+  when: not ansible_check_mode
@@ -0,0 +1,35 @@
+---
+- name: 安装 Ubuntu node_exporter 软件包
+  ansible.builtin.apt:
+    name: prometheus-node-exporter
+    state: present
+    update_cache: true
+    cache_valid_time: 3600
+    install_recommends: false
+    policy_rc_d: 101
+
+- name: 创建只允许 root 写入的 textfile 目录
+  ansible.builtin.file:
+    path: "{{ openbao_snapshot_metrics_dir }}"
+    state: directory
+    owner: root
+    group: root
+    mode: "0755"
+
+- name: 将 exporter 绑定到内网地址并启用 textfile
+  ansible.builtin.copy:
+    dest: /etc/default/prometheus-node-exporter
+    owner: root
+    group: root
+    mode: "0644"
+    content: |
+      # Ansible managed; no Bao credentials needed.
+      ARGS="--web.listen-address={{ openbao_metrics_listen_address }} --collector.textfile.directory={{ openbao_snapshot_metrics_dir }}"
+  notify: restart openbao node exporter
+
+- name: 启用主机指标服务
+  ansible.builtin.systemd:
+    name: prometheus-node-exporter
+    enabled: true
+    state: started
+  when: not ansible_check_mode
@@ -0,0 +1,81 @@
+"""用假 bao 验证失败不会推进成功时间或删除已有快照。"""
+import os
+from pathlib import Path
+import subprocess
+import tempfile
+import unittest
+from jinja2 import Template
+
+TEMPLATE = Path(__file__).resolve().parents[1] / 'roles/openbao_bootstrap/templates/bao-snapshot.sh.j2'
+
+
+class SnapshotTest(unittest.TestCase):
+    def setUp(self):
+        self.tmp = tempfile.TemporaryDirectory()
+        self.addCleanup(self.tmp.cleanup)
+        self.root = Path(self.tmp.name)
+        self.snap = self.root / 'snapshots'
+        self.metrics = self.root / 'metrics'
+        self.snap.mkdir()
+        self.metrics.mkdir()
+        self.token = self.root / 'token'
+        self.token.write_text('test-only')
+        self.script = self.root / 'snapshot.sh'
+        self.script.write_text(Template(TEMPLATE.read_text()).render(
+            ansible_managed='test', openbao_snapshot_dir=str(self.snap),
+            openbao_snapshot_metrics_dir=str(self.metrics), openbao_addr='https://invalid',
+            openbao_tls_dir='/unused', openbao_snapshot_keep=2,
+        ).replace('/etc/openbao/snapshot.token', str(self.token)))
+        bao = self.root / 'bao'
+        bao.write_text('''#!/usr/bin/env bash
+if [ "$1" = token ]; then
+  [ "${FAIL_AT:-}" != renew ]; exit $?
+fi
+printf snapshot > "$5"
+[ "${FAIL_AT:-}" != save ]
+''')
+        bao.chmod(0o755)
+        for n in range(3):
+            p = self.snap / f'openbao-old{n}.snap'
+            p.write_text('old snapshot')
+            os.utime(p, (100+n, 100+n))
+        self.success = self.metrics / 'openbao_snapshot_success.prom'
+        self.success.write_text('openbao_snapshot_last_success_timestamp_seconds 123\n')
+
+    def run_snapshot(self, failure=''):
+        return subprocess.run(['bash', str(self.script)], env=dict(os.environ,
+            PATH=str(self.root)+':'+os.environ['PATH'], FAIL_AT=failure), capture_output=True)
+
+    def test_renew_failure_preserves_success_and_snapshots(self):
+        self.assertNotEqual(self.run_snapshot('renew').returncode, 0)
+        self.check_failure()
+
+    def test_partial_snapshot_is_removed(self):
+        self.assertNotEqual(self.run_snapshot('save').returncode, 0)
+        self.check_failure()
+        self.assertEqual(list(self.snap.glob('*.partial')), [])
+
+    def test_missing_token_is_reported(self):
+        self.token.unlink()
+        self.assertNotEqual(self.run_snapshot().returncode, 0)
+        self.check_failure()
+
+    def check_failure(self):
+        self.assertEqual(self.success.read_text(), 'openbao_snapshot_last_success_timestamp_seconds 123\n')
+        self.assertIn('openbao_snapshot_last_run_success 0', (self.metrics/'openbao_snapshot_result.prom').read_text())
+        self.assertEqual(len(list(self.snap.glob('*.snap'))), 3)
+
+    def test_success_retention_and_permissions(self):
+        p = self.run_snapshot()
+        self.assertEqual(p.returncode, 0, p.stderr)
+        self.assertNotIn('seconds 123\n', self.success.read_text())
+        self.assertIn('openbao_snapshot_last_run_success 1', (self.metrics/'openbao_snapshot_result.prom').read_text())
+        snapshots = list(self.snap.glob('*.snap'))
+        self.assertEqual(len(snapshots), 2)
+        new = next(p for p in snapshots if 'old' not in p.name)
+        self.assertEqual(new.stat().st_mode & 0o777, 0o600)
+        self.assertEqual(self.success.stat().st_mode & 0o777, 0o644)
+
+
+if __name__ == '__main__':
+    unittest.main()