补齐 OpenBao 内部健康与快照新鲜度告警

Co-authored-by: panxiao81 <[email protected]>

infrastructure/openbao/ansible/roles/openbao_bootstrap/defaults/main.yml

@@ -99,3 +99,5 @@ openbao_config_managed_by_terraform: true
 
 # 仅在维护窗口显式启用，用于替换已失效的快照 token；不读取/打印 token。
 openbao_snapshot_rotate_token: false
+
+openbao_snapshot_metrics_dir: /var/lib/prometheus/node-exporter

infrastructure/openbao/ansible/roles/openbao_bootstrap/tasks/snapshot-runtime.yml

@@ -0,0 +1,34 @@
+---
+- name: Ensure the snapshot output directory exists
+  ansible.builtin.file:
+    path: "{{ openbao_snapshot_dir }}"
+    state: directory
+    owner: root
+    group: root
+    mode: "0700"
+
+- name: Install the snapshot script
+  ansible.builtin.template:
+    src: bao-snapshot.sh.j2
+    dest: /usr/local/bin/bao-snapshot.sh
+    owner: root
+    group: root
+    mode: "0755"
+
+- name: Install the snapshot systemd service + timer
+  ansible.builtin.template:
+    src: "{{ item }}.j2"
+    dest: "/etc/systemd/system/{{ item }}"
+    owner: root
+    group: root
+    mode: "0644"
+  loop:
+    - openbao-snapshot.service
+    - openbao-snapshot.timer
+
+- name: Enable and start the snapshot timer
+  ansible.builtin.systemd:
+    name: openbao-snapshot.timer
+    state: started
+    enabled: true
+    daemon_reload: true

infrastructure/openbao/ansible/roles/openbao_bootstrap/tasks/snapshots.yml

@@ -35,36 +35,9 @@
   when: (not snap_tok_stat.stat.exists) or (openbao_snapshot_rotate_token | bool)
   no_log: true
 
-- name: Ensure the snapshot output directory exists
-  ansible.builtin.file:
-    path: "{{ openbao_snapshot_dir }}"
-    state: directory
-    owner: root
-    group: root
-    mode: "0700"
+- name: 安装独立的主机与快照指标 exporter
+  ansible.builtin.include_role:
+    name: openbao_monitoring
 
-- name: Install the snapshot script
-  ansible.builtin.template:
-    src: bao-snapshot.sh.j2
-    dest: /usr/local/bin/bao-snapshot.sh
-    owner: root
-    group: root
-    mode: "0755"
-
-- name: Install the snapshot systemd service + timer
-  ansible.builtin.template:
-    src: "{{ item }}.j2"
-    dest: "/etc/systemd/system/{{ item }}"
-    owner: root
-    group: root
-    mode: "0644"
-  loop:
-    - openbao-snapshot.service
-    - openbao-snapshot.timer
-
-- name: Enable and start the snapshot timer
-  ansible.builtin.systemd:
-    name: openbao-snapshot.timer
-    state: started
-    enabled: true
-    daemon_reload: true
+- name: 部署快照脚本和定时器
+  ansible.builtin.import_tasks: snapshot-runtime.yml

infrastructure/openbao/ansible/roles/openbao_bootstrap/templates/bao-snapshot.sh.j2

@@ -1,27 +1,58 @@
 #!/usr/bin/env bash
 # {{ ansible_managed }}
-# Take a Raft snapshot and prune old ones. Ship {{ openbao_snapshot_dir }} off-box
-# separately (rsync/restic/scp) — a snapshot on the same host is not a backup.
+# 本机快照不等于异地备份。结果通过独立 node_exporter 上报。
 set -euo pipefail
 umask 077
 
+dir="{{ openbao_snapshot_dir }}"
+metrics_dir="{{ openbao_snapshot_metrics_dir }}"
+# systemd oneshot 不会并发；flock 同时阻止手动执行与 timer 竞争。
+exec 9>"${dir}/.snapshot.lock"
+flock -n 9 || exit 0
+partial=""
+metrics_tmp=""
+finish() {
+  rc=$?
+  trap - EXIT
+  [ -z "$partial" ] || rm -f -- "$partial"
+  # success 文件只在实际快照完成后更新；失败不能抹掉上次成功时间。
+  metrics_tmp="$(mktemp "${metrics_dir}/.openbao-result.XXXXXX")" || exit 1
+  {
+    echo '# HELP openbao_snapshot_last_run_success Whether the last snapshot run succeeded.'
+    echo '# TYPE openbao_snapshot_last_run_success gauge'
+    if [ "$rc" -eq 0 ]; then echo 'openbao_snapshot_last_run_success 1'; else echo 'openbao_snapshot_last_run_success 0'; fi
+    echo '# HELP openbao_snapshot_last_run_timestamp_seconds Completion time of the last snapshot attempt.'
+    echo '# TYPE openbao_snapshot_last_run_timestamp_seconds gauge'
+    echo "openbao_snapshot_last_run_timestamp_seconds $(date +%s)"
+  } > "$metrics_tmp"
+  chmod 644 "$metrics_tmp"
+  mv -f -- "$metrics_tmp" "${metrics_dir}/openbao_snapshot_result.prom"
+  exit "$rc"
+}
+trap finish EXIT
+
 export BAO_ADDR="{{ openbao_addr }}"
 export BAO_CACERT="{{ openbao_tls_dir }}/cert.pem"
 BAO_TOKEN="$(cat /etc/openbao/snapshot.token)"
 export BAO_TOKEN
-
-# periodic token 不会自动续期；在每日快照前续期，避免 768h 后永久失败。
 bao token renew >/dev/null
 
-dir="{{ openbao_snapshot_dir }}"
 stamp="$(date +%Y%m%d-%H%M%S)"
 out="${dir}/openbao-${stamp}.snap"
-
 partial="${out}.partial"
-trap 'rm -f -- "$partial"' EXIT
-bao operator raft snapshot save "${partial}"
-chmod 600 "${partial}"
-mv -- "${partial}" "${out}"
+bao operator raft snapshot save "$partial"
+chmod 600 "$partial"
+mv -- "$partial" "$out"
+partial=""
 
-# Retention: keep the newest {{ openbao_snapshot_keep }}.
+metrics_tmp="$(mktemp "${metrics_dir}/.openbao-success.XXXXXX")"
+{
+  echo '# HELP openbao_snapshot_last_success_timestamp_seconds Completion time of the last successful local Raft snapshot.'
+  echo '# TYPE openbao_snapshot_last_success_timestamp_seconds gauge'
+  echo "openbao_snapshot_last_success_timestamp_seconds $(date +%s)"
+} > "$metrics_tmp"
+chmod 644 "$metrics_tmp"
+mv -f -- "$metrics_tmp" "${metrics_dir}/openbao_snapshot_success.prom"
+
+# 只有产生完整快照后才做保留清理。
 ls -1t "${dir}"/openbao-*.snap 2>/dev/null | tail -n +{{ openbao_snapshot_keep + 1 }} | xargs -r rm -f

infrastructure/openbao/ansible/roles/openbao_monitoring/defaults/main.yml

@@ -0,0 +1,3 @@
+---
+openbao_metrics_listen_address: "{{ ansible_host }}:9100"
+openbao_snapshot_metrics_dir: /var/lib/prometheus/node-exporter

infrastructure/openbao/ansible/roles/openbao_monitoring/handlers/main.yml

@@ -0,0 +1,6 @@
+---
+- name: restart openbao node exporter
+  ansible.builtin.systemd:
+    name: prometheus-node-exporter
+    state: restarted
+  when: not ansible_check_mode

infrastructure/openbao/ansible/roles/openbao_monitoring/tasks/main.yml

@@ -0,0 +1,35 @@
+---
+- name: 安装 Ubuntu node_exporter 软件包
+  ansible.builtin.apt:
+    name: prometheus-node-exporter
+    state: present
+    update_cache: true
+    cache_valid_time: 3600
+    install_recommends: false
+    policy_rc_d: 101
+
+- name: 创建只允许 root 写入的 textfile 目录
+  ansible.builtin.file:
+    path: "{{ openbao_snapshot_metrics_dir }}"
+    state: directory
+    owner: root
+    group: root
+    mode: "0755"
+
+- name: 将 exporter 绑定到内网地址并启用 textfile
+  ansible.builtin.copy:
+    dest: /etc/default/prometheus-node-exporter
+    owner: root
+    group: root
+    mode: "0644"
+    content: |
+      # Ansible managed; no Bao credentials needed.
+      ARGS="--web.listen-address={{ openbao_metrics_listen_address }} --collector.textfile.directory={{ openbao_snapshot_metrics_dir }}"
+  notify: restart openbao node exporter
+
+- name: 启用主机指标服务
+  ansible.builtin.systemd:
+    name: prometheus-node-exporter
+    enabled: true
+    state: started
+  when: not ansible_check_mode