Merge pull request '补齐共享 etcd/PG 的备份、证书续签及 dev 健康告警' (#166) from feat/shared-pg-health into main

Reviewed-on: #166

infrastructure/shared-data-monitoring/.gitignore

@@ -0,0 +1 @@
+__pycache__/

infrastructure/shared-data-monitoring/ansible/ansible.cfg

@@ -0,0 +1,5 @@
+[defaults]
+inventory = inventory/hosts.yml
+host_key_checking = True
+retry_files_enabled = False
+interpreter_python = auto_silent

infrastructure/shared-data-monitoring/ansible/inventory/hosts.yml

@@ -0,0 +1,20 @@
+---
+all:
+  children:
+    shared_data_monitoring:
+      hosts:
+        laptop:
+          ansible_connection: local
+          ansible_user: panxiao81
+          ansible_host: 192.168.10.127
+          data_monitor_dev: true
+          data_monitor_pg: true
+          data_monitor_renewals: [homelab-etcd-renew, homelab-postgresql-renew]
+        etcd-pve1:
+          ansible_host: 10.60.0.20
+          ansible_user: root
+          data_monitor_pg: true
+        etcd-pve2:
+          ansible_host: 10.60.0.21
+          ansible_user: root
+          data_monitor_repository: true

infrastructure/shared-data-monitoring/ansible/site.yml

@@ -0,0 +1,146 @@
+---
+- name: 采集共享数据服务维护指标
+  hosts: shared_data_monitoring
+  become: true
+  gather_facts: false
+  vars:
+    data_monitor_root: /opt/homelab-data-monitoring
+    data_monitor_exporter_version: 1.9.1
+    data_monitor_download_base: https://github.com/prometheus/node_exporter/releases/download
+    data_monitor_archive: "node_exporter-{{ data_monitor_exporter_version }}.linux-amd64"
+  tasks:
+    - name: 检查仅支持 amd64
+      ansible.builtin.command: uname -m
+      register: data_monitor_arch
+      changed_when: false
+    - name: 拒绝错误架构
+      ansible.builtin.assert:
+        that: data_monitor_arch.stdout == 'x86_64'
+    - name: 创建 root 管理目录
+      ansible.builtin.file:
+        path: "{{ item }}"
+        state: directory
+        owner: root
+        group: root
+        mode: '0755'
+      loop:
+        - "{{ data_monitor_root }}"
+        - /etc/homelab-data-monitoring
+        - /var/lib/homelab-data-monitoring
+    - name: 下载 exporter 并校验同版本官方摘要
+      ansible.builtin.get_url:
+        url: >-
+          {{ data_monitor_download_base }}/v{{ data_monitor_exporter_version }}/{{ data_monitor_archive }}.tar.gz
+        dest: "{{ data_monitor_root }}/node_exporter.tar.gz"
+        checksum: >-
+          sha256:{{ data_monitor_download_base }}/v{{ data_monitor_exporter_version }}/sha256sums.txt
+        owner: root
+        group: root
+        mode: '0644'
+    - name: 解压 exporter（保留上游许可证）
+      ansible.builtin.unarchive:
+        src: "{{ data_monitor_root }}/node_exporter.tar.gz"
+        dest: "{{ data_monitor_root }}"
+        remote_src: true
+        creates: "{{ data_monitor_root }}/node_exporter-{{ data_monitor_exporter_version }}.linux-amd64/node_exporter"
+    - name: 安装只读采集程序
+      ansible.builtin.copy:
+        src: ../files/collect.py
+        dest: "{{ data_monitor_root }}/collect.py"
+        owner: root
+        group: root
+        mode: '0755'
+    - name: 配置明确的证书与检查清单
+      ansible.builtin.template:
+        src: config.json.j2
+        dest: /etc/homelab-data-monitoring/config.json
+        owner: root
+        group: root
+        mode: '0644'
+    - name: 安装采集 service
+      ansible.builtin.copy:
+        dest: /etc/systemd/system/homelab-data-collect.service
+        mode: '0644'
+        content: |
+          [Unit]
+          Description=Read-only shared data maintenance checks
+          [Service]
+          Type=oneshot
+          ExecStart=/usr/bin/python3 {{ data_monitor_root }}/collect.py \
+            --config /etc/homelab-data-monitoring/config.json \
+            --output /var/lib/homelab-data-monitoring/health.prom
+          Environment=LC_ALL=C
+          Environment=PGCONNECT_TIMEOUT=5
+          Environment="PGOPTIONS=-c statement_timeout=5000"
+          TimeoutStartSec=50
+          UMask=0022
+          ProtectSystem=strict
+          ReadWritePaths=/var/lib/homelab-data-monitoring
+          PrivateTmp=true
+          MemoryMax=96M
+          Nice=10
+    - name: 安装每分钟采集 timer
+      ansible.builtin.copy:
+        dest: /etc/systemd/system/homelab-data-collect.timer
+        mode: '0644'
+        content: |
+          [Unit]
+          Description=Collect shared data maintenance metrics every minute
+          [Timer]
+          OnBootSec=30s
+          OnUnitActiveSec=60s
+          AccuracySec=5s
+          [Install]
+          WantedBy=timers.target
+    - name: 安装仅内网 textfile exporter
+      ansible.builtin.copy:
+        dest: /etc/systemd/system/homelab-data-exporter.service
+        mode: '0644'
+        content: |
+          [Unit]
+          Description=Shared data textfile metrics
+          After=network-online.target
+          Wants=network-online.target
+          [Service]
+          DynamicUser=true
+          ExecStart={{ data_monitor_root }}/{{ data_monitor_archive }}/node_exporter \
+            --web.listen-address={{ ansible_host }}:9109 \
+            --collector.disable-defaults --collector.textfile \
+            --collector.textfile.directory=/var/lib/homelab-data-monitoring --web.disable-exporter-metrics
+          Restart=on-failure
+          NoNewPrivileges=true
+          ProtectSystem=strict
+          ProtectHome=true
+          PrivateTmp=true
+          MemoryMax=64M
+          [Install]
+          WantedBy=multi-user.target
+      notify: Restart shared data exporter
+    - name: 重载 unit 声明
+      ansible.builtin.systemd_service:
+        daemon_reload: true
+    - name: 首次只读采集并检查程序能够完成
+      ansible.builtin.command:
+        argv:
+          - /usr/bin/python3
+          - "{{ data_monitor_root }}/collect.py"
+          - --config
+          - /etc/homelab-data-monitoring/config.json
+          - --output
+          - /var/lib/homelab-data-monitoring/health.prom
+      environment:
+        LC_ALL: C
+        PGCONNECT_TIMEOUT: '5'
+        PGOPTIONS: '-c statement_timeout=5000'
+      changed_when: false
+    - name: 启用监控服务
+      ansible.builtin.systemd_service:
+        name: "{{ item }}"
+        enabled: true
+        state: started
+      loop: [homelab-data-collect.timer, homelab-data-exporter.service]
+  handlers:
+    - name: Restart shared data exporter
+      ansible.builtin.systemd_service:
+        name: homelab-data-exporter.service
+        state: restarted

infrastructure/shared-data-monitoring/ansible/templates/config.json.j2

@@ -0,0 +1,20 @@
+{
+  "snapshot_dir": "/var/backups/homelab-etcd",
+  "dev": {{ data_monitor_dev | default(false) | to_json }},
+  "repository": {{ data_monitor_repository | default(false) | to_json }},
+  "renewal_units": {{ data_monitor_renewals | default([]) | to_json }},
+  "certificates": [
+{% for name in ['server', 'peer', 'admin', 'gateway', 'ca'] %}
+    {"name": "etcd-{{ name }}", "path": "/etc/homelab-etcd/{{ name }}.crt"}{{ ',' if not loop.last else '' }}
+{% endfor %}
+{% if data_monitor_pg | default(false) %}
+    ,{"name": "pg-prod-server", "path": "/etc/homelab-postgresql/prod/server.crt"}
+    ,{"name": "pg-prod-etcd", "path": "/etc/homelab-postgresql/prod/etcd.crt"}
+    ,{"name": "pg-prod-ca", "path": "/etc/homelab-postgresql/prod/ca.crt"}
+{% endif %}
+{% if data_monitor_dev | default(false) %}
+    ,{"name": "pg-dev-server", "path": "/etc/homelab-postgresql/dev/server.crt"}
+    ,{"name": "pg-dev-ca", "path": "/etc/homelab-postgresql/dev/ca.crt"}
+{% endif %}
+  ]
+}

infrastructure/shared-data-monitoring/files/collect.py

@@ -0,0 +1,124 @@
+#!/usr/bin/env python3
+"""只读检查共享数据服务，原子发布无秘密 textfile 指标。"""
+import argparse
+import datetime
+import json
+import os
+from pathlib import Path
+import re
+import subprocess
+import tempfile
+import time
+
+
+def command(argv):
+    return subprocess.run(argv, check=True, capture_output=True, text=True, timeout=15).stdout.strip()
+
+
+def certificate_expiry(path):
+    # 检查完整 PEM 链；任何一张缺失/损坏都不能被当作健康。
+    blocks = re.findall(r'-----BEGIN CERTIFICATE-----.*?-----END CERTIFICATE-----',
+                        Path(path).read_text(), re.S)
+    if not blocks:
+        raise ValueError('empty certificate chain')
+    expiries = []
+    for block in blocks:
+        result = subprocess.run(['openssl', 'x509', '-noout', '-enddate'], input=block,
+                                capture_output=True, text=True, check=True, timeout=5)
+        date = result.stdout.strip().removeprefix('notAfter=')
+        expiries.append(datetime.datetime.strptime(date, '%b %d %H:%M:%S %Y GMT')
+                        .replace(tzinfo=datetime.timezone.utc).timestamp())
+    return min(expiries)
+
+
+def latest_backup(data):
+    stanza = next(x for x in data if x['name'] == 'prod')
+    if stanza['status']['code'] != 0:
+        raise ValueError('repository status not ok')
+    return max((b['timestamp']['stop'] for b in stanza['backup']
+                if b['type'] == 'full' and b.get('error') is False), default=0)
+
+
+def latest_snapshot(directory):
+    return max((p.stat().st_mtime for p in Path(directory).iterdir()
+                if re.fullmatch(r'\d{8}T\d{6}Z\.db', p.name) and p.is_file()), default=0)
+
+
+def collect(config):
+    lines = []
+
+    def emit(name, value, **labels):
+        suffix = '{' + ','.join(k + '=' + json.dumps(v) for k, v in sorted(labels.items())) + '}' if labels else ''
+        lines.append(f'homelab_data_{name}{suffix} {float(value)}')
+
+    def check(name, fn, **labels):
+        try:
+            value = fn()
+        except (OSError, ValueError, KeyError, TypeError, StopIteration, subprocess.SubprocessError):
+            # 不输出底层命令内容；某项失败不阻断其他检查，也不保留旧成功值。
+            emit(name, 0, **labels)
+            emit('check_success', 0, check=name, **labels)
+        else:
+            emit(name, value, **labels)
+            emit('check_success', 1, check=name, **labels)
+
+    for cert in config['certificates']:
+        check('certificate_expiry_timestamp_seconds', lambda c=cert: certificate_expiry(c['path']),
+              certificate=cert['name'])
+    check('backup_timestamp_seconds', lambda: latest_snapshot(config['snapshot_dir']), backup='etcd')
+    if config.get('repository'):
+        check('backup_timestamp_seconds', lambda: latest_backup(json.loads(command([
+            'runuser', '-u', 'pgbackup', '--', 'pgbackrest',
+            '--config=/etc/homelab-pgbackrest/repository.conf', '--stanza=prod', '--output=json', 'info'
+        ]))), backup='postgresql-prod')
+    if config.get('dev'):
+        def dev_health():
+            result = command(['runuser', '-u', 'pgdev', '--', '/usr/lib/postgresql/18/bin/psql',
+                              '-X', '-w', '-At', '-v', 'ON_ERROR_STOP=1',
+                              '-h', '/run/homelab-postgresql-dev', '-p', '5433',
+                              '-U', 'pgdev', '-d', 'postgres', '-c',
+                              "SELECT (NOT pg_is_in_recovery() AND current_setting('transaction_read_only')='off')::int"])
+            if result not in ('0', '1'):
+                raise ValueError('unexpected SQL response')
+            return int(result)
+        check('dev_sql_ready', dev_health)
+    for unit in config.get('renewal_units', []):
+        def renewal(u=unit):
+            props = dict(line.split('=', 1) for line in command([
+                'systemctl', 'show', u + '.service', '-p', 'Result', '-p', 'ExecMainExitTimestamp',
+                '-p', 'ExecMainStatus', '-p', 'LoadState']).splitlines())
+            if props.get('LoadState') != 'loaded' or props.get('Result') != 'success' or props.get('ExecMainStatus') != '0':
+                raise ValueError('renewal unsuccessful')
+            timestamp = command(['date', '-d', props['ExecMainExitTimestamp'], '+%s'])
+            if command(['systemctl', 'is-active', u + '.timer']) != 'active':
+                raise ValueError('renewal timer inactive')
+            return int(timestamp)
+        check('renewal_success_timestamp_seconds', renewal, unit=unit)
+    emit('collection_timestamp_seconds', time.time())
+    return '\n'.join(lines) + '\n'
+
+
+def publish(path, content):
+    path = Path(path)
+    fd, temporary = tempfile.mkstemp(prefix='.' + path.name, dir=path.parent)
+    try:
+        with os.fdopen(fd, 'w') as stream:
+            stream.write(content)
+            stream.flush()
+            os.fchmod(stream.fileno(), 0o644)
+        os.replace(temporary, path)
+    finally:
+        if os.path.exists(temporary):
+            os.unlink(temporary)
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--config', required=True)
+    parser.add_argument('--output', required=True)
+    args = parser.parse_args()
+    publish(args.output, collect(json.loads(Path(args.config).read_text())))
+
+
+if __name__ == '__main__':
+    main()

infrastructure/shared-data-monitoring/tests/test_collect.py

@@ -0,0 +1,76 @@
+import importlib.util
+import json
+from pathlib import Path
+import subprocess
+import tempfile
+import unittest
+from unittest.mock import patch
+
+spec = importlib.util.spec_from_file_location('collector', Path(__file__).parents[1] / 'files/collect.py')
+collector = importlib.util.module_from_spec(spec)
+spec.loader.exec_module(collector)
+
+
+class CollectorTests(unittest.TestCase):
+    def test_full_backup_ignores_failed_and_incremental(self):
+        data = [{'name': 'prod', 'status': {'code': 0}, 'backup': [
+            {'type': 'full', 'error': False, 'timestamp': {'stop': 100}},
+            {'type': 'full', 'error': True, 'timestamp': {'stop': 200}},
+            {'type': 'incr', 'error': False, 'timestamp': {'stop': 300}},
+        ]}]
+        self.assertEqual(collector.latest_backup(data), 100)
+        data[0]['backup'] = []
+        self.assertEqual(collector.latest_backup(data), 0)
+        data[0]['status']['code'] = 2
+        with self.assertRaises(ValueError):
+            collector.latest_backup(data)
+
+    def test_partial_snapshot_not_counted(self):
+        with tempfile.TemporaryDirectory() as d:
+            Path(d, '20260927T040000Z.db.partial').touch()
+            self.assertEqual(collector.latest_snapshot(d), 0)
+            p = Path(d, '20260926T040000Z.db')
+            p.touch()
+            self.assertEqual(collector.latest_snapshot(d), p.stat().st_mtime)
+
+    def test_missing_certificate_does_not_hide_other_failures(self):
+        with tempfile.TemporaryDirectory() as d:
+            config = {'certificates': [{'name': 'missing', 'path': d + '/none'}],
+                      'snapshot_dir': d, 'dev': True, 'repository': True}
+            with patch.object(collector, 'command', side_effect=subprocess.TimeoutExpired('probe', 15)):
+                output = collector.collect(config)
+            self.assertIn('certificate="missing"} 0.0', output)
+            self.assertIn('homelab_data_dev_sql_ready 0.0', output)
+            self.assertIn('backup="postgresql-prod"} 0.0', output)
+            self.assertIn('homelab_data_collection_timestamp_seconds', output)
+
+    def test_read_only_dev_fails_readiness(self):
+        with tempfile.TemporaryDirectory() as d:
+            with patch.object(collector, 'command', return_value='0'):
+                output = collector.collect({'certificates': [], 'snapshot_dir': d, 'dev': True})
+            self.assertIn('homelab_data_dev_sql_ready 0.0', output)
+            self.assertIn('check="dev_sql_ready"} 1.0', output)
+
+    def test_atomic_output_replaces_old_success(self):
+        with tempfile.TemporaryDirectory() as d:
+            p = Path(d, 'health.prom')
+            collector.publish(p, 'success 1\n')
+            collector.publish(p, 'success 0\n')
+            self.assertEqual(p.read_text(), 'success 0\n')
+            self.assertEqual(p.stat().st_mode & 0o777, 0o644)
+            self.assertEqual(list(Path(d).iterdir()), [p])
+
+    def test_actual_certificate_chain_earliest_expiry(self):
+        with tempfile.TemporaryDirectory() as d:
+            for days in (2, 30):
+                subprocess.run(['openssl', 'req', '-x509', '-newkey', 'ec', '-pkeyopt', 'ec_paramgen_curve:P-256',
+                                '-nodes', '-subj', '/CN=test', '-days', str(days),
+                                '-keyout', d + '/key', '-out', d + '/' + str(days)],
+                               check=True, capture_output=True)
+            p = Path(d, 'chain')
+            p.write_text(Path(d, '30').read_text() + Path(d, '2').read_text())
+            self.assertEqual(collector.certificate_expiry(p), collector.certificate_expiry(Path(d, '2')))
+
+
+if __name__ == '__main__':
+    unittest.main()