From 0988f5c37e4b3a828e9e0b9e0f784afc62a42780 Mon Sep 17 00:00:00 2001 From: panxiao81 Date: Sun, 27 Sep 2026 17:05:24 +0000 Subject: [PATCH 1/2] =?UTF-8?q?feat:=20=E8=A1=A5=E9=BD=90=E5=85=B1?= =?UTF-8?q?=E4=BA=AB=E6=95=B0=E6=8D=AE=E5=A4=87=E4=BB=BD=E3=80=81=E8=AF=81?= =?UTF-8?q?=E4=B9=A6=E7=BB=AD=E7=AD=BE=E4=B8=8E=E5=BC=80=E5=8F=91=E5=AE=9E?= =?UTF-8?q?=E4=BE=8B=E7=9B=91=E6=8E=A7?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../shared-data-monitoring/.gitignore | 1 + .../ansible/ansible.cfg | 5 + .../ansible/inventory/hosts.yml | 20 +++ .../shared-data-monitoring/ansible/site.yml | 145 +++++++++++++++ .../ansible/templates/config.json.j2 | 20 +++ .../shared-data-monitoring/files/collect.py | 124 +++++++++++++ .../tests/test_collect.py | 76 ++++++++ .../observability/metrics/kustomization.yaml | 2 + .../rules/shared-data-maintenance.yaml | 63 +++++++ .../scrapes/shared-data-maintenance.yaml | 17 ++ .../metrics/tests/check-rules.sh | 2 +- .../metrics/tests/shared-data.test.yaml | 168 ++++++++++++++++++ 12 files changed, 642 insertions(+), 1 deletion(-) create mode 100644 infrastructure/shared-data-monitoring/.gitignore create mode 100644 infrastructure/shared-data-monitoring/ansible/ansible.cfg create mode 100644 infrastructure/shared-data-monitoring/ansible/inventory/hosts.yml create mode 100644 infrastructure/shared-data-monitoring/ansible/site.yml create mode 100644 infrastructure/shared-data-monitoring/ansible/templates/config.json.j2 create mode 100644 infrastructure/shared-data-monitoring/files/collect.py create mode 100644 infrastructure/shared-data-monitoring/tests/test_collect.py create mode 100644 platform/observability/metrics/rules/shared-data-maintenance.yaml create mode 100644 platform/observability/metrics/scrapes/shared-data-maintenance.yaml create mode 100644 platform/observability/metrics/tests/shared-data.test.yaml diff --git a/infrastructure/shared-data-monitoring/.gitignore b/infrastructure/shared-data-monitoring/.gitignore new file mode 100644 index 0000000..c18dd8d --- /dev/null +++ b/infrastructure/shared-data-monitoring/.gitignore @@ -0,0 +1 @@ +__pycache__/ diff --git a/infrastructure/shared-data-monitoring/ansible/ansible.cfg b/infrastructure/shared-data-monitoring/ansible/ansible.cfg new file mode 100644 index 0000000..7f1d1ea --- /dev/null +++ b/infrastructure/shared-data-monitoring/ansible/ansible.cfg @@ -0,0 +1,5 @@ +[defaults] +inventory = inventory/hosts.yml +host_key_checking = True +retry_files_enabled = False +interpreter_python = auto_silent diff --git a/infrastructure/shared-data-monitoring/ansible/inventory/hosts.yml b/infrastructure/shared-data-monitoring/ansible/inventory/hosts.yml new file mode 100644 index 0000000..5d230d6 --- /dev/null +++ b/infrastructure/shared-data-monitoring/ansible/inventory/hosts.yml @@ -0,0 +1,20 @@ +--- +all: + children: + shared_data_monitoring: + hosts: + laptop: + ansible_connection: local + ansible_user: panxiao81 + ansible_host: 192.168.10.127 + data_monitor_dev: true + data_monitor_pg: true + data_monitor_renewals: [homelab-etcd-renew, homelab-postgresql-renew] + etcd-pve1: + ansible_host: 10.60.0.20 + ansible_user: root + data_monitor_pg: true + etcd-pve2: + ansible_host: 10.60.0.21 + ansible_user: root + data_monitor_repository: true diff --git a/infrastructure/shared-data-monitoring/ansible/site.yml b/infrastructure/shared-data-monitoring/ansible/site.yml new file mode 100644 index 0000000..791069b --- /dev/null +++ b/infrastructure/shared-data-monitoring/ansible/site.yml @@ -0,0 +1,145 @@ +--- +- name: 采集共享数据服务维护指标 + hosts: shared_data_monitoring + become: true + gather_facts: false + vars: + data_monitor_root: /opt/homelab-data-monitoring + data_monitor_exporter_version: 1.9.1 + data_monitor_download_base: https://github.com/prometheus/node_exporter/releases/download + data_monitor_archive: "node_exporter-{{ data_monitor_exporter_version }}.linux-amd64" + tasks: + - name: 检查仅支持 amd64 + ansible.builtin.command: uname -m + register: data_monitor_arch + changed_when: false + - name: 拒绝错误架构 + ansible.builtin.assert: + that: data_monitor_arch.stdout == 'x86_64' + - name: 创建 root 管理目录 + ansible.builtin.file: + path: "{{ item }}" + state: directory + owner: root + group: root + mode: '0755' + loop: + - "{{ data_monitor_root }}" + - /etc/homelab-data-monitoring + - /var/lib/homelab-data-monitoring + - name: 下载已固定校验和的上游 exporter + ansible.builtin.get_url: + url: >- + {{ data_monitor_download_base }}/v{{ data_monitor_exporter_version }}/{{ data_monitor_archive }}.tar.gz + dest: "{{ data_monitor_root }}/node_exporter.tar.gz" + checksum: sha256:becb950ee80daa8ae7331d77966d94a611af79ad0d3307380907e0ec08f5b4e8 + owner: root + group: root + mode: '0644' + - name: 解压 exporter（保留上游许可证） + ansible.builtin.unarchive: + src: "{{ data_monitor_root }}/node_exporter.tar.gz" + dest: "{{ data_monitor_root }}" + remote_src: true + creates: "{{ data_monitor_root }}/node_exporter-{{ data_monitor_exporter_version }}.linux-amd64/node_exporter" + - name: 安装只读采集程序 + ansible.builtin.copy: + src: ../files/collect.py + dest: "{{ data_monitor_root }}/collect.py" + owner: root + group: root + mode: '0755' + - name: 配置明确的证书与检查清单 + ansible.builtin.template: + src: config.json.j2 + dest: /etc/homelab-data-monitoring/config.json + owner: root + group: root + mode: '0644' + - name: 安装采集 service + ansible.builtin.copy: + dest: /etc/systemd/system/homelab-data-collect.service + mode: '0644' + content: | + [Unit] + Description=Read-only shared data maintenance checks + [Service] + Type=oneshot + ExecStart=/usr/bin/python3 {{ data_monitor_root }}/collect.py \ + --config /etc/homelab-data-monitoring/config.json \ + --output /var/lib/homelab-data-monitoring/health.prom + Environment=LC_ALL=C + Environment=PGCONNECT_TIMEOUT=5 + Environment="PGOPTIONS=-c statement_timeout=5000" + TimeoutStartSec=50 + UMask=0022 + ProtectSystem=strict + ReadWritePaths=/var/lib/homelab-data-monitoring + PrivateTmp=true + MemoryMax=96M + Nice=10 + - name: 安装每分钟采集 timer + ansible.builtin.copy: + dest: /etc/systemd/system/homelab-data-collect.timer + mode: '0644' + content: | + [Unit] + Description=Collect shared data maintenance metrics every minute + [Timer] + OnBootSec=30s + OnUnitActiveSec=60s + AccuracySec=5s + [Install] + WantedBy=timers.target + - name: 安装仅内网 textfile exporter + ansible.builtin.copy: + dest: /etc/systemd/system/homelab-data-exporter.service + mode: '0644' + content: | + [Unit] + Description=Shared data textfile metrics + After=network-online.target + Wants=network-online.target + [Service] + DynamicUser=true + ExecStart={{ data_monitor_root }}/{{ data_monitor_archive }}/node_exporter \ + --web.listen-address={{ ansible_host }}:9109 \ + --collector.disable-defaults --collector.textfile \ + --collector.textfile.directory=/var/lib/homelab-data-monitoring --web.disable-exporter-metrics + Restart=on-failure + NoNewPrivileges=true + ProtectSystem=strict + ProtectHome=true + PrivateTmp=true + MemoryMax=64M + [Install] + WantedBy=multi-user.target + notify: Restart shared data exporter + - name: 重载 unit 声明 + ansible.builtin.systemd_service: + daemon_reload: true + - name: 首次只读采集并检查程序能够完成 + ansible.builtin.command: + argv: + - /usr/bin/python3 + - "{{ data_monitor_root }}/collect.py" + - --config + - /etc/homelab-data-monitoring/config.json + - --output + - /var/lib/homelab-data-monitoring/health.prom + environment: + LC_ALL: C + PGCONNECT_TIMEOUT: '5' + PGOPTIONS: '-c statement_timeout=5000' + changed_when: false + - name: 启用监控服务 + ansible.builtin.systemd_service: + name: "{{ item }}" + enabled: true + state: started + loop: [homelab-data-collect.timer, homelab-data-exporter.service] + handlers: + - name: Restart shared data exporter + ansible.builtin.systemd_service: + name: homelab-data-exporter.service + state: restarted diff --git a/infrastructure/shared-data-monitoring/ansible/templates/config.json.j2 b/infrastructure/shared-data-monitoring/ansible/templates/config.json.j2 new file mode 100644 index 0000000..88ff34d --- /dev/null +++ b/infrastructure/shared-data-monitoring/ansible/templates/config.json.j2 @@ -0,0 +1,20 @@ +{ + "snapshot_dir": "/var/backups/homelab-etcd", + "dev": {{ data_monitor_dev | default(false) | to_json }}, + "repository": {{ data_monitor_repository | default(false) | to_json }}, + "renewal_units": {{ data_monitor_renewals | default([]) | to_json }}, + "certificates": [ +{% for name in ['server', 'peer', 'admin', 'gateway', 'ca'] %} + {"name": "etcd-{{ name }}", "path": "/etc/homelab-etcd/{{ name }}.crt"}{{ ',' if not loop.last else '' }} +{% endfor %} +{% if data_monitor_pg | default(false) %} + ,{"name": "pg-prod-server", "path": "/etc/homelab-postgresql/prod/server.crt"} + ,{"name": "pg-prod-etcd", "path": "/etc/homelab-postgresql/prod/etcd.crt"} + ,{"name": "pg-prod-ca", "path": "/etc/homelab-postgresql/prod/ca.crt"} +{% endif %} +{% if data_monitor_dev | default(false) %} + ,{"name": "pg-dev-server", "path": "/etc/homelab-postgresql/dev/server.crt"} + ,{"name": "pg-dev-ca", "path": "/etc/homelab-postgresql/dev/ca.crt"} +{% endif %} + ] +} diff --git a/infrastructure/shared-data-monitoring/files/collect.py b/infrastructure/shared-data-monitoring/files/collect.py new file mode 100644 index 0000000..068f93c --- /dev/null +++ b/infrastructure/shared-data-monitoring/files/collect.py @@ -0,0 +1,124 @@ +#!/usr/bin/env python3 +"""只读检查共享数据服务，原子发布无秘密 textfile 指标。""" +import argparse +import datetime +import json +import os +from pathlib import Path +import re +import subprocess +import tempfile +import time + + +def command(argv): + return subprocess.run(argv, check=True, capture_output=True, text=True, timeout=15).stdout.strip() + + +def certificate_expiry(path): + # 检查完整 PEM 链；任何一张缺失/损坏都不能被当作健康。 + blocks = re.findall(r'-----BEGIN CERTIFICATE-----.*?-----END CERTIFICATE-----', + Path(path).read_text(), re.S) + if not blocks: + raise ValueError('empty certificate chain') + expiries = [] + for block in blocks: + result = subprocess.run(['openssl', 'x509', '-noout', '-enddate'], input=block, + capture_output=True, text=True, check=True, timeout=5) + date = result.stdout.strip().removeprefix('notAfter=') + expiries.append(datetime.datetime.strptime(date, '%b %d %H:%M:%S %Y GMT') + .replace(tzinfo=datetime.timezone.utc).timestamp()) + return min(expiries) + + +def latest_backup(data): + stanza = next(x for x in data if x['name'] == 'prod') + if stanza['status']['code'] != 0: + raise ValueError('repository status not ok') + return max((b['timestamp']['stop'] for b in stanza['backup'] + if b['type'] == 'full' and b.get('error') is False), default=0) + + +def latest_snapshot(directory): + return max((p.stat().st_mtime for p in Path(directory).iterdir() + if re.fullmatch(r'\d{8}T\d{6}Z\.db', p.name) and p.is_file()), default=0) + + +def collect(config): + lines = [] + + def emit(name, value, **labels): + suffix = '{' + ','.join(k + '=' + json.dumps(v) for k, v in sorted(labels.items())) + '}' if labels else '' + lines.append(f'homelab_data_{name}{suffix} {float(value)}') + + def check(name, fn, **labels): + try: + value = fn() + except (OSError, ValueError, KeyError, TypeError, StopIteration, subprocess.SubprocessError): + # 不输出底层命令内容；某项失败不阻断其他检查，也不保留旧成功值。 + emit(name, 0, **labels) + emit('check_success', 0, check=name, **labels) + else: + emit(name, value, **labels) + emit('check_success', 1, check=name, **labels) + + for cert in config['certificates']: + check('certificate_expiry_timestamp_seconds', lambda c=cert: certificate_expiry(c['path']), + certificate=cert['name']) + check('backup_timestamp_seconds', lambda: latest_snapshot(config['snapshot_dir']), backup='etcd') + if config.get('repository'): + check('backup_timestamp_seconds', lambda: latest_backup(json.loads(command([ + 'runuser', '-u', 'pgbackup', '--', 'pgbackrest', + '--config=/etc/homelab-pgbackrest/repository.conf', '--stanza=prod', '--output=json', 'info' + ]))), backup='postgresql-prod') + if config.get('dev'): + def dev_health(): + result = command(['runuser', '-u', 'pgdev', '--', '/usr/lib/postgresql/18/bin/psql', + '-X', '-w', '-At', '-v', 'ON_ERROR_STOP=1', + '-h', '/run/homelab-postgresql-dev', '-p', '5433', + '-U', 'pgdev', '-d', 'postgres', '-c', + "SELECT (NOT pg_is_in_recovery() AND current_setting('transaction_read_only')='off')::int"]) + if result not in ('0', '1'): + raise ValueError('unexpected SQL response') + return int(result) + check('dev_sql_ready', dev_health) + for unit in config.get('renewal_units', []): + def renewal(u=unit): + props = dict(line.split('=', 1) for line in command([ + 'systemctl', 'show', u + '.service', '-p', 'Result', '-p', 'ExecMainExitTimestamp', + '-p', 'ExecMainStatus', '-p', 'LoadState']).splitlines()) + if props.get('LoadState') != 'loaded' or props.get('Result') != 'success' or props.get('ExecMainStatus') != '0': + raise ValueError('renewal unsuccessful') + timestamp = command(['date', '-d', props['ExecMainExitTimestamp'], '+%s']) + if command(['systemctl', 'is-active', u + '.timer']) != 'active': + raise ValueError('renewal timer inactive') + return int(timestamp) + check('renewal_success_timestamp_seconds', renewal, unit=unit) + emit('collection_timestamp_seconds', time.time()) + return '\n'.join(lines) + '\n' + + +def publish(path, content): + path = Path(path) + fd, temporary = tempfile.mkstemp(prefix='.' + path.name, dir=path.parent) + try: + with os.fdopen(fd, 'w') as stream: + stream.write(content) + stream.flush() + os.fchmod(stream.fileno(), 0o644) + os.replace(temporary, path) + finally: + if os.path.exists(temporary): + os.unlink(temporary) + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument('--config', required=True) + parser.add_argument('--output', required=True) + args = parser.parse_args() + publish(args.output, collect(json.loads(Path(args.config).read_text()))) + + +if __name__ == '__main__': + main() diff --git a/infrastructure/shared-data-monitoring/tests/test_collect.py b/infrastructure/shared-data-monitoring/tests/test_collect.py new file mode 100644 index 0000000..7785f0a --- /dev/null +++ b/infrastructure/shared-data-monitoring/tests/test_collect.py @@ -0,0 +1,76 @@ +import importlib.util +import json +from pathlib import Path +import subprocess +import tempfile +import unittest +from unittest.mock import patch + +spec = importlib.util.spec_from_file_location('collector', Path(__file__).parents[1] / 'files/collect.py') +collector = importlib.util.module_from_spec(spec) +spec.loader.exec_module(collector) + + +class CollectorTests(unittest.TestCase): + def test_full_backup_ignores_failed_and_incremental(self): + data = [{'name': 'prod', 'status': {'code': 0}, 'backup': [ + {'type': 'full', 'error': False, 'timestamp': {'stop': 100}}, + {'type': 'full', 'error': True, 'timestamp': {'stop': 200}}, + {'type': 'incr', 'error': False, 'timestamp': {'stop': 300}}, + ]}] + self.assertEqual(collector.latest_backup(data), 100) + data[0]['backup'] = [] + self.assertEqual(collector.latest_backup(data), 0) + data[0]['status']['code'] = 2 + with self.assertRaises(ValueError): + collector.latest_backup(data) + + def test_partial_snapshot_not_counted(self): + with tempfile.TemporaryDirectory() as d: + Path(d, '20260927T040000Z.db.partial').touch() + self.assertEqual(collector.latest_snapshot(d), 0) + p = Path(d, '20260926T040000Z.db') + p.touch() + self.assertEqual(collector.latest_snapshot(d), p.stat().st_mtime) + + def test_missing_certificate_does_not_hide_other_failures(self): + with tempfile.TemporaryDirectory() as d: + config = {'certificates': [{'name': 'missing', 'path': d + '/none'}], + 'snapshot_dir': d, 'dev': True, 'repository': True} + with patch.object(collector, 'command', side_effect=subprocess.TimeoutExpired('probe', 15)): + output = collector.collect(config) + self.assertIn('certificate="missing"} 0.0', output) + self.assertIn('homelab_data_dev_sql_ready 0.0', output) + self.assertIn('backup="postgresql-prod"} 0.0', output) + self.assertIn('homelab_data_collection_timestamp_seconds', output) + + def test_read_only_dev_fails_readiness(self): + with tempfile.TemporaryDirectory() as d: + with patch.object(collector, 'command', return_value='0'): + output = collector.collect({'certificates': [], 'snapshot_dir': d, 'dev': True}) + self.assertIn('homelab_data_dev_sql_ready 0.0', output) + self.assertIn('check="dev_sql_ready"} 1.0', output) + + def test_atomic_output_replaces_old_success(self): + with tempfile.TemporaryDirectory() as d: + p = Path(d, 'health.prom') + collector.publish(p, 'success 1\n') + collector.publish(p, 'success 0\n') + self.assertEqual(p.read_text(), 'success 0\n') + self.assertEqual(p.stat().st_mode & 0o777, 0o644) + self.assertEqual(list(Path(d).iterdir()), [p]) + + def test_actual_certificate_chain_earliest_expiry(self): + with tempfile.TemporaryDirectory() as d: + for days in (2, 30): + subprocess.run(['openssl', 'req', '-x509', '-newkey', 'ec', '-pkeyopt', 'ec_paramgen_curve:P-256', + '-nodes', '-subj', '/CN=test', '-days', str(days), + '-keyout', d + '/key', '-out', d + '/' + str(days)], + check=True, capture_output=True) + p = Path(d, 'chain') + p.write_text(Path(d, '30').read_text() + Path(d, '2').read_text()) + self.assertEqual(collector.certificate_expiry(p), collector.certificate_expiry(Path(d, '2'))) + + +if __name__ == '__main__': + unittest.main() diff --git a/platform/observability/metrics/kustomization.yaml b/platform/observability/metrics/kustomization.yaml index 55b8d52..74dd98f 100644 --- a/platform/observability/metrics/kustomization.yaml +++ b/platform/observability/metrics/kustomization.yaml @@ -32,3 +32,5 @@ resources: - rules/data-services.yaml - scrapes/openbao.yaml - rules/openbao.yaml + - scrapes/shared-data-maintenance.yaml + - rules/shared-data-maintenance.yaml diff --git a/platform/observability/metrics/rules/shared-data-maintenance.yaml b/platform/observability/metrics/rules/shared-data-maintenance.yaml new file mode 100644 index 0000000..7aaa738 --- /dev/null +++ b/platform/observability/metrics/rules/shared-data-maintenance.yaml @@ -0,0 +1,63 @@ +apiVersion: operator.victoriametrics.com/v1beta1 +kind: VMRule +metadata: + name: shared-data-maintenance + namespace: monitoring +spec: + groups: + - name: shared-data-maintenance + rules: + - alert: SharedDataMonitoringUnavailable + expr: up{job="shared-data-maintenance"} == 0 or absent(up{job="shared-data-maintenance"}) + for: 3m + labels: {severity: warning, service: shared-data} + annotations: + summary: 共享数据维护指标无法采集 + - alert: SharedDataCollectorStale + expr: >- + (time() - homelab_data_collection_timestamp_seconds{job="shared-data-maintenance"} > 180) + or (up{job="shared-data-maintenance"} == 1 + unless on(instance) homelab_data_collection_timestamp_seconds{job="shared-data-maintenance"}) + or node_textfile_scrape_error{job="shared-data-maintenance"} == 1 + for: 3m + labels: {severity: warning, service: shared-data} + annotations: + summary: 共享数据本地采集结果缺失或停止更新 + - alert: SharedDataCheckFailed + expr: homelab_data_check_success{job="shared-data-maintenance"} == 0 + for: 3m + labels: {severity: warning, service: shared-data} + annotations: + summary: 共享数据维护检查失败，请核对对应检查项 + - alert: SharedDataBackupStale + expr: time() - homelab_data_backup_timestamp_seconds{job="shared-data-maintenance"} > 36 * 3600 + for: 5m + labels: {severity: warning, service: shared-data} + annotations: + summary: 共享数据备份已超过 36 小时或没有成功备份 + - alert: SharedDataCertificateExpiringSoon + expr: >- + homelab_data_certificate_expiry_timestamp_seconds{job="shared-data-maintenance"} - time() < 10 * 86400 + and homelab_data_certificate_expiry_timestamp_seconds{job="shared-data-maintenance"} - time() >= 3 * 86400 + for: 5m + labels: {severity: warning, service: shared-data} + annotations: + summary: 共享数据证书剩余不足 10 天，请检查自动续签 + - alert: SharedDataCertificateExpiryCritical + expr: homelab_data_certificate_expiry_timestamp_seconds{job="shared-data-maintenance"} - time() < 3 * 86400 + for: 5m + labels: {severity: critical, service: shared-data} + annotations: + summary: 共享数据证书不足 3 天、已过期或无法读取 + - alert: SharedDataRenewalStale + expr: time() - homelab_data_renewal_success_timestamp_seconds{job="shared-data-maintenance"} > 36 * 3600 + for: 5m + labels: {severity: warning, service: shared-data} + annotations: + summary: 共享数据续签检查失败、停用或超过 36 小时未成功 + - alert: SharedPostgresDevUnavailable + expr: homelab_data_dev_sql_ready{job="shared-data-maintenance"} == 0 + for: 3m + labels: {severity: warning, service: shared-postgresql} + annotations: + summary: 共享 PG 开发实例本地 SQL 不可用或处于只读状态 diff --git a/platform/observability/metrics/scrapes/shared-data-maintenance.yaml b/platform/observability/metrics/scrapes/shared-data-maintenance.yaml new file mode 100644 index 0000000..cb43b8f --- /dev/null +++ b/platform/observability/metrics/scrapes/shared-data-maintenance.yaml @@ -0,0 +1,17 @@ +apiVersion: operator.victoriametrics.com/v1beta1 +kind: VMStaticScrape +metadata: + name: shared-data-maintenance + namespace: monitoring +spec: + jobName: shared-data-maintenance + targetEndpoints: + - targets: ['192.168.10.127:9109'] + labels: {member: laptop} + interval: 30s + - targets: ['10.60.0.20:9109'] + labels: {member: etcd-pve1} + interval: 30s + - targets: ['10.60.0.21:9109'] + labels: {member: etcd-pve2} + interval: 30s diff --git a/platform/observability/metrics/tests/check-rules.sh b/platform/observability/metrics/tests/check-rules.sh index a093dce..ecc3449 100644 --- a/platform/observability/metrics/tests/check-rules.sh +++ b/platform/observability/metrics/tests/check-rules.sh @@ -10,7 +10,7 @@ import sys import yaml groups = [] -for name in ("kubernetes-health", "host-health", "monitoring-delivery", "platform-controllers", "data-services", "openbao"): +for name in ("kubernetes-health", "host-health", "monitoring-delivery", "platform-controllers", "data-services", "openbao", "shared-data-maintenance"): path = pathlib.Path(sys.argv[1]) / f"{name}.yaml" groups.extend(yaml.safe_load(path.read_text())["spec"]["groups"]) pathlib.Path(sys.argv[2]).write_text(yaml.safe_dump({"groups": groups}, allow_unicode=True)) diff --git a/platform/observability/metrics/tests/shared-data.test.yaml b/platform/observability/metrics/tests/shared-data.test.yaml new file mode 100644 index 0000000..dcac681 --- /dev/null +++ b/platform/observability/metrics/tests/shared-data.test.yaml @@ -0,0 +1,168 @@ +rule_files: +- rules.yaml +evaluation_interval: 1m +tests: +- name: 过期证书持续 critical + interval: 1m + input_series: + - series: homelab_data_certificate_expiry_timestamp_seconds{job="shared-data-maintenance",instance="host:9109",certificate="etcd-peer"} + values: '0x15' + alert_rule_test: + - eval_time: 10m + alertname: SharedDataCertificateExpiryCritical + exp_alerts: + - exp_labels: + job: shared-data-maintenance + instance: host:9109 + certificate: etcd-peer + severity: critical + service: shared-data + exp_annotations: + summary: 共享数据证书不足 3 天、已过期或无法读取 +- name: critical 证书不再重复 warning + interval: 1m + input_series: + - series: homelab_data_certificate_expiry_timestamp_seconds{job="shared-data-maintenance",instance="host:9109",certificate="etcd-peer"} + values: '0x15' + alert_rule_test: + - eval_time: 10m + alertname: SharedDataCertificateExpiringSoon + exp_alerts: [] +- name: 正常证书不误报 + interval: 1m + input_series: + - series: homelab_data_certificate_expiry_timestamp_seconds{job="shared-data-maintenance",instance="host:9109",certificate="etcd-peer"} + values: 5184000x15 + alert_rule_test: + - eval_time: 10m + alertname: SharedDataCertificateExpiringSoon + exp_alerts: [] +- name: 七天内证书预警 + interval: 1m + input_series: + - series: homelab_data_certificate_expiry_timestamp_seconds{job="shared-data-maintenance",instance="host:9109",certificate="etcd-peer"} + values: 604800x15 + alert_rule_test: + - eval_time: 10m + alertname: SharedDataCertificateExpiringSoon + exp_alerts: + - exp_labels: + job: shared-data-maintenance + instance: host:9109 + certificate: etcd-peer + severity: warning + service: shared-data + exp_annotations: + summary: 共享数据证书剩余不足 10 天，请检查自动续签 +- name: 备份超过36小时 + interval: 1m + input_series: + - series: homelab_data_backup_timestamp_seconds{job="shared-data-maintenance",instance="host:9109",backup="etcd"} + values: -130000x15 + alert_rule_test: + - eval_time: 10m + alertname: SharedDataBackupStale + exp_alerts: + - exp_labels: + job: shared-data-maintenance + instance: host:9109 + backup: etcd + severity: warning + service: shared-data + exp_annotations: + summary: 共享数据备份已超过 36 小时或没有成功备份 +- name: 新备份没有告警 + interval: 1m + input_series: + - series: homelab_data_backup_timestamp_seconds{job="shared-data-maintenance",instance="host:9109",backup="etcd"} + values: 0+60x15 + alert_rule_test: + - eval_time: 10m + alertname: SharedDataBackupStale + exp_alerts: [] +- name: exporter 存活但没有采集文件 + interval: 1m + input_series: + - series: up{job="shared-data-maintenance",instance="host:9109"} + values: 1x15 + alert_rule_test: + - eval_time: 10m + alertname: SharedDataCollectorStale + exp_alerts: + - exp_labels: + job: shared-data-maintenance + instance: host:9109 + severity: warning + service: shared-data + exp_annotations: &id001 + summary: 共享数据本地采集结果缺失或停止更新 +- name: 采集心跳过期 + interval: 1m + input_series: + - series: homelab_data_collection_timestamp_seconds{job="shared-data-maintenance",instance="host:9109"} + values: '0x15' + alert_rule_test: + - eval_time: 10m + alertname: SharedDataCollectorStale + exp_alerts: + - exp_labels: + job: shared-data-maintenance + instance: host:9109 + severity: warning + service: shared-data + exp_annotations: *id001 +- name: SQL 拒绝和只读触发 dev 告警 + interval: 1m + input_series: + - series: homelab_data_dev_sql_ready{job="shared-data-maintenance",instance="host:9109"} + values: '0x15' + alert_rule_test: + - eval_time: 10m + alertname: SharedPostgresDevUnavailable + exp_alerts: + - exp_labels: + job: shared-data-maintenance + instance: host:9109 + severity: warning + service: shared-postgresql + exp_annotations: + summary: 共享 PG 开发实例本地 SQL 不可用或处于只读状态 +- name: dev 恢复后解除 + interval: 1m + input_series: + - series: homelab_data_dev_sql_ready{job="shared-data-maintenance",instance="host:9109"} + values: 0x4 1x10 + alert_rule_test: + - eval_time: 10m + alertname: SharedPostgresDevUnavailable + exp_alerts: [] +- name: 续签检查超过36小时 + interval: 1m + input_series: + - series: homelab_data_renewal_success_timestamp_seconds{job="shared-data-maintenance",instance="host:9109",unit="homelab-etcd-renew"} + values: -130000x15 + alert_rule_test: + - eval_time: 10m + alertname: SharedDataRenewalStale + exp_alerts: + - exp_labels: + job: shared-data-maintenance + instance: host:9109 + unit: homelab-etcd-renew + severity: warning + service: shared-data + exp_annotations: + summary: 共享数据续签检查失败、停用或超过 36 小时未成功 +- name: 整个 scrape 消失仍报警 + interval: 1m + input_series: [] + alert_rule_test: + - eval_time: 10m + alertname: SharedDataMonitoringUnavailable + exp_alerts: + - exp_labels: + severity: warning + service: shared-data + job: shared-data-maintenance + exp_annotations: + summary: 共享数据维护指标无法采集 -- 2.54.0 From 0d633cf0cc314dc41a483254050f96dfeb0d27f3 Mon Sep 17 00:00:00 2001 From: panxiao81 Date: Sun, 27 Sep 2026 17:29:40 +0000 Subject: [PATCH 2/2] =?UTF-8?q?fix:=20=E4=BD=BF=E7=94=A8=20PrometheusRule?= =?UTF-8?q?=20=E5=B9=B6=E6=8C=89=20exporter=20=E7=89=88=E6=9C=AC=E8=8E=B7?= =?UTF-8?q?=E5=8F=96=E5=AE=98=E6=96=B9=E6=A0=A1=E9=AA=8C=E5=92=8C?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- infrastructure/shared-data-monitoring/ansible/site.yml | 5 +++-- .../observability/metrics/rules/shared-data-maintenance.yaml | 4 ++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/infrastructure/shared-data-monitoring/ansible/site.yml b/infrastructure/shared-data-monitoring/ansible/site.yml index 791069b..cf64f3b 100644 --- a/infrastructure/shared-data-monitoring/ansible/site.yml +++ b/infrastructure/shared-data-monitoring/ansible/site.yml @@ -27,12 +27,13 @@ - "{{ data_monitor_root }}" - /etc/homelab-data-monitoring - /var/lib/homelab-data-monitoring - - name: 下载已固定校验和的上游 exporter + - name: 下载 exporter 并校验同版本官方摘要 ansible.builtin.get_url: url: >- {{ data_monitor_download_base }}/v{{ data_monitor_exporter_version }}/{{ data_monitor_archive }}.tar.gz dest: "{{ data_monitor_root }}/node_exporter.tar.gz" - checksum: sha256:becb950ee80daa8ae7331d77966d94a611af79ad0d3307380907e0ec08f5b4e8 + checksum: >- + sha256:{{ data_monitor_download_base }}/v{{ data_monitor_exporter_version }}/sha256sums.txt owner: root group: root mode: '0644' diff --git a/platform/observability/metrics/rules/shared-data-maintenance.yaml b/platform/observability/metrics/rules/shared-data-maintenance.yaml index 7aaa738..2b11e44 100644 --- a/platform/observability/metrics/rules/shared-data-maintenance.yaml +++ b/platform/observability/metrics/rules/shared-data-maintenance.yaml @@ -1,5 +1,5 @@ -apiVersion: operator.victoriametrics.com/v1beta1 -kind: VMRule +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule metadata: name: shared-data-maintenance namespace: monitoring -- 2.54.0