补齐共享 etcd/PG 的备份、证书续签及 dev 健康告警 #166

Merged
panxiao81 merged 2 commits from feat/shared-pg-health into main 2026-09-27 17:35:28 +00:00
12 changed files with 643 additions and 1 deletions
@@ -0,0 +1 @@
__pycache__/
@@ -0,0 +1,5 @@
[defaults]
inventory = inventory/hosts.yml
host_key_checking = True
retry_files_enabled = False
interpreter_python = auto_silent
@@ -0,0 +1,20 @@
---
all:
children:
shared_data_monitoring:
hosts:
laptop:
ansible_connection: local
ansible_user: panxiao81
ansible_host: 192.168.10.127
data_monitor_dev: true
data_monitor_pg: true
data_monitor_renewals: [homelab-etcd-renew, homelab-postgresql-renew]
etcd-pve1:
ansible_host: 10.60.0.20
ansible_user: root
data_monitor_pg: true
etcd-pve2:
ansible_host: 10.60.0.21
ansible_user: root
data_monitor_repository: true
@@ -0,0 +1,146 @@
---
- name: 采集共享数据服务维护指标
hosts: shared_data_monitoring
become: true
gather_facts: false
vars:
data_monitor_root: /opt/homelab-data-monitoring
data_monitor_exporter_version: 1.9.1
data_monitor_download_base: https://github.com/prometheus/node_exporter/releases/download
data_monitor_archive: "node_exporter-{{ data_monitor_exporter_version }}.linux-amd64"
tasks:
- name: 检查仅支持 amd64
ansible.builtin.command: uname -m
register: data_monitor_arch
changed_when: false
- name: 拒绝错误架构
ansible.builtin.assert:
that: data_monitor_arch.stdout == 'x86_64'
- name: 创建 root 管理目录
ansible.builtin.file:
path: "{{ item }}"
state: directory
owner: root
group: root
mode: '0755'
loop:
- "{{ data_monitor_root }}"
- /etc/homelab-data-monitoring
- /var/lib/homelab-data-monitoring
- name: 下载 exporter 并校验同版本官方摘要
ansible.builtin.get_url:
url: >-
{{ data_monitor_download_base }}/v{{ data_monitor_exporter_version }}/{{ data_monitor_archive }}.tar.gz
dest: "{{ data_monitor_root }}/node_exporter.tar.gz"
checksum: >-
sha256:{{ data_monitor_download_base }}/v{{ data_monitor_exporter_version }}/sha256sums.txt
owner: root
group: root
mode: '0644'
- name: 解压 exporter(保留上游许可证)
ansible.builtin.unarchive:
src: "{{ data_monitor_root }}/node_exporter.tar.gz"
dest: "{{ data_monitor_root }}"
remote_src: true
creates: "{{ data_monitor_root }}/node_exporter-{{ data_monitor_exporter_version }}.linux-amd64/node_exporter"
- name: 安装只读采集程序
ansible.builtin.copy:
src: ../files/collect.py
dest: "{{ data_monitor_root }}/collect.py"
owner: root
group: root
mode: '0755'
- name: 配置明确的证书与检查清单
ansible.builtin.template:
src: config.json.j2
dest: /etc/homelab-data-monitoring/config.json
owner: root
group: root
mode: '0644'
- name: 安装采集 service
ansible.builtin.copy:
dest: /etc/systemd/system/homelab-data-collect.service
mode: '0644'
content: |
[Unit]
Description=Read-only shared data maintenance checks
[Service]
Type=oneshot
ExecStart=/usr/bin/python3 {{ data_monitor_root }}/collect.py \
--config /etc/homelab-data-monitoring/config.json \
--output /var/lib/homelab-data-monitoring/health.prom
Environment=LC_ALL=C
Environment=PGCONNECT_TIMEOUT=5
Environment="PGOPTIONS=-c statement_timeout=5000"
TimeoutStartSec=50
UMask=0022
ProtectSystem=strict
ReadWritePaths=/var/lib/homelab-data-monitoring
PrivateTmp=true
MemoryMax=96M
Nice=10
- name: 安装每分钟采集 timer
ansible.builtin.copy:
dest: /etc/systemd/system/homelab-data-collect.timer
mode: '0644'
content: |
[Unit]
Description=Collect shared data maintenance metrics every minute
[Timer]
OnBootSec=30s
OnUnitActiveSec=60s
AccuracySec=5s
[Install]
WantedBy=timers.target
- name: 安装仅内网 textfile exporter
ansible.builtin.copy:
dest: /etc/systemd/system/homelab-data-exporter.service
mode: '0644'
content: |
[Unit]
Description=Shared data textfile metrics
After=network-online.target
Wants=network-online.target
[Service]
DynamicUser=true
ExecStart={{ data_monitor_root }}/{{ data_monitor_archive }}/node_exporter \
--web.listen-address={{ ansible_host }}:9109 \
--collector.disable-defaults --collector.textfile \
--collector.textfile.directory=/var/lib/homelab-data-monitoring --web.disable-exporter-metrics
Restart=on-failure
NoNewPrivileges=true
ProtectSystem=strict
ProtectHome=true
PrivateTmp=true
MemoryMax=64M
[Install]
WantedBy=multi-user.target
notify: Restart shared data exporter
- name: 重载 unit 声明
ansible.builtin.systemd_service:
daemon_reload: true
- name: 首次只读采集并检查程序能够完成
ansible.builtin.command:
argv:
- /usr/bin/python3
- "{{ data_monitor_root }}/collect.py"
- --config
- /etc/homelab-data-monitoring/config.json
- --output
- /var/lib/homelab-data-monitoring/health.prom
environment:
LC_ALL: C
PGCONNECT_TIMEOUT: '5'
PGOPTIONS: '-c statement_timeout=5000'
changed_when: false
- name: 启用监控服务
ansible.builtin.systemd_service:
name: "{{ item }}"
enabled: true
state: started
loop: [homelab-data-collect.timer, homelab-data-exporter.service]
handlers:
- name: Restart shared data exporter
ansible.builtin.systemd_service:
name: homelab-data-exporter.service
state: restarted
@@ -0,0 +1,20 @@
{
"snapshot_dir": "/var/backups/homelab-etcd",
"dev": {{ data_monitor_dev | default(false) | to_json }},
"repository": {{ data_monitor_repository | default(false) | to_json }},
"renewal_units": {{ data_monitor_renewals | default([]) | to_json }},
"certificates": [
{% for name in ['server', 'peer', 'admin', 'gateway', 'ca'] %}
{"name": "etcd-{{ name }}", "path": "/etc/homelab-etcd/{{ name }}.crt"}{{ ',' if not loop.last else '' }}
{% endfor %}
{% if data_monitor_pg | default(false) %}
,{"name": "pg-prod-server", "path": "/etc/homelab-postgresql/prod/server.crt"}
,{"name": "pg-prod-etcd", "path": "/etc/homelab-postgresql/prod/etcd.crt"}
,{"name": "pg-prod-ca", "path": "/etc/homelab-postgresql/prod/ca.crt"}
{% endif %}
{% if data_monitor_dev | default(false) %}
,{"name": "pg-dev-server", "path": "/etc/homelab-postgresql/dev/server.crt"}
,{"name": "pg-dev-ca", "path": "/etc/homelab-postgresql/dev/ca.crt"}
{% endif %}
]
}
@@ -0,0 +1,124 @@
#!/usr/bin/env python3
"""只读检查共享数据服务,原子发布无秘密 textfile 指标。"""
import argparse
import datetime
import json
import os
from pathlib import Path
import re
import subprocess
import tempfile
import time
def command(argv):
return subprocess.run(argv, check=True, capture_output=True, text=True, timeout=15).stdout.strip()
def certificate_expiry(path):
# 检查完整 PEM 链;任何一张缺失/损坏都不能被当作健康。
blocks = re.findall(r'-----BEGIN CERTIFICATE-----.*?-----END CERTIFICATE-----',
Path(path).read_text(), re.S)
if not blocks:
raise ValueError('empty certificate chain')
expiries = []
for block in blocks:
result = subprocess.run(['openssl', 'x509', '-noout', '-enddate'], input=block,
capture_output=True, text=True, check=True, timeout=5)
date = result.stdout.strip().removeprefix('notAfter=')
expiries.append(datetime.datetime.strptime(date, '%b %d %H:%M:%S %Y GMT')
.replace(tzinfo=datetime.timezone.utc).timestamp())
return min(expiries)
def latest_backup(data):
stanza = next(x for x in data if x['name'] == 'prod')
if stanza['status']['code'] != 0:
raise ValueError('repository status not ok')
return max((b['timestamp']['stop'] for b in stanza['backup']
if b['type'] == 'full' and b.get('error') is False), default=0)
def latest_snapshot(directory):
return max((p.stat().st_mtime for p in Path(directory).iterdir()
if re.fullmatch(r'\d{8}T\d{6}Z\.db', p.name) and p.is_file()), default=0)
def collect(config):
lines = []
def emit(name, value, **labels):
suffix = '{' + ','.join(k + '=' + json.dumps(v) for k, v in sorted(labels.items())) + '}' if labels else ''
lines.append(f'homelab_data_{name}{suffix} {float(value)}')
def check(name, fn, **labels):
try:
value = fn()
except (OSError, ValueError, KeyError, TypeError, StopIteration, subprocess.SubprocessError):
# 不输出底层命令内容;某项失败不阻断其他检查,也不保留旧成功值。
emit(name, 0, **labels)
emit('check_success', 0, check=name, **labels)
else:
emit(name, value, **labels)
emit('check_success', 1, check=name, **labels)
for cert in config['certificates']:
check('certificate_expiry_timestamp_seconds', lambda c=cert: certificate_expiry(c['path']),
certificate=cert['name'])
check('backup_timestamp_seconds', lambda: latest_snapshot(config['snapshot_dir']), backup='etcd')
if config.get('repository'):
check('backup_timestamp_seconds', lambda: latest_backup(json.loads(command([
'runuser', '-u', 'pgbackup', '--', 'pgbackrest',
'--config=/etc/homelab-pgbackrest/repository.conf', '--stanza=prod', '--output=json', 'info'
]))), backup='postgresql-prod')
if config.get('dev'):
def dev_health():
result = command(['runuser', '-u', 'pgdev', '--', '/usr/lib/postgresql/18/bin/psql',
'-X', '-w', '-At', '-v', 'ON_ERROR_STOP=1',
'-h', '/run/homelab-postgresql-dev', '-p', '5433',
'-U', 'pgdev', '-d', 'postgres', '-c',
"SELECT (NOT pg_is_in_recovery() AND current_setting('transaction_read_only')='off')::int"])
if result not in ('0', '1'):
raise ValueError('unexpected SQL response')
return int(result)
check('dev_sql_ready', dev_health)
for unit in config.get('renewal_units', []):
def renewal(u=unit):
props = dict(line.split('=', 1) for line in command([
'systemctl', 'show', u + '.service', '-p', 'Result', '-p', 'ExecMainExitTimestamp',
'-p', 'ExecMainStatus', '-p', 'LoadState']).splitlines())
if props.get('LoadState') != 'loaded' or props.get('Result') != 'success' or props.get('ExecMainStatus') != '0':
raise ValueError('renewal unsuccessful')
timestamp = command(['date', '-d', props['ExecMainExitTimestamp'], '+%s'])
if command(['systemctl', 'is-active', u + '.timer']) != 'active':
raise ValueError('renewal timer inactive')
return int(timestamp)
check('renewal_success_timestamp_seconds', renewal, unit=unit)
emit('collection_timestamp_seconds', time.time())
return '\n'.join(lines) + '\n'
def publish(path, content):
path = Path(path)
fd, temporary = tempfile.mkstemp(prefix='.' + path.name, dir=path.parent)
try:
with os.fdopen(fd, 'w') as stream:
stream.write(content)
stream.flush()
os.fchmod(stream.fileno(), 0o644)
os.replace(temporary, path)
finally:
if os.path.exists(temporary):
os.unlink(temporary)
def main():
parser = argparse.ArgumentParser()
parser.add_argument('--config', required=True)
parser.add_argument('--output', required=True)
args = parser.parse_args()
publish(args.output, collect(json.loads(Path(args.config).read_text())))
if __name__ == '__main__':
main()
@@ -0,0 +1,76 @@
import importlib.util
import json
from pathlib import Path
import subprocess
import tempfile
import unittest
from unittest.mock import patch
spec = importlib.util.spec_from_file_location('collector', Path(__file__).parents[1] / 'files/collect.py')
collector = importlib.util.module_from_spec(spec)
spec.loader.exec_module(collector)
class CollectorTests(unittest.TestCase):
def test_full_backup_ignores_failed_and_incremental(self):
data = [{'name': 'prod', 'status': {'code': 0}, 'backup': [
{'type': 'full', 'error': False, 'timestamp': {'stop': 100}},
{'type': 'full', 'error': True, 'timestamp': {'stop': 200}},
{'type': 'incr', 'error': False, 'timestamp': {'stop': 300}},
]}]
self.assertEqual(collector.latest_backup(data), 100)
data[0]['backup'] = []
self.assertEqual(collector.latest_backup(data), 0)
data[0]['status']['code'] = 2
with self.assertRaises(ValueError):
collector.latest_backup(data)
def test_partial_snapshot_not_counted(self):
with tempfile.TemporaryDirectory() as d:
Path(d, '20260927T040000Z.db.partial').touch()
self.assertEqual(collector.latest_snapshot(d), 0)
p = Path(d, '20260926T040000Z.db')
p.touch()
self.assertEqual(collector.latest_snapshot(d), p.stat().st_mtime)
def test_missing_certificate_does_not_hide_other_failures(self):
with tempfile.TemporaryDirectory() as d:
config = {'certificates': [{'name': 'missing', 'path': d + '/none'}],
'snapshot_dir': d, 'dev': True, 'repository': True}
with patch.object(collector, 'command', side_effect=subprocess.TimeoutExpired('probe', 15)):
output = collector.collect(config)
self.assertIn('certificate="missing"} 0.0', output)
self.assertIn('homelab_data_dev_sql_ready 0.0', output)
self.assertIn('backup="postgresql-prod"} 0.0', output)
self.assertIn('homelab_data_collection_timestamp_seconds', output)
def test_read_only_dev_fails_readiness(self):
with tempfile.TemporaryDirectory() as d:
with patch.object(collector, 'command', return_value='0'):
output = collector.collect({'certificates': [], 'snapshot_dir': d, 'dev': True})
self.assertIn('homelab_data_dev_sql_ready 0.0', output)
self.assertIn('check="dev_sql_ready"} 1.0', output)
def test_atomic_output_replaces_old_success(self):
with tempfile.TemporaryDirectory() as d:
p = Path(d, 'health.prom')
collector.publish(p, 'success 1\n')
collector.publish(p, 'success 0\n')
self.assertEqual(p.read_text(), 'success 0\n')
self.assertEqual(p.stat().st_mode & 0o777, 0o644)
self.assertEqual(list(Path(d).iterdir()), [p])
def test_actual_certificate_chain_earliest_expiry(self):
with tempfile.TemporaryDirectory() as d:
for days in (2, 30):
subprocess.run(['openssl', 'req', '-x509', '-newkey', 'ec', '-pkeyopt', 'ec_paramgen_curve:P-256',
'-nodes', '-subj', '/CN=test', '-days', str(days),
'-keyout', d + '/key', '-out', d + '/' + str(days)],
check=True, capture_output=True)
p = Path(d, 'chain')
p.write_text(Path(d, '30').read_text() + Path(d, '2').read_text())
self.assertEqual(collector.certificate_expiry(p), collector.certificate_expiry(Path(d, '2')))
if __name__ == '__main__':
unittest.main()
@@ -32,3 +32,5 @@ resources:
- rules/data-services.yaml - rules/data-services.yaml
- scrapes/openbao.yaml - scrapes/openbao.yaml
- rules/openbao.yaml - rules/openbao.yaml
- scrapes/shared-data-maintenance.yaml
- rules/shared-data-maintenance.yaml
@@ -0,0 +1,63 @@
apiVersion: monitoring.coreos.com/v1
kind: PrometheusRule
metadata:
name: shared-data-maintenance
namespace: monitoring
spec:
groups:
- name: shared-data-maintenance
rules:
- alert: SharedDataMonitoringUnavailable
expr: up{job="shared-data-maintenance"} == 0 or absent(up{job="shared-data-maintenance"})
for: 3m
labels: {severity: warning, service: shared-data}
annotations:
summary: 共享数据维护指标无法采集
- alert: SharedDataCollectorStale
expr: >-
(time() - homelab_data_collection_timestamp_seconds{job="shared-data-maintenance"} > 180)
or (up{job="shared-data-maintenance"} == 1
unless on(instance) homelab_data_collection_timestamp_seconds{job="shared-data-maintenance"})
or node_textfile_scrape_error{job="shared-data-maintenance"} == 1
for: 3m
labels: {severity: warning, service: shared-data}
annotations:
summary: 共享数据本地采集结果缺失或停止更新
- alert: SharedDataCheckFailed
expr: homelab_data_check_success{job="shared-data-maintenance"} == 0
for: 3m
labels: {severity: warning, service: shared-data}
annotations:
summary: 共享数据维护检查失败,请核对对应检查项
- alert: SharedDataBackupStale
expr: time() - homelab_data_backup_timestamp_seconds{job="shared-data-maintenance"} > 36 * 3600
for: 5m
labels: {severity: warning, service: shared-data}
annotations:
summary: 共享数据备份已超过 36 小时或没有成功备份
- alert: SharedDataCertificateExpiringSoon
expr: >-
homelab_data_certificate_expiry_timestamp_seconds{job="shared-data-maintenance"} - time() < 10 * 86400
and homelab_data_certificate_expiry_timestamp_seconds{job="shared-data-maintenance"} - time() >= 3 * 86400
for: 5m
labels: {severity: warning, service: shared-data}
annotations:
summary: 共享数据证书剩余不足 10 天,请检查自动续签
- alert: SharedDataCertificateExpiryCritical
expr: homelab_data_certificate_expiry_timestamp_seconds{job="shared-data-maintenance"} - time() < 3 * 86400
for: 5m
labels: {severity: critical, service: shared-data}
annotations:
summary: 共享数据证书不足 3 天、已过期或无法读取
- alert: SharedDataRenewalStale
expr: time() - homelab_data_renewal_success_timestamp_seconds{job="shared-data-maintenance"} > 36 * 3600
for: 5m
labels: {severity: warning, service: shared-data}
annotations:
summary: 共享数据续签检查失败、停用或超过 36 小时未成功
- alert: SharedPostgresDevUnavailable
expr: homelab_data_dev_sql_ready{job="shared-data-maintenance"} == 0
for: 3m
labels: {severity: warning, service: shared-postgresql}
annotations:
summary: 共享 PG 开发实例本地 SQL 不可用或处于只读状态
@@ -0,0 +1,17 @@
apiVersion: operator.victoriametrics.com/v1beta1
kind: VMStaticScrape
metadata:
name: shared-data-maintenance
namespace: monitoring
spec:
jobName: shared-data-maintenance
targetEndpoints:
- targets: ['192.168.10.127:9109']
labels: {member: laptop}
interval: 30s
- targets: ['10.60.0.20:9109']
labels: {member: etcd-pve1}
interval: 30s
- targets: ['10.60.0.21:9109']
labels: {member: etcd-pve2}
interval: 30s
@@ -10,7 +10,7 @@ import sys
import yaml import yaml
groups = [] groups = []
for name in ("kubernetes-health", "host-health", "monitoring-delivery", "platform-controllers", "data-services", "openbao"): for name in ("kubernetes-health", "host-health", "monitoring-delivery", "platform-controllers", "data-services", "openbao", "shared-data-maintenance"):
path = pathlib.Path(sys.argv[1]) / f"{name}.yaml" path = pathlib.Path(sys.argv[1]) / f"{name}.yaml"
groups.extend(yaml.safe_load(path.read_text())["spec"]["groups"]) groups.extend(yaml.safe_load(path.read_text())["spec"]["groups"])
pathlib.Path(sys.argv[2]).write_text(yaml.safe_dump({"groups": groups}, allow_unicode=True)) pathlib.Path(sys.argv[2]).write_text(yaml.safe_dump({"groups": groups}, allow_unicode=True))
@@ -0,0 +1,168 @@
rule_files:
- rules.yaml
evaluation_interval: 1m
tests:
- name: 过期证书持续 critical
interval: 1m
input_series:
- series: homelab_data_certificate_expiry_timestamp_seconds{job="shared-data-maintenance",instance="host:9109",certificate="etcd-peer"}
values: '0x15'
alert_rule_test:
- eval_time: 10m
alertname: SharedDataCertificateExpiryCritical
exp_alerts:
- exp_labels:
job: shared-data-maintenance
instance: host:9109
certificate: etcd-peer
severity: critical
service: shared-data
exp_annotations:
summary: 共享数据证书不足 3 天、已过期或无法读取
- name: critical 证书不再重复 warning
interval: 1m
input_series:
- series: homelab_data_certificate_expiry_timestamp_seconds{job="shared-data-maintenance",instance="host:9109",certificate="etcd-peer"}
values: '0x15'
alert_rule_test:
- eval_time: 10m
alertname: SharedDataCertificateExpiringSoon
exp_alerts: []
- name: 正常证书不误报
interval: 1m
input_series:
- series: homelab_data_certificate_expiry_timestamp_seconds{job="shared-data-maintenance",instance="host:9109",certificate="etcd-peer"}
values: 5184000x15
alert_rule_test:
- eval_time: 10m
alertname: SharedDataCertificateExpiringSoon
exp_alerts: []
- name: 七天内证书预警
interval: 1m
input_series:
- series: homelab_data_certificate_expiry_timestamp_seconds{job="shared-data-maintenance",instance="host:9109",certificate="etcd-peer"}
values: 604800x15
alert_rule_test:
- eval_time: 10m
alertname: SharedDataCertificateExpiringSoon
exp_alerts:
- exp_labels:
job: shared-data-maintenance
instance: host:9109
certificate: etcd-peer
severity: warning
service: shared-data
exp_annotations:
summary: 共享数据证书剩余不足 10 天,请检查自动续签
- name: 备份超过36小时
interval: 1m
input_series:
- series: homelab_data_backup_timestamp_seconds{job="shared-data-maintenance",instance="host:9109",backup="etcd"}
values: -130000x15
alert_rule_test:
- eval_time: 10m
alertname: SharedDataBackupStale
exp_alerts:
- exp_labels:
job: shared-data-maintenance
instance: host:9109
backup: etcd
severity: warning
service: shared-data
exp_annotations:
summary: 共享数据备份已超过 36 小时或没有成功备份
- name: 新备份没有告警
interval: 1m
input_series:
- series: homelab_data_backup_timestamp_seconds{job="shared-data-maintenance",instance="host:9109",backup="etcd"}
values: 0+60x15
alert_rule_test:
- eval_time: 10m
alertname: SharedDataBackupStale
exp_alerts: []
- name: exporter 存活但没有采集文件
interval: 1m
input_series:
- series: up{job="shared-data-maintenance",instance="host:9109"}
values: 1x15
alert_rule_test:
- eval_time: 10m
alertname: SharedDataCollectorStale
exp_alerts:
- exp_labels:
job: shared-data-maintenance
instance: host:9109
severity: warning
service: shared-data
exp_annotations: &id001
summary: 共享数据本地采集结果缺失或停止更新
- name: 采集心跳过期
interval: 1m
input_series:
- series: homelab_data_collection_timestamp_seconds{job="shared-data-maintenance",instance="host:9109"}
values: '0x15'
alert_rule_test:
- eval_time: 10m
alertname: SharedDataCollectorStale
exp_alerts:
- exp_labels:
job: shared-data-maintenance
instance: host:9109
severity: warning
service: shared-data
exp_annotations: *id001
- name: SQL 拒绝和只读触发 dev 告警
interval: 1m
input_series:
- series: homelab_data_dev_sql_ready{job="shared-data-maintenance",instance="host:9109"}
values: '0x15'
alert_rule_test:
- eval_time: 10m
alertname: SharedPostgresDevUnavailable
exp_alerts:
- exp_labels:
job: shared-data-maintenance
instance: host:9109
severity: warning
service: shared-postgresql
exp_annotations:
summary: 共享 PG 开发实例本地 SQL 不可用或处于只读状态
- name: dev 恢复后解除
interval: 1m
input_series:
- series: homelab_data_dev_sql_ready{job="shared-data-maintenance",instance="host:9109"}
values: 0x4 1x10
alert_rule_test:
- eval_time: 10m
alertname: SharedPostgresDevUnavailable
exp_alerts: []
- name: 续签检查超过36小时
interval: 1m
input_series:
- series: homelab_data_renewal_success_timestamp_seconds{job="shared-data-maintenance",instance="host:9109",unit="homelab-etcd-renew"}
values: -130000x15
alert_rule_test:
- eval_time: 10m
alertname: SharedDataRenewalStale
exp_alerts:
- exp_labels:
job: shared-data-maintenance
instance: host:9109
unit: homelab-etcd-renew
severity: warning
service: shared-data
exp_annotations:
summary: 共享数据续签检查失败、停用或超过 36 小时未成功
- name: 整个 scrape 消失仍报警
interval: 1m
input_series: []
alert_rule_test:
- eval_time: 10m
alertname: SharedDataMonitoringUnavailable
exp_alerts:
- exp_labels:
severity: warning
service: shared-data
job: shared-data-maintenance
exp_annotations:
summary: 共享数据维护指标无法采集