新增共享 PG 的 SQL 级指标与 dev 告警分流

- 数据库主机上按实例运行 prometheus-community/postgres_exporter 0.20.1，以实例 OS 用户走本地 socket，经 peer + ident 映射到仅有 pg_monitor 的 homelab_monitor 角色；无密码、不经 Bao。版本与官方 sha256sums.txt 同源。 - prod/dev 启用 pg_stat_statements；monitoring.yml 仅在未生效时 reload/重启（prod 走 Patroni pending restart），重复执行 changed=0。 - 告警以上游 postgres_mixin 为底改写为 PrometheusRule，阈值对齐 Patroni failover lag 与 max_slot_wal_keep_size；抓取目标带 env 标签。 - Alertmanager 新增 env="dev" 路由到独立 dev 频道，dev SQL 检查告警补 env。 Co-Authored-By: Claude Opus 5.5 <[email protected]>
2026-10-01 16:20:15 +00:00
parent cd9d461950
commit 0cd4a67e12
19 changed files with 551 additions and 5 deletions
@@ -10,7 +10,7 @@ import sys
 import yaml

 groups = []
-for name in ("kubernetes-health", "host-health", "monitoring-delivery", "platform-controllers", "data-services", "openbao", "shared-data-maintenance"):
+for name in ("kubernetes-health", "host-health", "monitoring-delivery", "platform-controllers", "data-services", "openbao", "shared-data-maintenance", "shared-postgresql-sql"):
    path = pathlib.Path(sys.argv[1]) / f"{name}.yaml"
    groups.extend(yaml.safe_load(path.read_text())["spec"]["groups"])
 pathlib.Path(sys.argv[2]).write_text(yaml.safe_dump({"groups": groups}, allow_unicode=True))
@@ -125,6 +125,7 @@ tests:
        instance: host:9109
        severity: warning
        service: shared-postgresql
+        env: dev
      exp_annotations:
        summary: 共享 PG 开发实例本地 SQL 不可用或处于只读状态
 - name: dev 恢复后解除
@@ -0,0 +1,139 @@
+rule_files:
+- rules.yaml
+evaluation_interval: 1m
+tests:
+- name: dev SQL 不通时告警带 env=dev 以便分流
+  interval: 1m
+  input_series:
+  - series: pg_up{job="shared-postgresql-sql",instance="h:9188",cluster="homelab-pg-dev",member="pg-laptop",env="dev"}
+    values: '0x5'
+  alert_rule_test:
+  - eval_time: 5m
+    alertname: SharedPostgresSqlUnreachable
+    exp_alerts:
+    - exp_labels: {job: shared-postgresql-sql, instance: 'h:9188', cluster: homelab-pg-dev, member: pg-laptop, env: dev, severity: warning, service: shared-postgresql}
+      exp_annotations:
+        summary: 共享 PG dev pg-laptop 无法以监控角色执行 SQL
+- name: 回放落后持续超过 1 MiB 告警
+  interval: 1m
+  input_series:
+  - series: pg_stat_replication_pg_wal_lsn_diff{job="shared-postgresql-sql",instance="p:9187",member="pg-pve1",env="prod",application_name="pg-laptop"}
+    values: '2097152x10'
+  alert_rule_test:
+  - eval_time: 8m
+    alertname: SharedPostgresReplicationLagging
+    exp_alerts:
+    - exp_labels: {job: shared-postgresql-sql, instance: 'p:9187', member: pg-pve1, env: prod, application_name: pg-laptop, severity: warning, service: shared-postgresql}
+      exp_annotations:
+        summary: 共享 PG prod 副本 pg-laptop 回放落后超过 1 MiB，已不满足自动切换条件
+- name: 短暂落后不告警
+  interval: 1m
+  input_series:
+  - series: pg_stat_replication_pg_wal_lsn_diff{job="shared-postgresql-sql",instance="p:9187",env="prod",application_name="pg-laptop"}
+    values: '2097152x2 0x8'
+  alert_rule_test:
+  - eval_time: 8m
+    alertname: SharedPostgresReplicationLagging
+    exp_alerts: []
+- name: 复制槽积压只在主库告警
+  interval: 1m
+  input_series:
+  - series: pg_replication_slots_pg_wal_lsn_diff{job="shared-postgresql-sql",instance="p:9187",env="prod",slot_name="pg_laptop"}
+    values: '2147483648x15'
+  - series: pg_replication_is_replica{job="shared-postgresql-sql",instance="p:9187",env="prod"}
+    values: '0x15'
+  - series: pg_replication_slots_pg_wal_lsn_diff{job="shared-postgresql-sql",instance="r:9187",env="prod",slot_name="pg_pve1"}
+    values: '2147483648x15'
+  - series: pg_replication_is_replica{job="shared-postgresql-sql",instance="r:9187",env="prod"}
+    values: '1x15'
+  alert_rule_test:
+  - eval_time: 12m
+    alertname: SharedPostgresReplicationSlotRetention
+    exp_alerts:
+    - exp_labels: {job: shared-postgresql-sql, instance: 'p:9187', env: prod, slot_name: pg_laptop, severity: warning, service: shared-postgresql}
+      exp_annotations:
+        summary: 共享 PG prod 复制槽 pg_laptop 保留 WAL 超过 1 GiB
+- name: 回卷 critical 时不重复 warning
+  interval: 1m
+  input_series:
+  - series: pg_database_wraparound_age_datfrozenxid_seconds{job="shared-postgresql-sql",instance="p:9187",member="pg-pve1",env="prod",datname="app"}
+    values: '1700000000x15'
+  - series: pg_database_wraparound_age_datminmxid_seconds{job="shared-postgresql-sql",instance="p:9187",member="pg-pve1",env="prod",datname="app"}
+    values: '1200000000x15'
+  alert_rule_test:
+  - eval_time: 12m
+    alertname: SharedPostgresWraparoundRisk
+    exp_alerts: []
+  - eval_time: 12m
+    alertname: SharedPostgresWraparoundImminent
+    exp_alerts:
+    - exp_labels: {job: shared-postgresql-sql, instance: 'p:9187', member: pg-pve1, env: prod, severity: critical, service: shared-postgresql}
+      exp_annotations:
+        summary: 共享 PG prod pg-pve1 XID/MXID 接近回卷停写，需立即 VACUUM FREEZE
+- name: 回卷 warning
+  interval: 1m
+  input_series:
+  - series: pg_database_wraparound_age_datfrozenxid_seconds{job="shared-postgresql-sql",instance="p:9187",member="pg-pve1",env="prod",datname="app"}
+    values: '1200000000x15'
+  alert_rule_test:
+  - eval_time: 12m
+    alertname: SharedPostgresWraparoundRisk
+    exp_alerts:
+    - exp_labels: {job: shared-postgresql-sql, instance: 'p:9187', member: pg-pve1, env: prod, severity: warning, service: shared-postgresql}
+      exp_annotations:
+        summary: 共享 PG prod pg-pve1 XID/MXID 年龄超过 10 亿
+- name: 只计客户端连接
+  interval: 1m
+  input_series:
+  - series: pg_stat_activity_count{job="shared-postgresql-sql",instance="p:9187",member="pg-pve1",env="prod",datname="app",state="active",usename="a",application_name="x",backend_type="client backend",wait_event_type="",wait_event=""}
+    values: '50x10'
+  - series: pg_stat_activity_count{job="shared-postgresql-sql",instance="p:9187",member="pg-pve1",env="prod",datname="app",state="idle",usename="b",application_name="y",backend_type="client backend",wait_event_type="",wait_event=""}
+    values: '20x10'
+  - series: pg_stat_activity_count{job="shared-postgresql-sql",instance="p:9187",member="pg-pve1",env="prod",datname="",state="",usename="",application_name="",backend_type="autovacuum worker",wait_event_type="",wait_event=""}
+    values: '30x10'
+  - series: pg_settings_max_connections{job="shared-postgresql-sql",instance="p:9187",member="pg-pve1",env="prod"}
+    values: '80x10'
+  alert_rule_test:
+  - eval_time: 8m
+    alertname: SharedPostgresConnectionsHigh
+    exp_alerts:
+    - exp_labels: {job: shared-postgresql-sql, instance: 'p:9187', member: pg-pve1, env: prod, severity: warning, service: shared-postgresql}
+      exp_annotations:
+        summary: 共享 PG prod pg-pve1 客户端连接超过 max_connections 的 80%
+- name: 后台进程不计入连接使用率
+  interval: 1m
+  input_series:
+  - series: pg_stat_activity_count{job="shared-postgresql-sql",instance="p:9187",env="prod",datname="app",state="active",usename="a",application_name="x",backend_type="client backend",wait_event_type="",wait_event=""}
+    values: '40x10'
+  - series: pg_stat_activity_count{job="shared-postgresql-sql",instance="p:9187",env="prod",datname="",state="",usename="",application_name="",backend_type="autovacuum worker",wait_event_type="",wait_event=""}
+    values: '40x10'
+  - series: pg_settings_max_connections{job="shared-postgresql-sql",instance="p:9187",env="prod"}
+    values: '80x10'
+  alert_rule_test:
+  - eval_time: 8m
+    alertname: SharedPostgresConnectionsHigh
+    exp_alerts: []
+- name: 死锁计数增长告警
+  interval: 1m
+  input_series:
+  - series: pg_stat_database_deadlocks{job="shared-postgresql-sql",instance="p:9187",member="pg-pve1",env="prod",datname="app"}
+    values: '0 0 0 1 1 1'
+  alert_rule_test:
+  - eval_time: 5m
+    alertname: SharedPostgresDeadlocks
+    exp_alerts:
+    - exp_labels: {job: shared-postgresql-sql, instance: 'p:9187', member: pg-pve1, env: prod, severity: warning, service: shared-postgresql}
+      exp_annotations:
+        summary: 共享 PG prod pg-pve1 15 分钟内出现死锁
+- name: 空闲事务超过 10 分钟
+  interval: 1m
+  input_series:
+  - series: pg_stat_activity_max_tx_duration{job="shared-postgresql-sql",instance="p:9187",member="pg-pve1",env="prod",datname="app",state="idle in transaction",usename="a",application_name="x",backend_type="client backend",wait_event_type="Client",wait_event="ClientRead"}
+    values: '700x5'
+  alert_rule_test:
+  - eval_time: 3m
+    alertname: SharedPostgresIdleInTransaction
+    exp_alerts:
+    - exp_labels: {job: shared-postgresql-sql, instance: 'p:9187', member: pg-pve1, env: prod, severity: warning, service: shared-postgresql}
+      exp_annotations:
+        summary: 共享 PG prod pg-pve1 有事务空闲未提交超过 10 分钟