新增共享 PG 的 SQL 级指标与 dev 告警分流

- 数据库主机上按实例运行 prometheus-community/postgres_exporter 0.20.1，以实例 OS 用户走本地 socket，经 peer + ident 映射到仅有 pg_monitor 的 homelab_monitor 角色；无密码、不经 Bao。版本与官方 sha256sums.txt 同源。 - prod/dev 启用 pg_stat_statements；monitoring.yml 仅在未生效时 reload/重启（prod 走 Patroni pending restart），重复执行 changed=0。 - 告警以上游 postgres_mixin 为底改写为 PrometheusRule，阈值对齐 Patroni failover lag 与 max_slot_wal_keep_size；抓取目标带 env 标签。 - Alertmanager 新增 env="dev" 路由到独立 dev 频道，dev SQL 检查告警补 env。 Co-Authored-By: Claude Opus 5.5 <[email protected]>
2026-10-01 16:20:15 +00:00
parent cd9d461950
commit 0cd4a67e12
19 changed files with 551 additions and 5 deletions
@@ -25,6 +25,8 @@ resources:
  - scrapes/shared-etcd.yaml
  - rules/shared-postgresql.yaml
  - scrapes/shared-postgresql.yaml
+  - rules/shared-postgresql-sql.yaml
+  - scrapes/shared-postgresql-sql.yaml
  - scrapes/platform-controllers.yaml
  - rules/platform-controllers.yaml
  - scrapes/cnpg.yaml
@@ -58,6 +58,7 @@ spec:
        - alert: SharedPostgresDevUnavailable
          expr: homelab_data_dev_sql_ready{job="shared-data-maintenance"} == 0
          for: 3m
-          labels: {severity: warning, service: shared-postgresql}
+          # env 供 Alertmanager 把 dev 分流到独立 channel。
+          labels: {severity: warning, service: shared-postgresql, env: dev}
          annotations:
            summary: 共享 PG 开发实例本地 SQL 不可用或处于只读状态
@@ -0,0 +1,109 @@
+# 以 postgres_exporter 上游 postgres_mixin 为底按本环境改写：实例停止、多主、角色变化已由
+# Patroni 规则（shared-postgresql.yaml）覆盖；QPS/缓存命中率/回滚数不是故障信号，未采用；
+# 上游 PostgresHasReplicationSlotUsed 会被 Patroni 在 standby 保留的非活跃槽持续触发，未采用。
+# 每条告警都带抓取目标的 env 标签，Alertmanager 按它把 dev 分流到独立 channel。
+apiVersion: monitoring.coreos.com/v1
+kind: PrometheusRule
+metadata:
+  name: shared-postgresql-sql
+  namespace: monitoring
+spec:
+  groups:
+    - name: shared-postgresql-sql
+      rules:
+        - alert: SharedPostgresSqlExporterDown
+          expr: up{job="shared-postgresql-sql"} == 0
+          for: 3m
+          labels: {severity: warning, service: shared-postgresql}
+          annotations:
+            summary: '共享 PG {{ $labels.env }} {{ $labels.member }} 的 SQL exporter 无法采集'
+        # 进程停止已有 Patroni critical；这里覆盖进程在但 SQL 不通（HBA、连接耗尽、dev 实例）。
+        - alert: SharedPostgresSqlUnreachable
+          expr: pg_up{job="shared-postgresql-sql"} == 0
+          for: 2m
+          labels: {severity: warning, service: shared-postgresql}
+          annotations:
+            summary: '共享 PG {{ $labels.env }} {{ $labels.member }} 无法以监控角色执行 SQL'
+        - alert: SharedPostgresSqlCollectorFailing
+          expr: pg_scrape_collector_success{job="shared-postgresql-sql"} == 0 or pg_exporter_last_scrape_error{job="shared-postgresql-sql"} == 1
+          for: 10m
+          labels: {severity: warning, service: shared-postgresql}
+          annotations:
+            summary: '共享 PG {{ $labels.env }} {{ $labels.member }} 部分 SQL 指标采集失败 {{ $labels.collector }}'
+        # 与 Patroni maximum_lag_on_failover（1 MiB）一致：持续超过即该 standby 不再是合格的切换候选。
+        - alert: SharedPostgresReplicationLagging
+          expr: pg_stat_replication_pg_wal_lsn_diff{job="shared-postgresql-sql"} > 1048576
+          for: 5m
+          labels: {severity: warning, service: shared-postgresql}
+          annotations:
+            summary: '共享 PG {{ $labels.env }} 副本 {{ $labels.application_name }} 回放落后超过 1 MiB，已不满足自动切换条件'
+        # 只看可写成员：standby 上 Patroni 维护的槽副本不反映真实保留压力。
+        # 1 GiB 是 max_slot_wal_keep_size（2GB）的一半，留出处理时间。
+        - alert: SharedPostgresReplicationSlotRetention
+          expr: >-
+            pg_replication_slots_pg_wal_lsn_diff{job="shared-postgresql-sql"} > 1073741824
+            and on(instance) pg_replication_is_replica{job="shared-postgresql-sql"} == 0
+          for: 10m
+          labels: {severity: warning, service: shared-postgresql}
+          annotations:
+            summary: '共享 PG {{ $labels.env }} 复制槽 {{ $labels.slot_name }} 保留 WAL 超过 1 GiB'
+        - alert: SharedPostgresWalArchiveFailing
+          expr: increase(pg_stat_archiver_failed_count{job="shared-postgresql-sql"}[15m]) > 0
+          labels: {severity: warning, service: shared-postgresql}
+          annotations:
+            summary: '共享 PG {{ $labels.env }} {{ $labels.member }} WAL 归档失败，PITR 链可能中断'
+        # XID/MXID 上限约 2^31；超过一半说明 freeze 跟不上，超过 75% 已接近强制停写；critical 时不重复 warning。
+        - alert: SharedPostgresWraparoundRisk
+          expr: >-
+            (max without(datname) (pg_database_wraparound_age_datfrozenxid_seconds{job="shared-postgresql-sql"}) > 1e9
+            or max without(datname) (pg_database_wraparound_age_datminmxid_seconds{job="shared-postgresql-sql"}) > 1e9)
+            unless (max without(datname) (pg_database_wraparound_age_datfrozenxid_seconds{job="shared-postgresql-sql"}) > 1.6e9
+            or max without(datname) (pg_database_wraparound_age_datminmxid_seconds{job="shared-postgresql-sql"}) > 1.6e9)
+          for: 10m
+          labels: {severity: warning, service: shared-postgresql}
+          annotations:
+            summary: '共享 PG {{ $labels.env }} {{ $labels.member }} XID/MXID 年龄超过 10 亿'
+        - alert: SharedPostgresWraparoundImminent
+          expr: >-
+            max without(datname) (pg_database_wraparound_age_datfrozenxid_seconds{job="shared-postgresql-sql"}) > 1.6e9
+            or max without(datname) (pg_database_wraparound_age_datminmxid_seconds{job="shared-postgresql-sql"}) > 1.6e9
+          for: 5m
+          labels: {severity: critical, service: shared-postgresql}
+          annotations:
+            summary: '共享 PG {{ $labels.env }} {{ $labels.member }} XID/MXID 接近回卷停写，需立即 VACUUM FREEZE'
+        - alert: SharedPostgresConnectionsHigh
+          expr: >-
+            sum without(datname, state, usename, application_name, backend_type, wait_event_type, wait_event)
+              (pg_stat_activity_count{job="shared-postgresql-sql", backend_type="client backend"})
+            / pg_settings_max_connections{job="shared-postgresql-sql"} > 0.8
+          for: 5m
+          labels: {severity: warning, service: shared-postgresql}
+          annotations:
+            summary: '共享 PG {{ $labels.env }} {{ $labels.member }} 客户端连接超过 max_connections 的 80%'
+        # 长事务阻止 vacuum 回收并持有锁；idle in transaction 通常是应用忘记提交。
+        - alert: SharedPostgresIdleInTransaction
+          expr: >-
+            max without(datname, state, usename, application_name, backend_type, wait_event_type, wait_event)
+              (pg_stat_activity_max_tx_duration{job="shared-postgresql-sql", state=~"idle in transaction.*"}) > 600
+          for: 1m
+          labels: {severity: warning, service: shared-postgresql}
+          annotations:
+            summary: '共享 PG {{ $labels.env }} {{ $labels.member }} 有事务空闲未提交超过 10 分钟'
+        - alert: SharedPostgresLongTransaction
+          expr: pg_long_running_transactions_oldest_timestamp_seconds{job="shared-postgresql-sql"} > 3600
+          for: 1m
+          labels: {severity: warning, service: shared-postgresql}
+          annotations:
+            summary: '共享 PG {{ $labels.env }} {{ $labels.member }} 最老事务已运行超过 1 小时'
+        - alert: SharedPostgresDeadlocks
+          expr: sum without(datname) (increase(pg_stat_database_deadlocks{job="shared-postgresql-sql"}[15m])) > 0
+          labels: {severity: warning, service: shared-postgresql}
+          annotations:
+            summary: '共享 PG {{ $labels.env }} {{ $labels.member }} 15 分钟内出现死锁'
+        # 每日 pgBackRest full 会请求一次 checkpoint；每小时多于 4 次通常意味着 max_wal_size 偏小。
+        - alert: SharedPostgresCheckpointsRequested
+          expr: increase(pg_stat_checkpointer_num_requested_total{job="shared-postgresql-sql"}[1h]) > 4
+          for: 15m
+          labels: {severity: warning, service: shared-postgresql}
+          annotations:
+            summary: '共享 PG {{ $labels.env }} {{ $labels.member }} 频繁请求 checkpoint，max_wal_size 可能偏小'
@@ -0,0 +1,18 @@
+# postgres_exporter 跑在数据库主机上，按实例一个端口；env 标签供 Alertmanager 分流 dev。
+apiVersion: operator.victoriametrics.com/v1beta1
+kind: VMStaticScrape
+metadata:
+  name: shared-postgresql-sql
+  namespace: monitoring
+spec:
+  jobName: shared-postgresql-sql
+  targetEndpoints:
+    - targets: ['192.168.10.127:9187']
+      labels: {cluster: homelab-pg-prod, member: pg-laptop, env: prod}
+      interval: 30s
+    - targets: ['10.60.0.20:9187']
+      labels: {cluster: homelab-pg-prod, member: pg-pve1, env: prod}
+      interval: 30s
+    - targets: ['192.168.10.127:9188']
+      labels: {cluster: homelab-pg-dev, member: pg-laptop, env: dev}
+      interval: 30s
@@ -10,7 +10,7 @@ import sys
 import yaml

 groups = []
-for name in ("kubernetes-health", "host-health", "monitoring-delivery", "platform-controllers", "data-services", "openbao", "shared-data-maintenance"):
+for name in ("kubernetes-health", "host-health", "monitoring-delivery", "platform-controllers", "data-services", "openbao", "shared-data-maintenance", "shared-postgresql-sql"):
    path = pathlib.Path(sys.argv[1]) / f"{name}.yaml"
    groups.extend(yaml.safe_load(path.read_text())["spec"]["groups"])
 pathlib.Path(sys.argv[2]).write_text(yaml.safe_dump({"groups": groups}, allow_unicode=True))
@@ -125,6 +125,7 @@ tests:
        instance: host:9109
        severity: warning
        service: shared-postgresql
+        env: dev
      exp_annotations:
        summary: 共享 PG 开发实例本地 SQL 不可用或处于只读状态
 - name: dev 恢复后解除
@@ -0,0 +1,139 @@
+rule_files:
+- rules.yaml
+evaluation_interval: 1m
+tests:
+- name: dev SQL 不通时告警带 env=dev 以便分流
+  interval: 1m
+  input_series:
+  - series: pg_up{job="shared-postgresql-sql",instance="h:9188",cluster="homelab-pg-dev",member="pg-laptop",env="dev"}
+    values: '0x5'
+  alert_rule_test:
+  - eval_time: 5m
+    alertname: SharedPostgresSqlUnreachable
+    exp_alerts:
+    - exp_labels: {job: shared-postgresql-sql, instance: 'h:9188', cluster: homelab-pg-dev, member: pg-laptop, env: dev, severity: warning, service: shared-postgresql}
+      exp_annotations:
+        summary: 共享 PG dev pg-laptop 无法以监控角色执行 SQL
+- name: 回放落后持续超过 1 MiB 告警
+  interval: 1m
+  input_series:
+  - series: pg_stat_replication_pg_wal_lsn_diff{job="shared-postgresql-sql",instance="p:9187",member="pg-pve1",env="prod",application_name="pg-laptop"}
+    values: '2097152x10'
+  alert_rule_test:
+  - eval_time: 8m
+    alertname: SharedPostgresReplicationLagging
+    exp_alerts:
+    - exp_labels: {job: shared-postgresql-sql, instance: 'p:9187', member: pg-pve1, env: prod, application_name: pg-laptop, severity: warning, service: shared-postgresql}
+      exp_annotations:
+        summary: 共享 PG prod 副本 pg-laptop 回放落后超过 1 MiB，已不满足自动切换条件
+- name: 短暂落后不告警
+  interval: 1m
+  input_series:
+  - series: pg_stat_replication_pg_wal_lsn_diff{job="shared-postgresql-sql",instance="p:9187",env="prod",application_name="pg-laptop"}
+    values: '2097152x2 0x8'
+  alert_rule_test:
+  - eval_time: 8m
+    alertname: SharedPostgresReplicationLagging
+    exp_alerts: []
+- name: 复制槽积压只在主库告警
+  interval: 1m
+  input_series:
+  - series: pg_replication_slots_pg_wal_lsn_diff{job="shared-postgresql-sql",instance="p:9187",env="prod",slot_name="pg_laptop"}
+    values: '2147483648x15'
+  - series: pg_replication_is_replica{job="shared-postgresql-sql",instance="p:9187",env="prod"}
+    values: '0x15'
+  - series: pg_replication_slots_pg_wal_lsn_diff{job="shared-postgresql-sql",instance="r:9187",env="prod",slot_name="pg_pve1"}
+    values: '2147483648x15'
+  - series: pg_replication_is_replica{job="shared-postgresql-sql",instance="r:9187",env="prod"}
+    values: '1x15'
+  alert_rule_test:
+  - eval_time: 12m
+    alertname: SharedPostgresReplicationSlotRetention
+    exp_alerts:
+    - exp_labels: {job: shared-postgresql-sql, instance: 'p:9187', env: prod, slot_name: pg_laptop, severity: warning, service: shared-postgresql}
+      exp_annotations:
+        summary: 共享 PG prod 复制槽 pg_laptop 保留 WAL 超过 1 GiB
+- name: 回卷 critical 时不重复 warning
+  interval: 1m
+  input_series:
+  - series: pg_database_wraparound_age_datfrozenxid_seconds{job="shared-postgresql-sql",instance="p:9187",member="pg-pve1",env="prod",datname="app"}
+    values: '1700000000x15'
+  - series: pg_database_wraparound_age_datminmxid_seconds{job="shared-postgresql-sql",instance="p:9187",member="pg-pve1",env="prod",datname="app"}
+    values: '1200000000x15'
+  alert_rule_test:
+  - eval_time: 12m
+    alertname: SharedPostgresWraparoundRisk
+    exp_alerts: []
+  - eval_time: 12m
+    alertname: SharedPostgresWraparoundImminent
+    exp_alerts:
+    - exp_labels: {job: shared-postgresql-sql, instance: 'p:9187', member: pg-pve1, env: prod, severity: critical, service: shared-postgresql}
+      exp_annotations:
+        summary: 共享 PG prod pg-pve1 XID/MXID 接近回卷停写，需立即 VACUUM FREEZE
+- name: 回卷 warning
+  interval: 1m
+  input_series:
+  - series: pg_database_wraparound_age_datfrozenxid_seconds{job="shared-postgresql-sql",instance="p:9187",member="pg-pve1",env="prod",datname="app"}
+    values: '1200000000x15'
+  alert_rule_test:
+  - eval_time: 12m
+    alertname: SharedPostgresWraparoundRisk
+    exp_alerts:
+    - exp_labels: {job: shared-postgresql-sql, instance: 'p:9187', member: pg-pve1, env: prod, severity: warning, service: shared-postgresql}
+      exp_annotations:
+        summary: 共享 PG prod pg-pve1 XID/MXID 年龄超过 10 亿
+- name: 只计客户端连接
+  interval: 1m
+  input_series:
+  - series: pg_stat_activity_count{job="shared-postgresql-sql",instance="p:9187",member="pg-pve1",env="prod",datname="app",state="active",usename="a",application_name="x",backend_type="client backend",wait_event_type="",wait_event=""}
+    values: '50x10'
+  - series: pg_stat_activity_count{job="shared-postgresql-sql",instance="p:9187",member="pg-pve1",env="prod",datname="app",state="idle",usename="b",application_name="y",backend_type="client backend",wait_event_type="",wait_event=""}
+    values: '20x10'
+  - series: pg_stat_activity_count{job="shared-postgresql-sql",instance="p:9187",member="pg-pve1",env="prod",datname="",state="",usename="",application_name="",backend_type="autovacuum worker",wait_event_type="",wait_event=""}
+    values: '30x10'
+  - series: pg_settings_max_connections{job="shared-postgresql-sql",instance="p:9187",member="pg-pve1",env="prod"}
+    values: '80x10'
+  alert_rule_test:
+  - eval_time: 8m
+    alertname: SharedPostgresConnectionsHigh
+    exp_alerts:
+    - exp_labels: {job: shared-postgresql-sql, instance: 'p:9187', member: pg-pve1, env: prod, severity: warning, service: shared-postgresql}
+      exp_annotations:
+        summary: 共享 PG prod pg-pve1 客户端连接超过 max_connections 的 80%
+- name: 后台进程不计入连接使用率
+  interval: 1m
+  input_series:
+  - series: pg_stat_activity_count{job="shared-postgresql-sql",instance="p:9187",env="prod",datname="app",state="active",usename="a",application_name="x",backend_type="client backend",wait_event_type="",wait_event=""}
+    values: '40x10'
+  - series: pg_stat_activity_count{job="shared-postgresql-sql",instance="p:9187",env="prod",datname="",state="",usename="",application_name="",backend_type="autovacuum worker",wait_event_type="",wait_event=""}
+    values: '40x10'
+  - series: pg_settings_max_connections{job="shared-postgresql-sql",instance="p:9187",env="prod"}
+    values: '80x10'
+  alert_rule_test:
+  - eval_time: 8m
+    alertname: SharedPostgresConnectionsHigh
+    exp_alerts: []
+- name: 死锁计数增长告警
+  interval: 1m
+  input_series:
+  - series: pg_stat_database_deadlocks{job="shared-postgresql-sql",instance="p:9187",member="pg-pve1",env="prod",datname="app"}
+    values: '0 0 0 1 1 1'
+  alert_rule_test:
+  - eval_time: 5m
+    alertname: SharedPostgresDeadlocks
+    exp_alerts:
+    - exp_labels: {job: shared-postgresql-sql, instance: 'p:9187', member: pg-pve1, env: prod, severity: warning, service: shared-postgresql}
+      exp_annotations:
+        summary: 共享 PG prod pg-pve1 15 分钟内出现死锁
+- name: 空闲事务超过 10 分钟
+  interval: 1m
+  input_series:
+  - series: pg_stat_activity_max_tx_duration{job="shared-postgresql-sql",instance="p:9187",member="pg-pve1",env="prod",datname="app",state="idle in transaction",usename="a",application_name="x",backend_type="client backend",wait_event_type="Client",wait_event="ClientRead"}
+    values: '700x5'
+  alert_rule_test:
+  - eval_time: 3m
+    alertname: SharedPostgresIdleInTransaction
+    exp_alerts:
+    - exp_labels: {job: shared-postgresql-sql, instance: 'p:9187', member: pg-pve1, env: prod, severity: warning, service: shared-postgresql}
+      exp_annotations:
+        summary: 共享 PG prod pg-pve1 有事务空闲未提交超过 10 分钟
@@ -16,6 +16,10 @@ spec:
      group_interval: 5m
      repeat_interval: 4h
      routes:
+        # dev 必须排第一且不 continue：否则 dev 告警会被后面的 severity 路由送进生产频道。
+        - receiver: telegram-dev
+          matchers:
+            - env="dev"
        - receiver: telegram
          matchers:
            - severity="critical"
@@ -37,6 +41,11 @@ spec:
          - bot_token_file: /etc/vm/secrets/alertmanager-telegram/telegram_bot_token
            chat_id: -1003956377923
            send_resolved: true
+      - name: telegram-dev
+        telegram_configs:
+          - bot_token_file: /etc/vm/secrets/alertmanager-telegram/telegram_bot_token
+            chat_id: -1003651477106
+            send_resolved: true
  resources:
    requests:
      cpu: 25m