diff --git a/infrastructure/shared-postgresql/README.md b/infrastructure/shared-postgresql/README.md index 2c2ef29..55c233b 100644 --- a/infrastructure/shared-postgresql/README.md +++ b/infrastructure/shared-postgresql/README.md @@ -116,7 +116,7 @@ dev 为 laptop `:9188`,仅监听实例内网地址。exporter 以实例 OS 用 规则 `platform/observability/metrics/rules/shared-postgresql-sql.yaml`(PrometheusRule)以上游 `postgres_mixin` 为底改写:采集/SQL 不通、复制回放落后 > 1 MiB(与 failover 阈值一致)、主库复制槽 -保留 > 1 GiB、WAL 归档失败、XID/MXID 年龄、客户端连接 > 80%、空闲事务 > 10 分钟、事务 > 1 小时、 +保留 > 1 GiB、主库落在 pve1(HDD)超过 15 分钟、WAL 归档失败、XID/MXID 年龄、客户端连接 > 80%、空闲事务 > 10 分钟、事务 > 1 小时、 死锁、频繁请求 checkpoint。目标带 `env` 标签,Alertmanager 把 `env="dev"` 送到独立 dev 频道。 2026-10-01 验证:隔离集成测试覆盖 peer+ident 登录(主/备/dev)、读取 `pg_stat_statements`、 diff --git a/platform/observability/metrics/rules/shared-postgresql-sql.yaml b/platform/observability/metrics/rules/shared-postgresql-sql.yaml index 85ef67d..c70219a 100644 --- a/platform/observability/metrics/rules/shared-postgresql-sql.yaml +++ b/platform/observability/metrics/rules/shared-postgresql-sql.yaml @@ -24,6 +24,15 @@ spec: labels: {severity: warning, service: shared-postgresql} annotations: summary: '共享 PG {{ $labels.env }} {{ $labels.member }} 无法以监控角色执行 SQL' + # 正常 primary 在 laptop ZFS SSD;pve1 的数据盘是 HDD。自动切换后主库安全落到 pve1 + # 不会触发任何“无主/多主”规则(2026-09-30 的切换整整一天无人察觉),所以单独提示回切。 + # 15m 留给计划内切换演练。 + - alert: SharedPostgresPrimaryOnStandbyHost + expr: pg_replication_is_replica{job="shared-postgresql-sql", env="prod", member="pg-pve1"} == 0 + for: 15m + labels: {severity: warning, service: shared-postgresql} + annotations: + summary: 共享 PG 生产主库运行在 pg-pve1(HDD),请确认切换原因并计划回切 laptop - alert: SharedPostgresSqlCollectorFailing expr: pg_scrape_collector_success{job="shared-postgresql-sql"} == 0 or pg_exporter_last_scrape_error{job="shared-postgresql-sql"} == 1 for: 10m diff --git a/platform/observability/metrics/tests/shared-postgresql-sql.test.yaml b/platform/observability/metrics/tests/shared-postgresql-sql.test.yaml index 724256b..60cc030 100644 --- a/platform/observability/metrics/tests/shared-postgresql-sql.test.yaml +++ b/platform/observability/metrics/tests/shared-postgresql-sql.test.yaml @@ -137,3 +137,29 @@ tests: - exp_labels: {job: shared-postgresql-sql, instance: 'p:9187', member: pg-pve1, env: prod, severity: warning, service: shared-postgresql} exp_annotations: summary: 共享 PG prod pg-pve1 有事务空闲未提交超过 10 分钟 +- name: 主库落在 pve1 持续 15 分钟提示回切 + interval: 1m + input_series: + - series: pg_replication_is_replica{job="shared-postgresql-sql",instance="10.60.0.20:9187",cluster="homelab-pg-prod",member="pg-pve1",env="prod"} + values: '0x20' + - series: pg_replication_is_replica{job="shared-postgresql-sql",instance="192.168.10.127:9187",cluster="homelab-pg-prod",member="pg-laptop",env="prod"} + values: '1x20' + alert_rule_test: + - eval_time: 10m + alertname: SharedPostgresPrimaryOnStandbyHost + exp_alerts: [] + - eval_time: 16m + alertname: SharedPostgresPrimaryOnStandbyHost + exp_alerts: + - exp_labels: {job: shared-postgresql-sql, instance: '10.60.0.20:9187', cluster: homelab-pg-prod, member: pg-pve1, env: prod, severity: warning, service: shared-postgresql} + exp_annotations: + summary: 共享 PG 生产主库运行在 pg-pve1(HDD),请确认切换原因并计划回切 laptop +- name: 主库在 laptop 不提示 + interval: 1m + input_series: + - series: pg_replication_is_replica{job="shared-postgresql-sql",instance="10.60.0.20:9187",member="pg-pve1",env="prod"} + values: '1x20' + alert_rule_test: + - eval_time: 16m + alertname: SharedPostgresPrimaryOnStandbyHost + exp_alerts: []