From 2b382e15790dc071054d01381be3e7464bb1418b Mon Sep 17 00:00:00 2001 From: panxiao81 Date: Thu, 1 Oct 2026 16:41:04 +0000 Subject: [PATCH] =?UTF-8?q?=E4=BF=AE=E6=AD=A3=E5=85=B1=E4=BA=AB=20PG=20?= =?UTF-8?q?=E8=AE=A1=E6=95=B0=E5=99=A8=E5=91=8A=E8=AD=A6=E5=9C=A8=E6=96=B0?= =?UTF-8?q?=E5=BA=8F=E5=88=97=E4=B8=8A=E7=9A=84=E8=AF=AF=E6=8A=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit MetricsQL 的 increase() 把新序列首个样本视为从 0 增长：#173 合并后 exporter 首次被抓取，laptop 上 2026-09-25 搭建期累计的 9 次归档失败被当成刚发生， 触发 SharedPostgresWalArchiveFailing。归档本身正常（切换后 .partial 已归档）。 归档失败、死锁、checkpoint 三条改为与 offset 相减：新序列返回空， 计数器重置得负值也不告警。promtool 按 Prometheus 语义执行，测不出这一差异， 新表达式已在 VictoriaMetrics 现网数据上核对为空；补充新序列与重置回归场景。 Co-Authored-By: Claude Opus 5.5 --- .../metrics/rules/shared-postgresql-sql.yaml | 15 ++++++-- .../tests/shared-postgresql-sql.test.yaml | 34 +++++++++++++++++-- 2 files changed, 44 insertions(+), 5 deletions(-) diff --git a/platform/observability/metrics/rules/shared-postgresql-sql.yaml b/platform/observability/metrics/rules/shared-postgresql-sql.yaml index c70219a..21a96a1 100644 --- a/platform/observability/metrics/rules/shared-postgresql-sql.yaml +++ b/platform/observability/metrics/rules/shared-postgresql-sql.yaml @@ -56,8 +56,13 @@ spec: labels: {severity: warning, service: shared-postgresql} annotations: summary: '共享 PG {{ $labels.env }} 复制槽 {{ $labels.slot_name }} 保留 WAL 超过 1 GiB' + # 计数器规则不用 increase()：MetricsQL 把新序列首个样本视为从 0 增长，exporter 新上线或 + # 角色变化时，历史累计值（如 2026-09-25 搭建期的 9 次归档失败）会被当成刚发生而误报。 + # 与 offset 相减对新序列返回空；计数器重置（重启/stats reset）得负值，同样不告警。 - alert: SharedPostgresWalArchiveFailing - expr: increase(pg_stat_archiver_failed_count{job="shared-postgresql-sql"}[15m]) > 0 + expr: >- + pg_stat_archiver_failed_count{job="shared-postgresql-sql"} + - pg_stat_archiver_failed_count{job="shared-postgresql-sql"} offset 15m > 0 labels: {severity: warning, service: shared-postgresql} annotations: summary: '共享 PG {{ $labels.env }} {{ $labels.member }} WAL 归档失败，PITR 链可能中断' @@ -105,13 +110,17 @@ spec: annotations: summary: '共享 PG {{ $labels.env }} {{ $labels.member }} 最老事务已运行超过 1 小时' - alert: SharedPostgresDeadlocks - expr: sum without(datname) (increase(pg_stat_database_deadlocks{job="shared-postgresql-sql"}[15m])) > 0 + expr: >- + sum without(datname) (pg_stat_database_deadlocks{job="shared-postgresql-sql"} + - pg_stat_database_deadlocks{job="shared-postgresql-sql"} offset 15m) > 0 labels: {severity: warning, service: shared-postgresql} annotations: summary: '共享 PG {{ $labels.env }} {{ $labels.member }} 15 分钟内出现死锁' # 每日 pgBackRest full 会请求一次 checkpoint；每小时多于 4 次通常意味着 max_wal_size 偏小。 - alert: SharedPostgresCheckpointsRequested - expr: increase(pg_stat_checkpointer_num_requested_total{job="shared-postgresql-sql"}[1h]) > 4 + expr: >- + pg_stat_checkpointer_num_requested_total{job="shared-postgresql-sql"} + - pg_stat_checkpointer_num_requested_total{job="shared-postgresql-sql"} offset 1h > 4 for: 15m labels: {severity: warning, service: shared-postgresql} annotations: diff --git a/platform/observability/metrics/tests/shared-postgresql-sql.test.yaml b/platform/observability/metrics/tests/shared-postgresql-sql.test.yaml index 60cc030..8704385 100644 --- a/platform/observability/metrics/tests/shared-postgresql-sql.test.yaml +++ b/platform/observability/metrics/tests/shared-postgresql-sql.test.yaml @@ -117,9 +117,9 @@ tests: interval: 1m input_series: - series: pg_stat_database_deadlocks{job="shared-postgresql-sql",instance="p:9187",member="pg-pve1",env="prod",datname="app"} - values: '0 0 0 1 1 1' + values: '0x20 1x5' alert_rule_test: - - eval_time: 5m + - eval_time: 22m alertname: SharedPostgresDeadlocks exp_alerts: - exp_labels: {job: shared-postgresql-sql, instance: 'p:9187', member: pg-pve1, env: prod, severity: warning, service: shared-postgresql} @@ -163,3 +163,33 @@ tests: - eval_time: 16m alertname: SharedPostgresPrimaryOnStandbyHost exp_alerts: [] +- name: 新出现的计数器序列带历史累计值不误报归档失败 + interval: 1m + input_series: + - series: pg_stat_archiver_failed_count{job="shared-postgresql-sql",instance="l:9187",member="pg-laptop",env="prod"} + values: '_x20 9x20' + alert_rule_test: + - eval_time: 25m + alertname: SharedPostgresWalArchiveFailing + exp_alerts: [] +- name: 归档失败计数增长告警 + interval: 1m + input_series: + - series: pg_stat_archiver_failed_count{job="shared-postgresql-sql",instance="l:9187",member="pg-laptop",env="prod"} + values: '9x20 10x5' + alert_rule_test: + - eval_time: 22m + alertname: SharedPostgresWalArchiveFailing + exp_alerts: + - exp_labels: {job: shared-postgresql-sql, instance: 'l:9187', member: pg-laptop, env: prod, severity: warning, service: shared-postgresql} + exp_annotations: + summary: 共享 PG prod pg-laptop WAL 归档失败，PITR 链可能中断 +- name: 计数器重置不误报 checkpoint + interval: 1m + input_series: + - series: pg_stat_checkpointer_num_requested_total{job="shared-postgresql-sql",instance="l:9187",member="pg-laptop",env="prod"} + values: '16x60 0x30' + alert_rule_test: + - eval_time: 85m + alertname: SharedPostgresCheckpointsRequested + exp_alerts: [] -- 2.54.0