新增共享 PG 的 SQL 级指标与 dev 告警分流
- 数据库主机上按实例运行 prometheus-community/postgres_exporter 0.20.1， 以实例 OS 用户走本地 socket，经 peer + ident 映射到仅有 pg_monitor 的 homelab_monitor 角色；无密码、不经 Bao。版本与官方 sha256sums.txt 同源。 - prod/dev 启用 pg_stat_statements；monitoring.yml 仅在未生效时 reload/重启 （prod 走 Patroni pending restart），重复执行 changed=0。 - 告警以上游 postgres_mixin 为底改写为 PrometheusRule，阈值对齐 Patroni failover lag 与 max_slot_wal_keep_size；抓取目标带 env 标签。 - Alertmanager 新增 env="dev" 路由到独立 dev 频道，dev SQL 检查告警补 env。 Co-Authored-By: Claude Opus 5.5 <[email protected]>
This commit is contained in:
Claude Opus 5.5
@@ -19,6 +19,7 @@ pg_profiles:
|
||||
user: pgprod
|
||||
port: 5432
|
||||
api_port: 8008
|
||||
exporter_port: 9187
|
||||
dns: pg-prod.ad.ddupan.top
|
||||
memory_high: 768M
|
||||
memory_max: 1G
|
||||
@@ -28,6 +29,7 @@ pg_profiles:
|
||||
user: pgdev
|
||||
port: 5433
|
||||
dns: pg-dev.ad.ddupan.top
|
||||
exporter_port: 9188
|
||||
memory_high: 256M
|
||||
memory_max: 384M
|
||||
shared_buffers: 32MB
|
||||
@@ -38,3 +40,8 @@ pg_replication_addresses: ['192.168.10.127/32', '10.60.0.20/32']
|
||||
pg_repo_host: 10.60.0.21
|
||||
pg_repo_user: pgbackup
|
||||
pg_repo_path: /var/lib/homelab-pgbackrest
|
||||
# SQL 指标：exporter 以实例 OS 用户经本地 socket 连接，peer + ident 映射到只读监控角色，
|
||||
# 因此无需密码或 Bao；角色仅有 pg_monitor，不复用实例管理账号。
|
||||
pg_monitor_user: homelab_monitor
|
||||
# 升级只改版本；安装包与同版本官方 sha256sums.txt 一起下载校验，不写死摘要。
|
||||
pg_exporter_version: 0.20.1
|
||||
|
||||
@@ -0,0 +1,76 @@
|
||||
---
|
||||
# SQL 指标：激活监控所需的 HBA/ident 与 pg_stat_statements，创建只读监控角色并安装 exporter。
|
||||
# 不需要 Bao：模板先由 site.yml 写入；本 play 只做激活与验证。会重启实例（仅在预加载库未生效时），
|
||||
# 因此不放进 site.yml。重复执行应为 changed=0。
|
||||
- name: 激活实例监控配置
|
||||
hosts: pg_hosts
|
||||
become: true
|
||||
gather_facts: false
|
||||
serial: 1
|
||||
any_errors_fatal: true
|
||||
tasks:
|
||||
- name: 逐实例激活
|
||||
ansible.builtin.include_tasks: tasks/monitoring-activate.yml
|
||||
loop: "{{ pg_instances }}"
|
||||
loop_control:
|
||||
loop_var: pg_instance
|
||||
|
||||
- name: 重启 pending 的生产成员以加载 pg_stat_statements
|
||||
hosts: pg-laptop
|
||||
become: true
|
||||
become_user: "{{ pg_profiles.prod.user }}"
|
||||
gather_facts: false
|
||||
vars:
|
||||
pg_patronictl: [/opt/homelab-patroni/bin/patronictl, -c, "{{ pg_config_root }}/prod/patroni.yml"]
|
||||
tasks:
|
||||
- name: 读取集群成员状态
|
||||
ansible.builtin.command:
|
||||
argv: "{{ pg_patronictl + ['list', '-e', '-f', 'json'] }}"
|
||||
register: pg_members
|
||||
changed_when: false
|
||||
# Patroni 在原位重启 primary，不触发 failover；写入会短暂中断，客户端需重试。
|
||||
- name: 仅重启 pending restart 的成员
|
||||
ansible.builtin.command:
|
||||
argv: "{{ pg_patronictl + ['restart', 'pg-prod', '--pending', '--force'] }}"
|
||||
when: pg_members.stdout | from_json | selectattr('Pending restart', 'equalto', '*') | list | length > 0
|
||||
changed_when: true
|
||||
- name: 等待两个成员恢复且无 pending
|
||||
ansible.builtin.command:
|
||||
argv: "{{ pg_patronictl + ['list', '-e', '-f', 'json'] }}"
|
||||
register: pg_members_after
|
||||
changed_when: false
|
||||
retries: 30
|
||||
delay: 4
|
||||
until: >-
|
||||
(pg_members_after.stdout | from_json | length) == 2
|
||||
and (pg_members_after.stdout | from_json | selectattr('Pending restart', 'equalto', '*') | list | length) == 0
|
||||
and (pg_members_after.stdout | from_json | selectattr('State', 'in', ['running', 'streaming']) | list | length) == 2
|
||||
|
||||
- name: 创建监控角色并安装 exporter
|
||||
hosts: pg_hosts
|
||||
become: true
|
||||
gather_facts: false
|
||||
tasks:
|
||||
- name: 下载 postgres_exporter 并校验同版本官方摘要
|
||||
ansible.builtin.get_url:
|
||||
url: "{{ pg_exporter_base }}/postgres_exporter-{{ pg_exporter_version }}.linux-amd64.tar.gz"
|
||||
dest: /opt/homelab-postgres-exporter.tar.gz
|
||||
checksum: "sha256:{{ pg_exporter_base }}/sha256sums.txt"
|
||||
mode: '0644'
|
||||
vars:
|
||||
pg_exporter_base: https://github.com/prometheus-community/postgres_exporter/releases/download/v{{ pg_exporter_version }}
|
||||
register: pg_exporter_download
|
||||
retries: 5
|
||||
delay: 10
|
||||
until: pg_exporter_download is succeeded
|
||||
- name: 解压 exporter（保留上游许可证）
|
||||
ansible.builtin.unarchive:
|
||||
src: /opt/homelab-postgres-exporter.tar.gz
|
||||
dest: /opt
|
||||
remote_src: true
|
||||
creates: /opt/postgres_exporter-{{ pg_exporter_version }}.linux-amd64/postgres_exporter
|
||||
- name: 逐实例配置
|
||||
ansible.builtin.include_tasks: tasks/monitoring-exporter.yml
|
||||
loop: "{{ pg_instances }}"
|
||||
loop_control:
|
||||
loop_var: pg_instance
|
||||
@@ -71,7 +71,7 @@
|
||||
owner: root
|
||||
group: "{{ pg_profile.user }}"
|
||||
mode: '0640'
|
||||
loop: "{{ ['patroni.yml', 'pgbackrest.conf'] if pg_instance == 'prod' else ['postgresql.conf', 'pg_hba.conf'] }}"
|
||||
loop: "{{ ['patroni.yml', 'pgbackrest.conf'] if pg_instance == 'prod' else ['postgresql.conf', 'pg_hba.conf', 'pg_ident.conf'] }}"
|
||||
no_log: true
|
||||
- name: 安装实例专用服务（不自动启动或重启）
|
||||
ansible.builtin.template:
|
||||
|
||||
@@ -66,6 +66,8 @@ postgresql:
|
||||
unix_socket_directories: {{ pg_socket_dir }}
|
||||
unix_socket_permissions: '0700'
|
||||
password_encryption: scram-sha-256
|
||||
# postmaster 级参数：变更后需由 monitoring.yml 执行 Patroni pending restart。
|
||||
shared_preload_libraries: pg_stat_statements
|
||||
shared_buffers: {{ pg_profile.shared_buffers }}
|
||||
work_mem: 4MB
|
||||
maintenance_work_mem: 64MB
|
||||
@@ -76,6 +78,7 @@ postgresql:
|
||||
log_statement: none
|
||||
log_min_error_statement: panic
|
||||
pg_hba:
|
||||
- local all {{ pg_monitor_user }} peer map=monitor
|
||||
- local all {{ pg_profile.user }} peer
|
||||
- local replication {{ pg_profile.user }} peer
|
||||
- local replication replicator scram-sha-256
|
||||
@@ -88,6 +91,9 @@ postgresql:
|
||||
{% endfor %}
|
||||
- host all all 0.0.0.0/0 reject
|
||||
- host all all ::/0 reject
|
||||
# 只有实例 OS 用户（exporter 运行身份）可映射为监控角色。
|
||||
pg_ident:
|
||||
- monitor {{ pg_profile.user }} {{ pg_monitor_user }}
|
||||
remove_data_directory_on_rewind_failure: false
|
||||
remove_data_directory_on_diverged_timelines: false
|
||||
watchdog:
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
local all {{ pg_monitor_user }} peer map=monitor
|
||||
local all {{ pg_profile.user }} peer
|
||||
{% for cidr in pg_client_cidrs %}
|
||||
hostssl all all {{ cidr }} scram-sha-256
|
||||
|
||||
@@ -0,0 +1,2 @@
|
||||
# 只有实例 OS 用户（exporter 运行身份）可映射为监控角色。
|
||||
monitor {{ pg_profile.user }} {{ pg_monitor_user }}
|
||||
+3
@@ -3,6 +3,7 @@ port = {{ pg_profile.port }}
|
||||
unix_socket_directories = '{{ pg_socket_dir }}'
|
||||
unix_socket_permissions = 0700
|
||||
hba_file = '{{ pg_config_dir }}/pg_hba.conf'
|
||||
ident_file = '{{ pg_config_dir }}/pg_ident.conf'
|
||||
password_encryption = 'scram-sha-256'
|
||||
shared_buffers = '{{ pg_profile.shared_buffers }}'
|
||||
max_connections = {{ pg_profile.max_connections }}
|
||||
@@ -16,3 +17,5 @@ ssl_ca_file = '{{ pg_config_dir }}/ca.crt'
|
||||
archive_mode = off
|
||||
log_statement = none
|
||||
log_min_error_statement = panic
|
||||
# postmaster 级参数：变更后需由 monitoring.yml 重启 dev 实例。
|
||||
shared_preload_libraries = 'pg_stat_statements'
|
||||
|
||||
@@ -0,0 +1,70 @@
|
||||
---
|
||||
- name: 设置现有实例路径
|
||||
ansible.builtin.set_fact:
|
||||
pg_profile: "{{ pg_profiles[pg_instance] }}"
|
||||
pg_config_dir: "{{ pg_config_root }}/{{ pg_instance }}"
|
||||
pg_socket_dir: "/run/homelab-postgresql-{{ pg_instance }}"
|
||||
- name: 激活前必须存在运行中的受管服务
|
||||
ansible.builtin.command:
|
||||
argv: [systemctl, is-active, "homelab-postgresql-{{ pg_instance }}.service"]
|
||||
changed_when: false
|
||||
# 生产：Patroni 只在自身 reload 时把本地配置写成数据目录里的 postgresql.conf/pg_ident.conf，
|
||||
# 所以文件视图能说明 Patroni 是否已接收新配置；预加载库是否已生效另看 current_setting。
|
||||
# dev：ident_file 是 postmaster 级参数，视图读取的是“当前生效”的那个文件，
|
||||
# 所以启动时仍指向旧路径就会读不到映射；两列都为真才算已加载。
|
||||
- name: 读取监控配置的激活状态
|
||||
ansible.builtin.command:
|
||||
argv:
|
||||
- "{{ pg_bin_dir }}/psql"
|
||||
- -X
|
||||
- -At
|
||||
- -h
|
||||
- "{{ pg_socket_dir }}"
|
||||
- -p
|
||||
- "{{ pg_profile.port | string }}"
|
||||
- -d
|
||||
- postgres
|
||||
- -c
|
||||
- >-
|
||||
SELECT
|
||||
EXISTS (SELECT 1 FROM pg_ident_file_mappings WHERE map_name = 'monitor' AND error IS NULL)
|
||||
AND EXISTS (SELECT 1 FROM pg_file_settings WHERE name = 'shared_preload_libraries'
|
||||
AND setting ~ 'pg_stat_statements' AND error IS NULL),
|
||||
current_setting('shared_preload_libraries') ~ 'pg_stat_statements'
|
||||
become: true
|
||||
become_user: "{{ pg_profile.user }}"
|
||||
changed_when: false
|
||||
register: pg_monitor_state
|
||||
- name: 让 Patroni 接收新的本地配置（reload，不重启）
|
||||
ansible.builtin.command:
|
||||
argv: [systemctl, kill, --kill-whom=main, --signal=HUP, homelab-postgresql-prod.service]
|
||||
when: pg_instance == 'prod' and pg_monitor_state.stdout.split('|')[0] != 't'
|
||||
changed_when: true
|
||||
- name: 等待 Patroni 写出新配置
|
||||
ansible.builtin.command:
|
||||
argv: "{{ pg_monitor_state.cmd }}"
|
||||
become: true
|
||||
become_user: "{{ pg_profile.user }}"
|
||||
when: pg_instance == 'prod' and pg_monitor_state.stdout.split('|')[0] != 't'
|
||||
changed_when: false
|
||||
register: pg_monitor_reloaded
|
||||
retries: 10
|
||||
delay: 3
|
||||
until: pg_monitor_reloaded.stdout.split('|')[0] == 't'
|
||||
# dev 没有 Patroni；预加载库与 ident_file 都是 postmaster 级参数，只能重启。
|
||||
- name: 重启 dev 以加载监控配置
|
||||
ansible.builtin.systemd_service:
|
||||
name: homelab-postgresql-dev.service
|
||||
state: restarted
|
||||
when: pg_instance == 'dev' and pg_monitor_state.stdout != 't|t'
|
||||
- name: 检查 dev 重启后监控配置已生效
|
||||
ansible.builtin.command:
|
||||
argv: "{{ pg_monitor_state.cmd }}"
|
||||
become: true
|
||||
become_user: "{{ pg_profile.user }}"
|
||||
when: pg_instance == 'dev'
|
||||
changed_when: false
|
||||
register: pg_monitor_dev
|
||||
retries: 10
|
||||
delay: 3
|
||||
until: pg_monitor_dev.rc == 0 and pg_monitor_dev.stdout == 't|t'
|
||||
@@ -0,0 +1,61 @@
|
||||
---
|
||||
- name: 设置现有实例路径
|
||||
ansible.builtin.set_fact:
|
||||
pg_profile: "{{ pg_profiles[pg_instance] }}"
|
||||
pg_config_dir: "{{ pg_config_root }}/{{ pg_instance }}"
|
||||
pg_parent_dir: "{{ pg_data_root }}/{{ pg_instance }}"
|
||||
pg_socket_dir: "/run/homelab-postgresql-{{ pg_instance }}"
|
||||
pg_psql: ["{{ pg_bin_dir }}/psql", -X, -At, -v, ON_ERROR_STOP=1, -h, "/run/homelab-postgresql-{{ pg_instance }}",
|
||||
-p, "{{ pg_profiles[pg_instance].port | string }}", -d, postgres]
|
||||
# 角色与扩展只在可写实例上创建，standby 经复制获得。
|
||||
- name: 检查监控角色与扩展是否符合声明
|
||||
ansible.builtin.command:
|
||||
argv: "{{ pg_psql + ['-c', pg_monitor_check] }}"
|
||||
vars:
|
||||
pg_monitor_check: >-
|
||||
SELECT pg_is_in_recovery(),
|
||||
COALESCE((SELECT rolcanlogin AND NOT rolsuper AND NOT rolcreaterole AND NOT rolcreatedb
|
||||
AND NOT rolreplication AND NOT rolbypassrls AND rolconnlimit = 3 AND rolpassword IS NULL
|
||||
AND pg_has_role(oid, 'pg_monitor', 'MEMBER')
|
||||
FROM pg_authid WHERE rolname = '{{ pg_monitor_user }}'), false)
|
||||
AND EXISTS (SELECT 1 FROM pg_extension WHERE extname = 'pg_stat_statements')
|
||||
become: true
|
||||
become_user: "{{ pg_profile.user }}"
|
||||
changed_when: false
|
||||
register: pg_monitor_role
|
||||
- name: 创建或修正监控角色（无密码，仅能经本地 ident 映射登录）
|
||||
ansible.builtin.command:
|
||||
argv: "{{ pg_psql }}"
|
||||
stdin: |
|
||||
SELECT 'CREATE ROLE {{ pg_monitor_user }}'
|
||||
WHERE NOT EXISTS (SELECT 1 FROM pg_roles WHERE rolname = '{{ pg_monitor_user }}') \gexec
|
||||
ALTER ROLE {{ pg_monitor_user }} LOGIN NOSUPERUSER NOCREATEDB NOCREATEROLE NOREPLICATION NOBYPASSRLS
|
||||
CONNECTION LIMIT 3 PASSWORD NULL;
|
||||
GRANT pg_monitor TO {{ pg_monitor_user }};
|
||||
CREATE EXTENSION IF NOT EXISTS pg_stat_statements;
|
||||
become: true
|
||||
become_user: "{{ pg_profile.user }}"
|
||||
when: pg_monitor_role.stdout == 'f|f'
|
||||
changed_when: true
|
||||
- name: 安装实例 exporter 服务
|
||||
ansible.builtin.template:
|
||||
src: postgres-exporter.service.j2
|
||||
dest: "/etc/systemd/system/homelab-postgresql-{{ pg_instance }}-exporter.service"
|
||||
mode: '0644'
|
||||
register: pg_exporter_unit
|
||||
- name: 启用 exporter
|
||||
ansible.builtin.systemd_service:
|
||||
name: "homelab-postgresql-{{ pg_instance }}-exporter.service"
|
||||
daemon_reload: "{{ pg_exporter_unit is changed }}"
|
||||
enabled: true
|
||||
state: "{{ 'restarted' if pg_exporter_unit is changed else 'started' }}"
|
||||
# 同时证明 HBA/ident 已加载、角色可登录、exporter 能查询。
|
||||
- name: 检查 exporter 已连上数据库
|
||||
ansible.builtin.uri:
|
||||
url: "http://{{ pg_address }}:{{ pg_profile.exporter_port }}/metrics"
|
||||
return_content: true
|
||||
register: pg_exporter_metrics
|
||||
changed_when: false
|
||||
retries: 10
|
||||
delay: 3
|
||||
until: pg_exporter_metrics.status == 200 and pg_exporter_metrics.content is search('(?m)^pg_up 1$')
|
||||
@@ -0,0 +1,27 @@
|
||||
[Unit]
|
||||
Description=PostgreSQL {{ pg_instance }} SQL metrics
|
||||
After=homelab-postgresql-{{ pg_instance }}.service
|
||||
[Service]
|
||||
# 本地 socket 上 TLS 无意义，sslmode=disable 只是避免驱动默认尝试 TLS。
|
||||
# 以实例 OS 用户运行才能进入 0700 的 socket 目录；peer + ident 把它映射成只读监控角色。
|
||||
# 监控角色只约束 exporter 自己发出的查询；这不是隔离边界——同一 OS 用户仍可经 peer 以
|
||||
# superuser 登录，exporter 被攻破即等同实例属主。InaccessiblePaths 只是不让它直接读写
|
||||
# 数据目录、私钥与 pgpass。暴露面因此仅限内网监听地址。
|
||||
User={{ pg_profile.user }}
|
||||
Group={{ pg_profile.user }}
|
||||
Environment="DATA_SOURCE_NAME=host={{ pg_socket_dir }} port={{ pg_profile.port }} user={{ pg_monitor_user }} dbname=postgres sslmode=disable application_name=postgres_exporter"
|
||||
ExecStart=/opt/postgres_exporter-{{ pg_exporter_version }}.linux-amd64/postgres_exporter \
|
||||
--web.listen-address={{ pg_address }}:{{ pg_profile.exporter_port }} \
|
||||
--collector.database_wraparound --collector.long_running_transactions \
|
||||
--collector.stat_checkpointer --collector.stat_wal_receiver --collector.stat_statements
|
||||
Restart=on-failure
|
||||
RestartSec=5
|
||||
InaccessiblePaths={{ pg_parent_dir }} {{ pg_config_dir }}
|
||||
NoNewPrivileges=true
|
||||
ProtectSystem=strict
|
||||
ProtectHome=true
|
||||
PrivateTmp=true
|
||||
PrivateDevices=true
|
||||
MemoryMax=64M
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
@@ -88,9 +88,9 @@ def main():
|
||||
pg_etcd_endpoints=[f'{h["etcd_address"]}:23379' for h in hosts.values()],
|
||||
pg_config_dir='/node/config',pg_parent_dir='/node',pg_data_dir='/node/data',pg_socket_dir='/node/socket',
|
||||
pg_bin_dir='/usr/lib/postgresql/18/bin',pg_replication_addresses=['127.0.0.0/8'],pg_client_cidrs=['127.0.0.0/8'],
|
||||
pg_repo_remote=False,pg_repo_path='/repo')
|
||||
pg_repo_remote=False,pg_repo_path='/repo',pg_monitor_user='homelab_monitor')
|
||||
values[name]=v
|
||||
for template in (['postgresql.conf','pg_hba.conf'] if name=='dev' else ['patroni.yml','pgbackrest.conf']):
|
||||
for template in (['postgresql.conf','pg_hba.conf','pg_ident.conf'] if name=='dev' else ['patroni.yml','pgbackrest.conf']):
|
||||
(cfg/template).write_text(env.get_template(template+'.j2').render(**v))
|
||||
for folder in ['socket','backup-lock','backup-spool']:(d/folder).mkdir()
|
||||
(pg/'repo').mkdir();(pg/'restore').mkdir();(pg/'pitr').mkdir()
|
||||
@@ -131,6 +131,20 @@ def main():
|
||||
initial_backup=json.loads(backrest('p1','--output=json','info').stdout)[0]['backup'][-1]['label']
|
||||
recovery_target=sql('p1','SELECT clock_timestamp()::text')
|
||||
print('PASS: separate dev, TLS verify-full, non-superuser Ayatori native privileges, initial backup',flush=True)
|
||||
# 与 monitoring.yml 相同的声明：无密码，只能由实例 OS 用户经 peer + ident 映射登录。
|
||||
monitor_role="CREATE ROLE homelab_monitor LOGIN CONNECTION LIMIT 3 PASSWORD NULL; GRANT pg_monitor TO homelab_monitor; CREATE EXTENSION pg_stat_statements;"
|
||||
sql('p1',monitor_role);sql('dev',monitor_role)
|
||||
def monitor(name,query):
|
||||
return run(['docker','exec','-i',prefix+'-'+name,'psql','-X','-v','ON_ERROR_STOP=1','-At','-h','/node/socket','-p','25432','-U','homelab_monitor','postgres'],input=query).stdout.strip()
|
||||
for name in ['p1','p2','dev']:
|
||||
wait_for(lambda: monitor(name,"SELECT current_setting('shared_preload_libraries')||(SELECT count(*)>=0 FROM pg_stat_statements)")=='pg_stat_statementstrue')
|
||||
for name in ['p1','dev']:
|
||||
try:
|
||||
monitor(name,'CREATE TABLE public.monitor_write(id int)')
|
||||
raise AssertionError(f'monitor role could write on {name}')
|
||||
except subprocess.CalledProcessError:
|
||||
pass
|
||||
print('PASS: monitor role logs in via peer+ident on primary/standby/dev, reads pg_stat_statements, cannot write',flush=True)
|
||||
proxycfg=w/'proxy.cfg'
|
||||
# 文件父目录仍由调用者拥有，新增代理配置不触碰 PG 文件。
|
||||
proxycfg.write_text(env.get_template('haproxy.cfg.j2').render(pg_proxy_bind='127.0.2.10:26432',pg_proxy_members=[dict(name=n,address=values[n]['pg_address'],port=25432,api_port=28008) for n in ['p1','p2']]))
|
||||
|
||||
@@ -25,6 +25,8 @@ resources:
|
||||
- scrapes/shared-etcd.yaml
|
||||
- rules/shared-postgresql.yaml
|
||||
- scrapes/shared-postgresql.yaml
|
||||
- rules/shared-postgresql-sql.yaml
|
||||
- scrapes/shared-postgresql-sql.yaml
|
||||
- scrapes/platform-controllers.yaml
|
||||
- rules/platform-controllers.yaml
|
||||
- scrapes/cnpg.yaml
|
||||
|
||||
@@ -58,6 +58,7 @@ spec:
|
||||
- alert: SharedPostgresDevUnavailable
|
||||
expr: homelab_data_dev_sql_ready{job="shared-data-maintenance"} == 0
|
||||
for: 3m
|
||||
labels: {severity: warning, service: shared-postgresql}
|
||||
# env 供 Alertmanager 把 dev 分流到独立 channel。
|
||||
labels: {severity: warning, service: shared-postgresql, env: dev}
|
||||
annotations:
|
||||
summary: 共享 PG 开发实例本地 SQL 不可用或处于只读状态
|
||||
|
||||
@@ -0,0 +1,109 @@
|
||||
# 以 postgres_exporter 上游 postgres_mixin 为底按本环境改写：实例停止、多主、角色变化已由
|
||||
# Patroni 规则（shared-postgresql.yaml）覆盖；QPS/缓存命中率/回滚数不是故障信号，未采用；
|
||||
# 上游 PostgresHasReplicationSlotUsed 会被 Patroni 在 standby 保留的非活跃槽持续触发，未采用。
|
||||
# 每条告警都带抓取目标的 env 标签，Alertmanager 按它把 dev 分流到独立 channel。
|
||||
apiVersion: monitoring.coreos.com/v1
|
||||
kind: PrometheusRule
|
||||
metadata:
|
||||
name: shared-postgresql-sql
|
||||
namespace: monitoring
|
||||
spec:
|
||||
groups:
|
||||
- name: shared-postgresql-sql
|
||||
rules:
|
||||
- alert: SharedPostgresSqlExporterDown
|
||||
expr: up{job="shared-postgresql-sql"} == 0
|
||||
for: 3m
|
||||
labels: {severity: warning, service: shared-postgresql}
|
||||
annotations:
|
||||
summary: '共享 PG {{ $labels.env }} {{ $labels.member }} 的 SQL exporter 无法采集'
|
||||
# 进程停止已有 Patroni critical；这里覆盖进程在但 SQL 不通（HBA、连接耗尽、dev 实例）。
|
||||
- alert: SharedPostgresSqlUnreachable
|
||||
expr: pg_up{job="shared-postgresql-sql"} == 0
|
||||
for: 2m
|
||||
labels: {severity: warning, service: shared-postgresql}
|
||||
annotations:
|
||||
summary: '共享 PG {{ $labels.env }} {{ $labels.member }} 无法以监控角色执行 SQL'
|
||||
- alert: SharedPostgresSqlCollectorFailing
|
||||
expr: pg_scrape_collector_success{job="shared-postgresql-sql"} == 0 or pg_exporter_last_scrape_error{job="shared-postgresql-sql"} == 1
|
||||
for: 10m
|
||||
labels: {severity: warning, service: shared-postgresql}
|
||||
annotations:
|
||||
summary: '共享 PG {{ $labels.env }} {{ $labels.member }} 部分 SQL 指标采集失败 {{ $labels.collector }}'
|
||||
# 与 Patroni maximum_lag_on_failover（1 MiB）一致：持续超过即该 standby 不再是合格的切换候选。
|
||||
- alert: SharedPostgresReplicationLagging
|
||||
expr: pg_stat_replication_pg_wal_lsn_diff{job="shared-postgresql-sql"} > 1048576
|
||||
for: 5m
|
||||
labels: {severity: warning, service: shared-postgresql}
|
||||
annotations:
|
||||
summary: '共享 PG {{ $labels.env }} 副本 {{ $labels.application_name }} 回放落后超过 1 MiB，已不满足自动切换条件'
|
||||
# 只看可写成员：standby 上 Patroni 维护的槽副本不反映真实保留压力。
|
||||
# 1 GiB 是 max_slot_wal_keep_size（2GB）的一半，留出处理时间。
|
||||
- alert: SharedPostgresReplicationSlotRetention
|
||||
expr: >-
|
||||
pg_replication_slots_pg_wal_lsn_diff{job="shared-postgresql-sql"} > 1073741824
|
||||
and on(instance) pg_replication_is_replica{job="shared-postgresql-sql"} == 0
|
||||
for: 10m
|
||||
labels: {severity: warning, service: shared-postgresql}
|
||||
annotations:
|
||||
summary: '共享 PG {{ $labels.env }} 复制槽 {{ $labels.slot_name }} 保留 WAL 超过 1 GiB'
|
||||
- alert: SharedPostgresWalArchiveFailing
|
||||
expr: increase(pg_stat_archiver_failed_count{job="shared-postgresql-sql"}[15m]) > 0
|
||||
labels: {severity: warning, service: shared-postgresql}
|
||||
annotations:
|
||||
summary: '共享 PG {{ $labels.env }} {{ $labels.member }} WAL 归档失败，PITR 链可能中断'
|
||||
# XID/MXID 上限约 2^31；超过一半说明 freeze 跟不上，超过 75% 已接近强制停写；critical 时不重复 warning。
|
||||
- alert: SharedPostgresWraparoundRisk
|
||||
expr: >-
|
||||
(max without(datname) (pg_database_wraparound_age_datfrozenxid_seconds{job="shared-postgresql-sql"}) > 1e9
|
||||
or max without(datname) (pg_database_wraparound_age_datminmxid_seconds{job="shared-postgresql-sql"}) > 1e9)
|
||||
unless (max without(datname) (pg_database_wraparound_age_datfrozenxid_seconds{job="shared-postgresql-sql"}) > 1.6e9
|
||||
or max without(datname) (pg_database_wraparound_age_datminmxid_seconds{job="shared-postgresql-sql"}) > 1.6e9)
|
||||
for: 10m
|
||||
labels: {severity: warning, service: shared-postgresql}
|
||||
annotations:
|
||||
summary: '共享 PG {{ $labels.env }} {{ $labels.member }} XID/MXID 年龄超过 10 亿'
|
||||
- alert: SharedPostgresWraparoundImminent
|
||||
expr: >-
|
||||
max without(datname) (pg_database_wraparound_age_datfrozenxid_seconds{job="shared-postgresql-sql"}) > 1.6e9
|
||||
or max without(datname) (pg_database_wraparound_age_datminmxid_seconds{job="shared-postgresql-sql"}) > 1.6e9
|
||||
for: 5m
|
||||
labels: {severity: critical, service: shared-postgresql}
|
||||
annotations:
|
||||
summary: '共享 PG {{ $labels.env }} {{ $labels.member }} XID/MXID 接近回卷停写，需立即 VACUUM FREEZE'
|
||||
- alert: SharedPostgresConnectionsHigh
|
||||
expr: >-
|
||||
sum without(datname, state, usename, application_name, backend_type, wait_event_type, wait_event)
|
||||
(pg_stat_activity_count{job="shared-postgresql-sql", backend_type="client backend"})
|
||||
/ pg_settings_max_connections{job="shared-postgresql-sql"} > 0.8
|
||||
for: 5m
|
||||
labels: {severity: warning, service: shared-postgresql}
|
||||
annotations:
|
||||
summary: '共享 PG {{ $labels.env }} {{ $labels.member }} 客户端连接超过 max_connections 的 80%'
|
||||
# 长事务阻止 vacuum 回收并持有锁；idle in transaction 通常是应用忘记提交。
|
||||
- alert: SharedPostgresIdleInTransaction
|
||||
expr: >-
|
||||
max without(datname, state, usename, application_name, backend_type, wait_event_type, wait_event)
|
||||
(pg_stat_activity_max_tx_duration{job="shared-postgresql-sql", state=~"idle in transaction.*"}) > 600
|
||||
for: 1m
|
||||
labels: {severity: warning, service: shared-postgresql}
|
||||
annotations:
|
||||
summary: '共享 PG {{ $labels.env }} {{ $labels.member }} 有事务空闲未提交超过 10 分钟'
|
||||
- alert: SharedPostgresLongTransaction
|
||||
expr: pg_long_running_transactions_oldest_timestamp_seconds{job="shared-postgresql-sql"} > 3600
|
||||
for: 1m
|
||||
labels: {severity: warning, service: shared-postgresql}
|
||||
annotations:
|
||||
summary: '共享 PG {{ $labels.env }} {{ $labels.member }} 最老事务已运行超过 1 小时'
|
||||
- alert: SharedPostgresDeadlocks
|
||||
expr: sum without(datname) (increase(pg_stat_database_deadlocks{job="shared-postgresql-sql"}[15m])) > 0
|
||||
labels: {severity: warning, service: shared-postgresql}
|
||||
annotations:
|
||||
summary: '共享 PG {{ $labels.env }} {{ $labels.member }} 15 分钟内出现死锁'
|
||||
# 每日 pgBackRest full 会请求一次 checkpoint；每小时多于 4 次通常意味着 max_wal_size 偏小。
|
||||
- alert: SharedPostgresCheckpointsRequested
|
||||
expr: increase(pg_stat_checkpointer_num_requested_total{job="shared-postgresql-sql"}[1h]) > 4
|
||||
for: 15m
|
||||
labels: {severity: warning, service: shared-postgresql}
|
||||
annotations:
|
||||
summary: '共享 PG {{ $labels.env }} {{ $labels.member }} 频繁请求 checkpoint，max_wal_size 可能偏小'
|
||||
@@ -0,0 +1,18 @@
|
||||
# postgres_exporter 跑在数据库主机上，按实例一个端口；env 标签供 Alertmanager 分流 dev。
|
||||
apiVersion: operator.victoriametrics.com/v1beta1
|
||||
kind: VMStaticScrape
|
||||
metadata:
|
||||
name: shared-postgresql-sql
|
||||
namespace: monitoring
|
||||
spec:
|
||||
jobName: shared-postgresql-sql
|
||||
targetEndpoints:
|
||||
- targets: ['192.168.10.127:9187']
|
||||
labels: {cluster: homelab-pg-prod, member: pg-laptop, env: prod}
|
||||
interval: 30s
|
||||
- targets: ['10.60.0.20:9187']
|
||||
labels: {cluster: homelab-pg-prod, member: pg-pve1, env: prod}
|
||||
interval: 30s
|
||||
- targets: ['192.168.10.127:9188']
|
||||
labels: {cluster: homelab-pg-dev, member: pg-laptop, env: dev}
|
||||
interval: 30s
|
||||
@@ -10,7 +10,7 @@ import sys
|
||||
import yaml
|
||||
|
||||
groups = []
|
||||
for name in ("kubernetes-health", "host-health", "monitoring-delivery", "platform-controllers", "data-services", "openbao", "shared-data-maintenance"):
|
||||
for name in ("kubernetes-health", "host-health", "monitoring-delivery", "platform-controllers", "data-services", "openbao", "shared-data-maintenance", "shared-postgresql-sql"):
|
||||
path = pathlib.Path(sys.argv[1]) / f"{name}.yaml"
|
||||
groups.extend(yaml.safe_load(path.read_text())["spec"]["groups"])
|
||||
pathlib.Path(sys.argv[2]).write_text(yaml.safe_dump({"groups": groups}, allow_unicode=True))
|
||||
|
||||
@@ -125,6 +125,7 @@ tests:
|
||||
instance: host:9109
|
||||
severity: warning
|
||||
service: shared-postgresql
|
||||
env: dev
|
||||
exp_annotations:
|
||||
summary: 共享 PG 开发实例本地 SQL 不可用或处于只读状态
|
||||
- name: dev 恢复后解除
|
||||
|
||||
@@ -0,0 +1,139 @@
|
||||
rule_files:
|
||||
- rules.yaml
|
||||
evaluation_interval: 1m
|
||||
tests:
|
||||
- name: dev SQL 不通时告警带 env=dev 以便分流
|
||||
interval: 1m
|
||||
input_series:
|
||||
- series: pg_up{job="shared-postgresql-sql",instance="h:9188",cluster="homelab-pg-dev",member="pg-laptop",env="dev"}
|
||||
values: '0x5'
|
||||
alert_rule_test:
|
||||
- eval_time: 5m
|
||||
alertname: SharedPostgresSqlUnreachable
|
||||
exp_alerts:
|
||||
- exp_labels: {job: shared-postgresql-sql, instance: 'h:9188', cluster: homelab-pg-dev, member: pg-laptop, env: dev, severity: warning, service: shared-postgresql}
|
||||
exp_annotations:
|
||||
summary: 共享 PG dev pg-laptop 无法以监控角色执行 SQL
|
||||
- name: 回放落后持续超过 1 MiB 告警
|
||||
interval: 1m
|
||||
input_series:
|
||||
- series: pg_stat_replication_pg_wal_lsn_diff{job="shared-postgresql-sql",instance="p:9187",member="pg-pve1",env="prod",application_name="pg-laptop"}
|
||||
values: '2097152x10'
|
||||
alert_rule_test:
|
||||
- eval_time: 8m
|
||||
alertname: SharedPostgresReplicationLagging
|
||||
exp_alerts:
|
||||
- exp_labels: {job: shared-postgresql-sql, instance: 'p:9187', member: pg-pve1, env: prod, application_name: pg-laptop, severity: warning, service: shared-postgresql}
|
||||
exp_annotations:
|
||||
summary: 共享 PG prod 副本 pg-laptop 回放落后超过 1 MiB，已不满足自动切换条件
|
||||
- name: 短暂落后不告警
|
||||
interval: 1m
|
||||
input_series:
|
||||
- series: pg_stat_replication_pg_wal_lsn_diff{job="shared-postgresql-sql",instance="p:9187",env="prod",application_name="pg-laptop"}
|
||||
values: '2097152x2 0x8'
|
||||
alert_rule_test:
|
||||
- eval_time: 8m
|
||||
alertname: SharedPostgresReplicationLagging
|
||||
exp_alerts: []
|
||||
- name: 复制槽积压只在主库告警
|
||||
interval: 1m
|
||||
input_series:
|
||||
- series: pg_replication_slots_pg_wal_lsn_diff{job="shared-postgresql-sql",instance="p:9187",env="prod",slot_name="pg_laptop"}
|
||||
values: '2147483648x15'
|
||||
- series: pg_replication_is_replica{job="shared-postgresql-sql",instance="p:9187",env="prod"}
|
||||
values: '0x15'
|
||||
- series: pg_replication_slots_pg_wal_lsn_diff{job="shared-postgresql-sql",instance="r:9187",env="prod",slot_name="pg_pve1"}
|
||||
values: '2147483648x15'
|
||||
- series: pg_replication_is_replica{job="shared-postgresql-sql",instance="r:9187",env="prod"}
|
||||
values: '1x15'
|
||||
alert_rule_test:
|
||||
- eval_time: 12m
|
||||
alertname: SharedPostgresReplicationSlotRetention
|
||||
exp_alerts:
|
||||
- exp_labels: {job: shared-postgresql-sql, instance: 'p:9187', env: prod, slot_name: pg_laptop, severity: warning, service: shared-postgresql}
|
||||
exp_annotations:
|
||||
summary: 共享 PG prod 复制槽 pg_laptop 保留 WAL 超过 1 GiB
|
||||
- name: 回卷 critical 时不重复 warning
|
||||
interval: 1m
|
||||
input_series:
|
||||
- series: pg_database_wraparound_age_datfrozenxid_seconds{job="shared-postgresql-sql",instance="p:9187",member="pg-pve1",env="prod",datname="app"}
|
||||
values: '1700000000x15'
|
||||
- series: pg_database_wraparound_age_datminmxid_seconds{job="shared-postgresql-sql",instance="p:9187",member="pg-pve1",env="prod",datname="app"}
|
||||
values: '1200000000x15'
|
||||
alert_rule_test:
|
||||
- eval_time: 12m
|
||||
alertname: SharedPostgresWraparoundRisk
|
||||
exp_alerts: []
|
||||
- eval_time: 12m
|
||||
alertname: SharedPostgresWraparoundImminent
|
||||
exp_alerts:
|
||||
- exp_labels: {job: shared-postgresql-sql, instance: 'p:9187', member: pg-pve1, env: prod, severity: critical, service: shared-postgresql}
|
||||
exp_annotations:
|
||||
summary: 共享 PG prod pg-pve1 XID/MXID 接近回卷停写，需立即 VACUUM FREEZE
|
||||
- name: 回卷 warning
|
||||
interval: 1m
|
||||
input_series:
|
||||
- series: pg_database_wraparound_age_datfrozenxid_seconds{job="shared-postgresql-sql",instance="p:9187",member="pg-pve1",env="prod",datname="app"}
|
||||
values: '1200000000x15'
|
||||
alert_rule_test:
|
||||
- eval_time: 12m
|
||||
alertname: SharedPostgresWraparoundRisk
|
||||
exp_alerts:
|
||||
- exp_labels: {job: shared-postgresql-sql, instance: 'p:9187', member: pg-pve1, env: prod, severity: warning, service: shared-postgresql}
|
||||
exp_annotations:
|
||||
summary: 共享 PG prod pg-pve1 XID/MXID 年龄超过 10 亿
|
||||
- name: 只计客户端连接
|
||||
interval: 1m
|
||||
input_series:
|
||||
- series: pg_stat_activity_count{job="shared-postgresql-sql",instance="p:9187",member="pg-pve1",env="prod",datname="app",state="active",usename="a",application_name="x",backend_type="client backend",wait_event_type="",wait_event=""}
|
||||
values: '50x10'
|
||||
- series: pg_stat_activity_count{job="shared-postgresql-sql",instance="p:9187",member="pg-pve1",env="prod",datname="app",state="idle",usename="b",application_name="y",backend_type="client backend",wait_event_type="",wait_event=""}
|
||||
values: '20x10'
|
||||
- series: pg_stat_activity_count{job="shared-postgresql-sql",instance="p:9187",member="pg-pve1",env="prod",datname="",state="",usename="",application_name="",backend_type="autovacuum worker",wait_event_type="",wait_event=""}
|
||||
values: '30x10'
|
||||
- series: pg_settings_max_connections{job="shared-postgresql-sql",instance="p:9187",member="pg-pve1",env="prod"}
|
||||
values: '80x10'
|
||||
alert_rule_test:
|
||||
- eval_time: 8m
|
||||
alertname: SharedPostgresConnectionsHigh
|
||||
exp_alerts:
|
||||
- exp_labels: {job: shared-postgresql-sql, instance: 'p:9187', member: pg-pve1, env: prod, severity: warning, service: shared-postgresql}
|
||||
exp_annotations:
|
||||
summary: 共享 PG prod pg-pve1 客户端连接超过 max_connections 的 80%
|
||||
- name: 后台进程不计入连接使用率
|
||||
interval: 1m
|
||||
input_series:
|
||||
- series: pg_stat_activity_count{job="shared-postgresql-sql",instance="p:9187",env="prod",datname="app",state="active",usename="a",application_name="x",backend_type="client backend",wait_event_type="",wait_event=""}
|
||||
values: '40x10'
|
||||
- series: pg_stat_activity_count{job="shared-postgresql-sql",instance="p:9187",env="prod",datname="",state="",usename="",application_name="",backend_type="autovacuum worker",wait_event_type="",wait_event=""}
|
||||
values: '40x10'
|
||||
- series: pg_settings_max_connections{job="shared-postgresql-sql",instance="p:9187",env="prod"}
|
||||
values: '80x10'
|
||||
alert_rule_test:
|
||||
- eval_time: 8m
|
||||
alertname: SharedPostgresConnectionsHigh
|
||||
exp_alerts: []
|
||||
- name: 死锁计数增长告警
|
||||
interval: 1m
|
||||
input_series:
|
||||
- series: pg_stat_database_deadlocks{job="shared-postgresql-sql",instance="p:9187",member="pg-pve1",env="prod",datname="app"}
|
||||
values: '0 0 0 1 1 1'
|
||||
alert_rule_test:
|
||||
- eval_time: 5m
|
||||
alertname: SharedPostgresDeadlocks
|
||||
exp_alerts:
|
||||
- exp_labels: {job: shared-postgresql-sql, instance: 'p:9187', member: pg-pve1, env: prod, severity: warning, service: shared-postgresql}
|
||||
exp_annotations:
|
||||
summary: 共享 PG prod pg-pve1 15 分钟内出现死锁
|
||||
- name: 空闲事务超过 10 分钟
|
||||
interval: 1m
|
||||
input_series:
|
||||
- series: pg_stat_activity_max_tx_duration{job="shared-postgresql-sql",instance="p:9187",member="pg-pve1",env="prod",datname="app",state="idle in transaction",usename="a",application_name="x",backend_type="client backend",wait_event_type="Client",wait_event="ClientRead"}
|
||||
values: '700x5'
|
||||
alert_rule_test:
|
||||
- eval_time: 3m
|
||||
alertname: SharedPostgresIdleInTransaction
|
||||
exp_alerts:
|
||||
- exp_labels: {job: shared-postgresql-sql, instance: 'p:9187', member: pg-pve1, env: prod, severity: warning, service: shared-postgresql}
|
||||
exp_annotations:
|
||||
summary: 共享 PG prod pg-pve1 有事务空闲未提交超过 10 分钟
|
||||
@@ -16,6 +16,10 @@ spec:
|
||||
group_interval: 5m
|
||||
repeat_interval: 4h
|
||||
routes:
|
||||
# dev 必须排第一且不 continue：否则 dev 告警会被后面的 severity 路由送进生产频道。
|
||||
- receiver: telegram-dev
|
||||
matchers:
|
||||
- env="dev"
|
||||
- receiver: telegram
|
||||
matchers:
|
||||
- severity="critical"
|
||||
@@ -37,6 +41,11 @@ spec:
|
||||
- bot_token_file: /etc/vm/secrets/alertmanager-telegram/telegram_bot_token
|
||||
chat_id: -1003956377923
|
||||
send_resolved: true
|
||||
- name: telegram-dev
|
||||
telegram_configs:
|
||||
- bot_token_file: /etc/vm/secrets/alertmanager-telegram/telegram_bot_token
|
||||
chat_id: -1003651477106
|
||||
send_resolved: true
|
||||
resources:
|
||||
requests:
|
||||
cpu: 25m
|
||||
|
||||
Reference in New Issue
Block a user