新增共享 PG 的 SQL 级指标与 dev 告警分流

- 数据库主机上按实例运行 prometheus-community/postgres_exporter 0.20.1，以实例 OS 用户走本地 socket，经 peer + ident 映射到仅有 pg_monitor 的 homelab_monitor 角色；无密码、不经 Bao。版本与官方 sha256sums.txt 同源。 - prod/dev 启用 pg_stat_statements；monitoring.yml 仅在未生效时 reload/重启（prod 走 Patroni pending restart），重复执行 changed=0。 - 告警以上游 postgres_mixin 为底改写为 PrometheusRule，阈值对齐 Patroni failover lag 与 max_slot_wal_keep_size；抓取目标带 env 标签。 - Alertmanager 新增 env="dev" 路由到独立 dev 频道，dev SQL 检查告警补 env。 Co-Authored-By: Claude Opus 5.5 <[email protected]>
2026-10-01 16:20:15 +00:00
parent cd9d461950
commit 0cd4a67e12
19 changed files with 551 additions and 5 deletions
@@ -19,6 +19,7 @@ pg_profiles:
    user: pgprod
    port: 5432
    api_port: 8008
+    exporter_port: 9187
    dns: pg-prod.ad.ddupan.top
    memory_high: 768M
    memory_max: 1G
@@ -28,6 +29,7 @@ pg_profiles:
    user: pgdev
    port: 5433
    dns: pg-dev.ad.ddupan.top
+    exporter_port: 9188
    memory_high: 256M
    memory_max: 384M
    shared_buffers: 32MB
@@ -38,3 +40,8 @@ pg_replication_addresses: ['192.168.10.127/32', '10.60.0.20/32']
 pg_repo_host: 10.60.0.21
 pg_repo_user: pgbackup
 pg_repo_path: /var/lib/homelab-pgbackrest
+# SQL 指标：exporter 以实例 OS 用户经本地 socket 连接，peer + ident 映射到只读监控角色，
+# 因此无需密码或 Bao；角色仅有 pg_monitor，不复用实例管理账号。
+pg_monitor_user: homelab_monitor
+# 升级只改版本；安装包与同版本官方 sha256sums.txt 一起下载校验，不写死摘要。
+pg_exporter_version: 0.20.1
@@ -0,0 +1,76 @@
+---
+# SQL 指标：激活监控所需的 HBA/ident 与 pg_stat_statements，创建只读监控角色并安装 exporter。
+# 不需要 Bao：模板先由 site.yml 写入；本 play 只做激活与验证。会重启实例（仅在预加载库未生效时），
+# 因此不放进 site.yml。重复执行应为 changed=0。
+- name: 激活实例监控配置
+  hosts: pg_hosts
+  become: true
+  gather_facts: false
+  serial: 1
+  any_errors_fatal: true
+  tasks:
+    - name: 逐实例激活
+      ansible.builtin.include_tasks: tasks/monitoring-activate.yml
+      loop: "{{ pg_instances }}"
+      loop_control:
+        loop_var: pg_instance
+
+- name: 重启 pending 的生产成员以加载 pg_stat_statements
+  hosts: pg-laptop
+  become: true
+  become_user: "{{ pg_profiles.prod.user }}"
+  gather_facts: false
+  vars:
+    pg_patronictl: [/opt/homelab-patroni/bin/patronictl, -c, "{{ pg_config_root }}/prod/patroni.yml"]
+  tasks:
+    - name: 读取集群成员状态
+      ansible.builtin.command:
+        argv: "{{ pg_patronictl + ['list', '-e', '-f', 'json'] }}"
+      register: pg_members
+      changed_when: false
+    # Patroni 在原位重启 primary，不触发 failover；写入会短暂中断，客户端需重试。
+    - name: 仅重启 pending restart 的成员
+      ansible.builtin.command:
+        argv: "{{ pg_patronictl + ['restart', 'pg-prod', '--pending', '--force'] }}"
+      when: pg_members.stdout | from_json | selectattr('Pending restart', 'equalto', '*') | list | length > 0
+      changed_when: true
+    - name: 等待两个成员恢复且无 pending
+      ansible.builtin.command:
+        argv: "{{ pg_patronictl + ['list', '-e', '-f', 'json'] }}"
+      register: pg_members_after
+      changed_when: false
+      retries: 30
+      delay: 4
+      until: >-
+        (pg_members_after.stdout | from_json | length) == 2
+        and (pg_members_after.stdout | from_json | selectattr('Pending restart', 'equalto', '*') | list | length) == 0
+        and (pg_members_after.stdout | from_json | selectattr('State', 'in', ['running', 'streaming']) | list | length) == 2
+
+- name: 创建监控角色并安装 exporter
+  hosts: pg_hosts
+  become: true
+  gather_facts: false
+  tasks:
+    - name: 下载 postgres_exporter 并校验同版本官方摘要
+      ansible.builtin.get_url:
+        url: "{{ pg_exporter_base }}/postgres_exporter-{{ pg_exporter_version }}.linux-amd64.tar.gz"
+        dest: /opt/homelab-postgres-exporter.tar.gz
+        checksum: "sha256:{{ pg_exporter_base }}/sha256sums.txt"
+        mode: '0644'
+      vars:
+        pg_exporter_base: https://github.com/prometheus-community/postgres_exporter/releases/download/v{{ pg_exporter_version }}
+      register: pg_exporter_download
+      retries: 5
+      delay: 10
+      until: pg_exporter_download is succeeded
+    - name: 解压 exporter（保留上游许可证）
+      ansible.builtin.unarchive:
+        src: /opt/homelab-postgres-exporter.tar.gz
+        dest: /opt
+        remote_src: true
+        creates: /opt/postgres_exporter-{{ pg_exporter_version }}.linux-amd64/postgres_exporter
+    - name: 逐实例配置
+      ansible.builtin.include_tasks: tasks/monitoring-exporter.yml
+      loop: "{{ pg_instances }}"
+      loop_control:
+        loop_var: pg_instance
@@ -71,7 +71,7 @@
    owner: root
    group: "{{ pg_profile.user }}"
    mode: '0640'
-  loop: "{{ ['patroni.yml', 'pgbackrest.conf'] if pg_instance == 'prod' else ['postgresql.conf', 'pg_hba.conf'] }}"
+  loop: "{{ ['patroni.yml', 'pgbackrest.conf'] if pg_instance == 'prod' else ['postgresql.conf', 'pg_hba.conf', 'pg_ident.conf'] }}"
  no_log: true
 - name: 安装实例专用服务（不自动启动或重启）
  ansible.builtin.template:
@@ -66,6 +66,8 @@ postgresql:
    unix_socket_directories: {{ pg_socket_dir }}
    unix_socket_permissions: '0700'
    password_encryption: scram-sha-256
+    # postmaster 级参数：变更后需由 monitoring.yml 执行 Patroni pending restart。
+    shared_preload_libraries: pg_stat_statements
    shared_buffers: {{ pg_profile.shared_buffers }}
    work_mem: 4MB
    maintenance_work_mem: 64MB
@@ -76,6 +78,7 @@ postgresql:
    log_statement: none
    log_min_error_statement: panic
  pg_hba:
+    - local all {{ pg_monitor_user }} peer map=monitor
    - local all {{ pg_profile.user }} peer
    - local replication {{ pg_profile.user }} peer
    - local replication replicator scram-sha-256
@@ -88,6 +91,9 @@ postgresql:
 {% endfor %}
    - host all all 0.0.0.0/0 reject
    - host all all ::/0 reject
+  # 只有实例 OS 用户（exporter 运行身份）可映射为监控角色。
+  pg_ident:
+    - monitor {{ pg_profile.user }} {{ pg_monitor_user }}
  remove_data_directory_on_rewind_failure: false
  remove_data_directory_on_diverged_timelines: false
 watchdog:
@@ -1,3 +1,4 @@
+local all {{ pg_monitor_user }} peer map=monitor
 local all {{ pg_profile.user }} peer
 {% for cidr in pg_client_cidrs %}
 hostssl all all {{ cidr }} scram-sha-256
@@ -0,0 +1,2 @@
+# 只有实例 OS 用户（exporter 运行身份）可映射为监控角色。
+monitor {{ pg_profile.user }} {{ pg_monitor_user }}
@@ -3,6 +3,7 @@ port = {{ pg_profile.port }}
 unix_socket_directories = '{{ pg_socket_dir }}'
 unix_socket_permissions = 0700
 hba_file = '{{ pg_config_dir }}/pg_hba.conf'
+ident_file = '{{ pg_config_dir }}/pg_ident.conf'
 password_encryption = 'scram-sha-256'
 shared_buffers = '{{ pg_profile.shared_buffers }}'
 max_connections = {{ pg_profile.max_connections }}
@@ -16,3 +17,5 @@ ssl_ca_file = '{{ pg_config_dir }}/ca.crt'
 archive_mode = off
 log_statement = none
 log_min_error_statement = panic
+# postmaster 级参数：变更后需由 monitoring.yml 重启 dev 实例。
+shared_preload_libraries = 'pg_stat_statements'
@@ -0,0 +1,70 @@
+---
+- name: 设置现有实例路径
+  ansible.builtin.set_fact:
+    pg_profile: "{{ pg_profiles[pg_instance] }}"
+    pg_config_dir: "{{ pg_config_root }}/{{ pg_instance }}"
+    pg_socket_dir: "/run/homelab-postgresql-{{ pg_instance }}"
+- name: 激活前必须存在运行中的受管服务
+  ansible.builtin.command:
+    argv: [systemctl, is-active, "homelab-postgresql-{{ pg_instance }}.service"]
+  changed_when: false
+# 生产：Patroni 只在自身 reload 时把本地配置写成数据目录里的 postgresql.conf/pg_ident.conf，
+# 所以文件视图能说明 Patroni 是否已接收新配置；预加载库是否已生效另看 current_setting。
+# dev：ident_file 是 postmaster 级参数，视图读取的是“当前生效”的那个文件，
+# 所以启动时仍指向旧路径就会读不到映射；两列都为真才算已加载。
+- name: 读取监控配置的激活状态
+  ansible.builtin.command:
+    argv:
+      - "{{ pg_bin_dir }}/psql"
+      - -X
+      - -At
+      - -h
+      - "{{ pg_socket_dir }}"
+      - -p
+      - "{{ pg_profile.port | string }}"
+      - -d
+      - postgres
+      - -c
+      - >-
+        SELECT
+        EXISTS (SELECT 1 FROM pg_ident_file_mappings WHERE map_name = 'monitor' AND error IS NULL)
+        AND EXISTS (SELECT 1 FROM pg_file_settings WHERE name = 'shared_preload_libraries'
+                    AND setting ~ 'pg_stat_statements' AND error IS NULL),
+        current_setting('shared_preload_libraries') ~ 'pg_stat_statements'
+  become: true
+  become_user: "{{ pg_profile.user }}"
+  changed_when: false
+  register: pg_monitor_state
+- name: 让 Patroni 接收新的本地配置（reload，不重启）
+  ansible.builtin.command:
+    argv: [systemctl, kill, --kill-whom=main, --signal=HUP, homelab-postgresql-prod.service]
+  when: pg_instance == 'prod' and pg_monitor_state.stdout.split('|')[0] != 't'
+  changed_when: true
+- name: 等待 Patroni 写出新配置
+  ansible.builtin.command:
+    argv: "{{ pg_monitor_state.cmd }}"
+  become: true
+  become_user: "{{ pg_profile.user }}"
+  when: pg_instance == 'prod' and pg_monitor_state.stdout.split('|')[0] != 't'
+  changed_when: false
+  register: pg_monitor_reloaded
+  retries: 10
+  delay: 3
+  until: pg_monitor_reloaded.stdout.split('|')[0] == 't'
+# dev 没有 Patroni；预加载库与 ident_file 都是 postmaster 级参数，只能重启。
+- name: 重启 dev 以加载监控配置
+  ansible.builtin.systemd_service:
+    name: homelab-postgresql-dev.service
+    state: restarted
+  when: pg_instance == 'dev' and pg_monitor_state.stdout != 't|t'
+- name: 检查 dev 重启后监控配置已生效
+  ansible.builtin.command:
+    argv: "{{ pg_monitor_state.cmd }}"
+  become: true
+  become_user: "{{ pg_profile.user }}"
+  when: pg_instance == 'dev'
+  changed_when: false
+  register: pg_monitor_dev
+  retries: 10
+  delay: 3
+  until: pg_monitor_dev.rc == 0 and pg_monitor_dev.stdout == 't|t'
@@ -0,0 +1,61 @@
+---
+- name: 设置现有实例路径
+  ansible.builtin.set_fact:
+    pg_profile: "{{ pg_profiles[pg_instance] }}"
+    pg_config_dir: "{{ pg_config_root }}/{{ pg_instance }}"
+    pg_parent_dir: "{{ pg_data_root }}/{{ pg_instance }}"
+    pg_socket_dir: "/run/homelab-postgresql-{{ pg_instance }}"
+    pg_psql: ["{{ pg_bin_dir }}/psql", -X, -At, -v, ON_ERROR_STOP=1, -h, "/run/homelab-postgresql-{{ pg_instance }}",
+              -p, "{{ pg_profiles[pg_instance].port | string }}", -d, postgres]
+# 角色与扩展只在可写实例上创建，standby 经复制获得。
+- name: 检查监控角色与扩展是否符合声明
+  ansible.builtin.command:
+    argv: "{{ pg_psql + ['-c', pg_monitor_check] }}"
+  vars:
+    pg_monitor_check: >-
+      SELECT pg_is_in_recovery(),
+      COALESCE((SELECT rolcanlogin AND NOT rolsuper AND NOT rolcreaterole AND NOT rolcreatedb
+                AND NOT rolreplication AND NOT rolbypassrls AND rolconnlimit = 3 AND rolpassword IS NULL
+                AND pg_has_role(oid, 'pg_monitor', 'MEMBER')
+                FROM pg_authid WHERE rolname = '{{ pg_monitor_user }}'), false)
+      AND EXISTS (SELECT 1 FROM pg_extension WHERE extname = 'pg_stat_statements')
+  become: true
+  become_user: "{{ pg_profile.user }}"
+  changed_when: false
+  register: pg_monitor_role
+- name: 创建或修正监控角色（无密码，仅能经本地 ident 映射登录）
+  ansible.builtin.command:
+    argv: "{{ pg_psql }}"
+    stdin: |
+      SELECT 'CREATE ROLE {{ pg_monitor_user }}'
+      WHERE NOT EXISTS (SELECT 1 FROM pg_roles WHERE rolname = '{{ pg_monitor_user }}') \gexec
+      ALTER ROLE {{ pg_monitor_user }} LOGIN NOSUPERUSER NOCREATEDB NOCREATEROLE NOREPLICATION NOBYPASSRLS
+        CONNECTION LIMIT 3 PASSWORD NULL;
+      GRANT pg_monitor TO {{ pg_monitor_user }};
+      CREATE EXTENSION IF NOT EXISTS pg_stat_statements;
+  become: true
+  become_user: "{{ pg_profile.user }}"
+  when: pg_monitor_role.stdout == 'f|f'
+  changed_when: true
+- name: 安装实例 exporter 服务
+  ansible.builtin.template:
+    src: postgres-exporter.service.j2
+    dest: "/etc/systemd/system/homelab-postgresql-{{ pg_instance }}-exporter.service"
+    mode: '0644'
+  register: pg_exporter_unit
+- name: 启用 exporter
+  ansible.builtin.systemd_service:
+    name: "homelab-postgresql-{{ pg_instance }}-exporter.service"
+    daemon_reload: "{{ pg_exporter_unit is changed }}"
+    enabled: true
+    state: "{{ 'restarted' if pg_exporter_unit is changed else 'started' }}"
+# 同时证明 HBA/ident 已加载、角色可登录、exporter 能查询。
+- name: 检查 exporter 已连上数据库
+  ansible.builtin.uri:
+    url: "http://{{ pg_address }}:{{ pg_profile.exporter_port }}/metrics"
+    return_content: true
+  register: pg_exporter_metrics
+  changed_when: false
+  retries: 10
+  delay: 3
+  until: pg_exporter_metrics.status == 200 and pg_exporter_metrics.content is search('(?m)^pg_up 1$')
@@ -0,0 +1,27 @@
+[Unit]
+Description=PostgreSQL {{ pg_instance }} SQL metrics
+After=homelab-postgresql-{{ pg_instance }}.service
+[Service]
+# 本地 socket 上 TLS 无意义，sslmode=disable 只是避免驱动默认尝试 TLS。
+# 以实例 OS 用户运行才能进入 0700 的 socket 目录；peer + ident 把它映射成只读监控角色。
+# 监控角色只约束 exporter 自己发出的查询；这不是隔离边界——同一 OS 用户仍可经 peer 以
+# superuser 登录，exporter 被攻破即等同实例属主。InaccessiblePaths 只是不让它直接读写
+# 数据目录、私钥与 pgpass。暴露面因此仅限内网监听地址。
+User={{ pg_profile.user }}
+Group={{ pg_profile.user }}
+Environment="DATA_SOURCE_NAME=host={{ pg_socket_dir }} port={{ pg_profile.port }} user={{ pg_monitor_user }} dbname=postgres sslmode=disable application_name=postgres_exporter"
+ExecStart=/opt/postgres_exporter-{{ pg_exporter_version }}.linux-amd64/postgres_exporter \
+  --web.listen-address={{ pg_address }}:{{ pg_profile.exporter_port }} \
+  --collector.database_wraparound --collector.long_running_transactions \
+  --collector.stat_checkpointer --collector.stat_wal_receiver --collector.stat_statements
+Restart=on-failure
+RestartSec=5
+InaccessiblePaths={{ pg_parent_dir }} {{ pg_config_dir }}
+NoNewPrivileges=true
+ProtectSystem=strict
+ProtectHome=true
+PrivateTmp=true
+PrivateDevices=true
+MemoryMax=64M
+[Install]
+WantedBy=multi-user.target