feat: 纳管共享 etcd 与 k3s 外 PostgreSQL 高可用及备份

2026-09-25 19:34:48 +00:00
parent acd4b55722
commit 3a2fe5fa0c
92 changed files with 4211 additions and 0 deletions
@@ -0,0 +1,61 @@
+# 参考 Pigsty v4.5.0 etcd 指标与阈值，适配原生 up 和现有标签；无 Pigsty UI 依赖。
+apiVersion: operator.victoriametrics.com/v1beta1
+kind: VMRule
+metadata:
+  name: shared-etcd
+  namespace: monitoring
+spec:
+  groups:
+  - name: shared-etcd
+    rules:
+    - alert: SharedEtcdMemberUnavailable
+      expr: up{job="shared-etcd"} == 0
+      for: 2m
+      labels:
+        severity: warning
+        service: shared-etcd
+      annotations:
+        summary: 共享 etcd 成员 {{ $labels.member }} 无法采集
+    - alert: SharedEtcdMonitoringQuorumUnavailable
+      expr: (sum(up{job="shared-etcd"}) or vector(0)) < 2
+      for: 2m
+      labels:
+        severity: critical
+        service: shared-etcd
+      annotations:
+        summary: 共享 etcd 不足两个成员可采集，请用 endpoint health 核实 quorum
+    - alert: SharedEtcdNoLeader
+      expr: etcd_server_has_leader{job="shared-etcd"} == 0
+      for: 1m
+      labels:
+        severity: critical
+        service: shared-etcd
+      annotations:
+        summary: 共享 etcd 成员 {{ $labels.member }} 无法确认 leader
+    - alert: SharedEtcdQuotaHigh
+      expr: etcd_mvcc_db_total_size_in_bytes{job="shared-etcd"} / etcd_server_quota_backend_bytes{job="shared-etcd"}
+        > 0.8
+      for: 10m
+      labels:
+        severity: warning
+        service: shared-etcd
+      annotations:
+        summary: 共享 etcd 后端容量超过 80%
+    - alert: SharedEtcdSlowFsync
+      expr: histogram_quantile(0.95, rate(etcd_disk_wal_fsync_duration_seconds_bucket{job="shared-etcd"}[5m]))
+        > 0.05
+      for: 10m
+      labels:
+        severity: warning
+        service: shared-etcd
+      annotations:
+        summary: 共享 etcd {{ $labels.member }} WAL fsync p95 超过 50ms
+    - alert: SharedEtcdFrequentElections
+      expr: increase(etcd_server_leader_changes_seen_total{job="shared-etcd"}[15m])
+        > 3
+      for: 5m
+      labels:
+        severity: warning
+        service: shared-etcd
+      annotations:
+        summary: 共享 etcd 15 分钟内多次选举
@@ -0,0 +1,40 @@
+# 使用 Patroni 原生指标及现有 Alertmanager；不新增监控栈。
+apiVersion: operator.victoriametrics.com/v1beta1
+kind: VMRule
+metadata:
+  name: shared-postgresql
+  namespace: monitoring
+spec:
+  groups:
+    - name: shared-postgresql
+      rules:
+        - alert: SharedPostgresMemberUnavailable
+          expr: up{job="shared-postgresql-patroni"} == 0
+          for: 2m
+          labels: {severity: warning, service: shared-postgresql}
+          annotations:
+            summary: '共享 PG 成员 {{ $labels.member }} 无法采集'
+        - alert: SharedPostgresNoObservedPrimary
+          expr: (sum(patroni_primary{job="shared-postgresql-patroni"}) or vector(0)) == 0
+          for: 1m
+          labels: {severity: critical, service: shared-postgresql}
+          annotations:
+            summary: 共享 PG 监控未观察到 primary，请检查实际角色和入口
+        - alert: SharedPostgresMultiplePrimaries
+          expr: sum(patroni_primary{job="shared-postgresql-patroni"}) > 1
+          for: 30s
+          labels: {severity: critical, service: shared-postgresql}
+          annotations:
+            summary: 共享 PG 监控观察到多个 primary，需立即核实
+        - alert: SharedPostgresReplicaNotStreaming
+          expr: patroni_replica{job="shared-postgresql-patroni"} == 1 and patroni_postgres_streaming{job="shared-postgresql-patroni"} == 0
+          for: 2m
+          labels: {severity: warning, service: shared-postgresql}
+          annotations:
+            summary: '共享 PG replica {{ $labels.member }} 没有流复制'
+        - alert: SharedPostgresProcessDown
+          expr: patroni_postgres_running{job="shared-postgresql-patroni"} == 0
+          for: 1m
+          labels: {severity: critical, service: shared-postgresql}
+          annotations:
+            summary: '共享 PG 成员 {{ $labels.member }} 数据库进程未运行'