feat: 纳管共享 etcd 与 k3s 外 PostgreSQL 高可用及备份
This commit is contained in:
@@ -21,3 +21,7 @@ resources:
|
||||
- rules/kubernetes-health.yaml
|
||||
- rules/host-health.yaml
|
||||
- rules/monitoring-delivery.yaml
|
||||
- rules/shared-etcd.yaml
|
||||
- scrapes/shared-etcd.yaml
|
||||
- rules/shared-postgresql.yaml
|
||||
- scrapes/shared-postgresql.yaml
|
||||
|
||||
@@ -0,0 +1,61 @@
|
||||
# 参考 Pigsty v4.5.0 etcd 指标与阈值,适配原生 up 和现有标签;无 Pigsty UI 依赖。
|
||||
apiVersion: operator.victoriametrics.com/v1beta1
|
||||
kind: VMRule
|
||||
metadata:
|
||||
name: shared-etcd
|
||||
namespace: monitoring
|
||||
spec:
|
||||
groups:
|
||||
- name: shared-etcd
|
||||
rules:
|
||||
- alert: SharedEtcdMemberUnavailable
|
||||
expr: up{job="shared-etcd"} == 0
|
||||
for: 2m
|
||||
labels:
|
||||
severity: warning
|
||||
service: shared-etcd
|
||||
annotations:
|
||||
summary: 共享 etcd 成员 {{ $labels.member }} 无法采集
|
||||
- alert: SharedEtcdMonitoringQuorumUnavailable
|
||||
expr: (sum(up{job="shared-etcd"}) or vector(0)) < 2
|
||||
for: 2m
|
||||
labels:
|
||||
severity: critical
|
||||
service: shared-etcd
|
||||
annotations:
|
||||
summary: 共享 etcd 不足两个成员可采集,请用 endpoint health 核实 quorum
|
||||
- alert: SharedEtcdNoLeader
|
||||
expr: etcd_server_has_leader{job="shared-etcd"} == 0
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
service: shared-etcd
|
||||
annotations:
|
||||
summary: 共享 etcd 成员 {{ $labels.member }} 无法确认 leader
|
||||
- alert: SharedEtcdQuotaHigh
|
||||
expr: etcd_mvcc_db_total_size_in_bytes{job="shared-etcd"} / etcd_server_quota_backend_bytes{job="shared-etcd"}
|
||||
> 0.8
|
||||
for: 10m
|
||||
labels:
|
||||
severity: warning
|
||||
service: shared-etcd
|
||||
annotations:
|
||||
summary: 共享 etcd 后端容量超过 80%
|
||||
- alert: SharedEtcdSlowFsync
|
||||
expr: histogram_quantile(0.95, rate(etcd_disk_wal_fsync_duration_seconds_bucket{job="shared-etcd"}[5m]))
|
||||
> 0.05
|
||||
for: 10m
|
||||
labels:
|
||||
severity: warning
|
||||
service: shared-etcd
|
||||
annotations:
|
||||
summary: 共享 etcd {{ $labels.member }} WAL fsync p95 超过 50ms
|
||||
- alert: SharedEtcdFrequentElections
|
||||
expr: increase(etcd_server_leader_changes_seen_total{job="shared-etcd"}[15m])
|
||||
> 3
|
||||
for: 5m
|
||||
labels:
|
||||
severity: warning
|
||||
service: shared-etcd
|
||||
annotations:
|
||||
summary: 共享 etcd 15 分钟内多次选举
|
||||
@@ -0,0 +1,40 @@
|
||||
# 使用 Patroni 原生指标及现有 Alertmanager;不新增监控栈。
|
||||
apiVersion: operator.victoriametrics.com/v1beta1
|
||||
kind: VMRule
|
||||
metadata:
|
||||
name: shared-postgresql
|
||||
namespace: monitoring
|
||||
spec:
|
||||
groups:
|
||||
- name: shared-postgresql
|
||||
rules:
|
||||
- alert: SharedPostgresMemberUnavailable
|
||||
expr: up{job="shared-postgresql-patroni"} == 0
|
||||
for: 2m
|
||||
labels: {severity: warning, service: shared-postgresql}
|
||||
annotations:
|
||||
summary: '共享 PG 成员 {{ $labels.member }} 无法采集'
|
||||
- alert: SharedPostgresNoObservedPrimary
|
||||
expr: (sum(patroni_primary{job="shared-postgresql-patroni"}) or vector(0)) == 0
|
||||
for: 1m
|
||||
labels: {severity: critical, service: shared-postgresql}
|
||||
annotations:
|
||||
summary: 共享 PG 监控未观察到 primary,请检查实际角色和入口
|
||||
- alert: SharedPostgresMultiplePrimaries
|
||||
expr: sum(patroni_primary{job="shared-postgresql-patroni"}) > 1
|
||||
for: 30s
|
||||
labels: {severity: critical, service: shared-postgresql}
|
||||
annotations:
|
||||
summary: 共享 PG 监控观察到多个 primary,需立即核实
|
||||
- alert: SharedPostgresReplicaNotStreaming
|
||||
expr: patroni_replica{job="shared-postgresql-patroni"} == 1 and patroni_postgres_streaming{job="shared-postgresql-patroni"} == 0
|
||||
for: 2m
|
||||
labels: {severity: warning, service: shared-postgresql}
|
||||
annotations:
|
||||
summary: '共享 PG replica {{ $labels.member }} 没有流复制'
|
||||
- alert: SharedPostgresProcessDown
|
||||
expr: patroni_postgres_running{job="shared-postgresql-patroni"} == 0
|
||||
for: 1m
|
||||
labels: {severity: critical, service: shared-postgresql}
|
||||
annotations:
|
||||
summary: '共享 PG 成员 {{ $labels.member }} 数据库进程未运行'
|
||||
@@ -0,0 +1,26 @@
|
||||
apiVersion: operator.victoriametrics.com/v1beta1
|
||||
kind: VMStaticScrape
|
||||
metadata:
|
||||
name: shared-etcd
|
||||
namespace: monitoring
|
||||
spec:
|
||||
jobName: shared-etcd
|
||||
targetEndpoints:
|
||||
- targets:
|
||||
- 192.168.10.127:2381
|
||||
labels:
|
||||
cluster: homelab-etcd
|
||||
member: etcd-laptop
|
||||
interval: 30s
|
||||
- targets:
|
||||
- 10.60.0.20:2381
|
||||
labels:
|
||||
cluster: homelab-etcd
|
||||
member: etcd-pve1
|
||||
interval: 30s
|
||||
- targets:
|
||||
- 10.60.0.21:2381
|
||||
labels:
|
||||
cluster: homelab-etcd
|
||||
member: etcd-pve2
|
||||
interval: 30s
|
||||
@@ -0,0 +1,14 @@
|
||||
apiVersion: operator.victoriametrics.com/v1beta1
|
||||
kind: VMStaticScrape
|
||||
metadata:
|
||||
name: shared-postgresql
|
||||
namespace: monitoring
|
||||
spec:
|
||||
jobName: shared-postgresql-patroni
|
||||
targetEndpoints:
|
||||
- targets: [192.168.10.127:8008]
|
||||
labels: {cluster: homelab-pg-prod, member: pg-laptop}
|
||||
interval: 15s
|
||||
- targets: [10.60.0.20:8008]
|
||||
labels: {cluster: homelab-pg-prod, member: pg-pve1}
|
||||
interval: 15s
|
||||
Reference in New Issue
Block a user