补齐证书、秘密同步、GitOps 与数据服务监控

2026-09-25 19:35:38 +00:00
parent 18ded1e4a8
commit 9a800f55fc
9 changed files with 791 additions and 3 deletions
@@ -0,0 +1,148 @@
+apiVersion: monitoring.coreos.com/v1
+kind: PrometheusRule
+metadata:
+  name: data-services
+  namespace: monitoring
+spec:
+  groups:
+  - name: data-services
+    interval: 30s
+    rules:
+    - alert: DataServiceMetricsUnavailable
+      expr: up{job="cnpg"} == 0 or absent(up{job="cnpg"})
+      for: 5m
+      labels:
+        severity: critical
+        job: cnpg
+      annotations:
+        summary: 数据服务 {{ $labels.job }} 指标不可用
+        description: 检查监控对象、Pod 与 exporter；目标消失同样触发。
+        runbook_url: https://git.ddupan.top/panxiao81/homelab-wiki/src/branch/main/guides/monitoring-data-services.md
+    - alert: DataServiceMetricsUnavailable
+      expr: up{job="seaweedfs-master"} == 0 or absent(up{job="seaweedfs-master"})
+      for: 5m
+      labels:
+        severity: critical
+        job: seaweedfs-master
+      annotations:
+        summary: 数据服务 {{ $labels.job }} 指标不可用
+        description: 检查监控对象、Pod 与 exporter；目标消失同样触发。
+        runbook_url: https://git.ddupan.top/panxiao81/homelab-wiki/src/branch/main/guides/monitoring-data-services.md
+    - alert: DataServiceMetricsUnavailable
+      expr: up{job="seaweedfs-volume"} == 0 or absent(up{job="seaweedfs-volume"})
+      for: 5m
+      labels:
+        severity: critical
+        job: seaweedfs-volume
+      annotations:
+        summary: 数据服务 {{ $labels.job }} 指标不可用
+        description: 检查监控对象、Pod 与 exporter；目标消失同样触发。
+        runbook_url: https://git.ddupan.top/panxiao81/homelab-wiki/src/branch/main/guides/monitoring-data-services.md
+    - alert: DataServiceMetricsUnavailable
+      expr: up{job="seaweedfs-filer"} == 0 or absent(up{job="seaweedfs-filer"})
+      for: 5m
+      labels:
+        severity: critical
+        job: seaweedfs-filer
+      annotations:
+        summary: 数据服务 {{ $labels.job }} 指标不可用
+        description: 检查监控对象、Pod 与 exporter；目标消失同样触发。
+        runbook_url: https://git.ddupan.top/panxiao81/homelab-wiki/src/branch/main/guides/monitoring-data-services.md
+    - alert: CnpgPostgresDown
+      expr: cnpg_collector_up{job="cnpg"} == 0
+      for: 2m
+      labels:
+        severity: critical
+      annotations:
+        summary: CNPG {{ $labels.pod }} PostgreSQL 不可用
+        description: 检查数据库 Pod 状态、实例日志和存储；exporter up 不代表数据库可用。
+        runbook_url: https://git.ddupan.top/panxiao81/homelab-wiki/src/branch/main/guides/monitoring-data-services.md
+    - alert: CnpgMetricsCollectionFailed
+      expr: cnpg_collector_last_collection_error{job="cnpg"} == 1
+      for: 5m
+      labels:
+        severity: warning
+      annotations:
+        summary: CNPG {{ $labels.pod }} SQL 指标采集失败
+        description: 检查内置 exporter 日志、数据库连接与查询兼容性。
+        runbook_url: https://git.ddupan.top/panxiao81/homelab-wiki/src/branch/main/guides/monitoring-data-services.md
+    - alert: CnpgConnectionsHigh
+      expr: sum by (job,namespace,pod) (cnpg_backends_total{job="cnpg"}) / on(job,namespace,pod)
+        max by(job,namespace,pod) (cnpg_pg_settings_setting{job="cnpg",name="max_connections"})
+        > 0.8
+      for: 10m
+      labels:
+        severity: warning
+      annotations:
+        summary: CNPG {{ $labels.pod }} 连接使用率超过 80%
+        description: 检查连接池、空闲连接和连接泄漏，不直接提高 max_connections。
+        runbook_url: https://git.ddupan.top/panxiao81/homelab-wiki/src/branch/main/guides/monitoring-data-services.md
+    - alert: CnpgLongTransaction
+      expr: cnpg_backends_max_tx_duration_seconds{job="cnpg",state!="idle"} > 300
+      for: 5m
+      labels:
+        severity: warning
+      annotations:
+        summary: CNPG {{ $labels.pod }} 存在长事务
+        description: 事务持续超过五分钟并维持五分钟，检查阻塞与 idle in transaction；先确认业务再处理。
+        runbook_url: https://git.ddupan.top/panxiao81/homelab-wiki/src/branch/main/guides/monitoring-data-services.md
+    - alert: NatsJetStreamStorageHigh
+      expr: nats_varz_jetstream_stats_storage{job="nats/nats"} / nats_varz_jetstream_config_max_storage{job="nats/nats"}
+        > 0.8
+      for: 10m
+      labels:
+        severity: warning
+      annotations:
+        summary: NATS JetStream 文件容量超过 80%
+        description: 检查服务端配额与消息保留策略；该指标不代表单个 Account 或 stream 配额。
+        runbook_url: https://git.ddupan.top/panxiao81/homelab-wiki/src/branch/main/guides/monitoring-data-services.md
+    - alert: NatsJetStreamMemoryHigh
+      expr: nats_varz_jetstream_stats_memory{job="nats/nats"} / nats_varz_jetstream_config_max_memory{job="nats/nats"}
+        > 0.8
+      for: 10m
+      labels:
+        severity: warning
+      annotations:
+        summary: NATS JetStream 内存容量超过 80%
+        description: 检查 memory store 消息保留与服务端内存配额。
+        runbook_url: https://git.ddupan.top/panxiao81/homelab-wiki/src/branch/main/guides/monitoring-data-services.md
+    - alert: NatsSlowConsumers
+      expr: increase(nats_varz_slow_consumers{job="nats/nats"}[10m]) > 0
+      for: 5m
+      labels:
+        severity: warning
+      annotations:
+        summary: NATS 检测到慢消费者
+        description: 检查消费者处理速率与网络；此规则不能代替 JetStream consumer 积压监控。
+        runbook_url: https://git.ddupan.top/panxiao81/homelab-wiki/src/branch/main/guides/monitoring-data-services.md
+    - alert: SeaweedVolumeDiskLow
+      expr: SeaweedFS_volumeServer_resource{job="seaweedfs-volume",type="avail"} /
+        ignoring(type) SeaweedFS_volumeServer_resource{job="seaweedfs-volume",type="all"}
+        < 0.1
+      for: 10m
+      labels:
+        severity: warning
+      annotations:
+        summary: SeaweedFS {{ $labels.name }} 可用空间不足 10%
+        description: 检查 volume 文件系统容量、保留策略与底层 ZFS；不要直接删除卷文件。
+        runbook_url: https://git.ddupan.top/panxiao81/homelab-wiki/src/branch/main/guides/monitoring-data-services.md
+    - alert: SeaweedVolumeDiskReadOnly
+      expr: SeaweedFS_volumeServer_read_only_volumes{job="seaweedfs-volume",type="isDiskSpaceLow"}
+        > 0
+      for: 5m
+      labels:
+        severity: critical
+      annotations:
+        summary: SeaweedFS 因磁盘空间不足限制卷写入
+        description: 检查磁盘容量和 volume 日志；主动只读或满卷切换不按此故障处理。
+        runbook_url: https://git.ddupan.top/panxiao81/homelab-wiki/src/branch/main/guides/monitoring-data-services.md
+    - alert: SeaweedVolumeWriteFailures
+      expr: increase(SeaweedFS_volumeServer_file_write_failures{job="seaweedfs-volume"}[5m])
+        > 0
+      for: 2m
+      labels:
+        severity: warning
+      annotations:
+        summary: SeaweedFS 卷写入失败
+        description: 检查 volume 日志、磁盘状态及上游请求；不代表已经验证 S3 端到端可用。
+        runbook_url: https://git.ddupan.top/panxiao81/homelab-wiki/src/branch/main/guides/monitoring-data-services.md
@@ -0,0 +1,161 @@
+apiVersion: monitoring.coreos.com/v1
+kind: PrometheusRule
+metadata:
+  name: platform-controllers
+  namespace: monitoring
+spec:
+  groups:
+  - name: platform-controllers
+    interval: 30s
+    rules:
+    - alert: PlatformControllerMetricsUnavailable
+      expr: up{job="cert-manager"} == 0 or absent(up{job="cert-manager"})
+      for: 5m
+      labels:
+        severity: warning
+        job: cert-manager
+      annotations:
+        summary: 控制器 {{ $labels.job }} 指标不可用
+        description: 检查对应 Pod、监控 CR 转换与 targets；目标消失同样触发。
+        runbook_url: https://git.ddupan.top/panxiao81/homelab-wiki/src/branch/main/guides/monitoring-controllers.md
+    - alert: PlatformControllerMetricsUnavailable
+      expr: up{job="external-secrets"} == 0 or absent(up{job="external-secrets"})
+      for: 5m
+      labels:
+        severity: warning
+        job: external-secrets
+      annotations:
+        summary: 控制器 {{ $labels.job }} 指标不可用
+        description: 检查对应 Pod、监控 CR 转换与 targets；目标消失同样触发。
+        runbook_url: https://git.ddupan.top/panxiao81/homelab-wiki/src/branch/main/guides/monitoring-controllers.md
+    - alert: PlatformControllerMetricsUnavailable
+      expr: up{job="external-secrets-cert-controller"} == 0 or absent(up{job="external-secrets-cert-controller"})
+      for: 5m
+      labels:
+        severity: warning
+        job: external-secrets-cert-controller
+      annotations:
+        summary: 控制器 {{ $labels.job }} 指标不可用
+        description: 检查对应 Pod、监控 CR 转换与 targets；目标消失同样触发。
+        runbook_url: https://git.ddupan.top/panxiao81/homelab-wiki/src/branch/main/guides/monitoring-controllers.md
+    - alert: PlatformControllerMetricsUnavailable
+      expr: up{job="external-secrets-webhook"} == 0 or absent(up{job="external-secrets-webhook"})
+      for: 5m
+      labels:
+        severity: warning
+        job: external-secrets-webhook
+      annotations:
+        summary: 控制器 {{ $labels.job }} 指标不可用
+        description: 检查对应 Pod、监控 CR 转换与 targets；目标消失同样触发。
+        runbook_url: https://git.ddupan.top/panxiao81/homelab-wiki/src/branch/main/guides/monitoring-controllers.md
+    - alert: PlatformControllerMetricsUnavailable
+      expr: up{job="helm-controller"} == 0 or absent(up{job="helm-controller"})
+      for: 5m
+      labels:
+        severity: warning
+        job: helm-controller
+      annotations:
+        summary: 控制器 {{ $labels.job }} 指标不可用
+        description: 检查对应 Pod、监控 CR 转换与 targets；目标消失同样触发。
+        runbook_url: https://git.ddupan.top/panxiao81/homelab-wiki/src/branch/main/guides/monitoring-controllers.md
+    - alert: PlatformControllerMetricsUnavailable
+      expr: up{job="kustomize-controller"} == 0 or absent(up{job="kustomize-controller"})
+      for: 5m
+      labels:
+        severity: warning
+        job: kustomize-controller
+      annotations:
+        summary: 控制器 {{ $labels.job }} 指标不可用
+        description: 检查对应 Pod、监控 CR 转换与 targets；目标消失同样触发。
+        runbook_url: https://git.ddupan.top/panxiao81/homelab-wiki/src/branch/main/guides/monitoring-controllers.md
+    - alert: PlatformControllerMetricsUnavailable
+      expr: up{job="source-controller"} == 0 or absent(up{job="source-controller"})
+      for: 5m
+      labels:
+        severity: warning
+        job: source-controller
+      annotations:
+        summary: 控制器 {{ $labels.job }} 指标不可用
+        description: 检查对应 Pod、监控 CR 转换与 targets；目标消失同样触发。
+        runbook_url: https://git.ddupan.top/panxiao81/homelab-wiki/src/branch/main/guides/monitoring-controllers.md
+    - alert: PlatformControllerMetricsUnavailable
+      expr: up{job="notification-controller"} == 0 or absent(up{job="notification-controller"})
+      for: 5m
+      labels:
+        severity: warning
+        job: notification-controller
+      annotations:
+        summary: 控制器 {{ $labels.job }} 指标不可用
+        description: 检查对应 Pod、监控 CR 转换与 targets；目标消失同样触发。
+        runbook_url: https://git.ddupan.top/panxiao81/homelab-wiki/src/branch/main/guides/monitoring-controllers.md
+    - alert: CertificateNotReady
+      expr: certmanager_certificate_ready_status{job="cert-manager",condition="True"}
+        == 0
+      for: 15m
+      labels:
+        severity: warning
+      annotations:
+        summary: 证书 {{ $labels.namespace }}/{{ $labels.name }} 未就绪
+        description: 检查 Certificate、CertificateRequest、Issuer、Order 与 Challenge；允许短暂签发过程。
+        runbook_url: https://git.ddupan.top/panxiao81/homelab-wiki/src/branch/main/guides/monitoring-controllers.md
+    - alert: CertificateExpiringSoon
+      expr: (certmanager_certificate_expiration_timestamp_seconds{job="cert-manager"}
+        - time() < 7 * 86400) and (certmanager_certificate_expiration_timestamp_seconds{job="cert-manager"}
+        - time() >= 86400)
+      for: 15m
+      labels:
+        severity: warning
+      annotations:
+        summary: 证书 {{ $labels.namespace }}/{{ $labels.name }} 将在七天内到期
+        description: 检查自动续期链路及实际入口证书；一天内到期由 critical 规则接替。
+        runbook_url: https://git.ddupan.top/panxiao81/homelab-wiki/src/branch/main/guides/monitoring-controllers.md
+    - alert: CertificateExpiryCritical
+      expr: certmanager_certificate_expiration_timestamp_seconds{job="cert-manager"}
+        - time() < 86400
+      for: 5m
+      labels:
+        severity: critical
+      annotations:
+        summary: 证书 {{ $labels.namespace }}/{{ $labels.name }} 即将到期或已过期
+        description: 不足一天或已经过期；立即检查续期与入口证书。
+        runbook_url: https://git.ddupan.top/panxiao81/homelab-wiki/src/branch/main/guides/monitoring-controllers.md
+    - alert: ExternalSecretNotReady
+      expr: externalsecret_status_condition{job="external-secrets",condition="Ready",status="True"}
+        == 0
+      for: 10m
+      labels:
+        severity: warning
+      annotations:
+        summary: Secret 同步 {{ $labels.namespace }}/{{ $labels.name }} 未就绪
+        description: 检查 ExternalSecret 状态及 provider 权限、网络；不要输出 Secret 内容。
+        runbook_url: https://git.ddupan.top/panxiao81/homelab-wiki/src/branch/main/guides/monitoring-controllers.md
+    - alert: ClusterSecretStoreNotReady
+      expr: clustersecretstore_status_condition{job="external-secrets",condition="Ready",status="True"}
+        == 0
+      for: 5m
+      labels:
+        severity: critical
+      annotations:
+        summary: ClusterSecretStore {{ $labels.name }} 未就绪
+        description: 检查 OpenBao 可达性与 ESO 身份鉴权；可能影响多个命名空间。
+        runbook_url: https://git.ddupan.top/panxiao81/homelab-wiki/src/branch/main/guides/monitoring-controllers.md
+    - alert: FluxResourceNotReady
+      expr: gotk_resource_info{job="kube-state-metrics",ready!="True",suspended!="true"}
+        == 1
+      for: 15m
+      labels:
+        severity: warning
+      annotations:
+        summary: Flux {{ $labels.customresource_kind }} {{ $labels.resource_namespace
+          }}/{{ $labels.name }} 未就绪
+        description: 检查对象 conditions、依赖和 controller 日志；主动 suspend 不触发，不自动解除暂停。
+        runbook_url: https://git.ddupan.top/panxiao81/homelab-wiki/src/branch/main/guides/monitoring-controllers.md
+    - alert: FluxStateMetricsMissing
+      expr: absent(gotk_resource_info{job="kube-state-metrics",customresource_kind="Kustomization",resource_namespace="flux-system",name="flux-system"})
+      for: 5m
+      labels:
+        severity: warning
+      annotations:
+        summary: Flux 对象状态指标缺失
+        description: 检查 kube-state-metrics 自定义资源配置、RBAC 与采集；不能把空结果当作健康。
+        runbook_url: https://git.ddupan.top/panxiao81/homelab-wiki/src/branch/main/guides/monitoring-controllers.md