补齐证书、秘密同步、GitOps 与数据服务监控 #160
@@ -58,3 +58,197 @@ spec:
|
|||||||
honorLabels: true
|
honorLabels: true
|
||||||
selfMonitor:
|
selfMonitor:
|
||||||
enabled: true
|
enabled: true
|
||||||
|
rbac:
|
||||||
|
extraRules:
|
||||||
|
- apiGroups:
|
||||||
|
- kustomize.toolkit.fluxcd.io
|
||||||
|
resources:
|
||||||
|
- kustomizations
|
||||||
|
verbs:
|
||||||
|
- list
|
||||||
|
- watch
|
||||||
|
- apiGroups:
|
||||||
|
- helm.toolkit.fluxcd.io
|
||||||
|
resources:
|
||||||
|
- helmreleases
|
||||||
|
verbs:
|
||||||
|
- list
|
||||||
|
- watch
|
||||||
|
- apiGroups:
|
||||||
|
- source.toolkit.fluxcd.io
|
||||||
|
resources:
|
||||||
|
- gitrepositories
|
||||||
|
- helmrepositories
|
||||||
|
- helmcharts
|
||||||
|
- ocirepositories
|
||||||
|
verbs:
|
||||||
|
- list
|
||||||
|
- watch
|
||||||
|
customResourceState:
|
||||||
|
enabled: true
|
||||||
|
config:
|
||||||
|
kind: CustomResourceStateMetrics
|
||||||
|
spec:
|
||||||
|
resources:
|
||||||
|
- groupVersionKind:
|
||||||
|
group: kustomize.toolkit.fluxcd.io
|
||||||
|
version: v1
|
||||||
|
kind: Kustomization
|
||||||
|
metricNamePrefix: gotk
|
||||||
|
metrics:
|
||||||
|
- name: resource_info
|
||||||
|
help: The current state of a GitOps Toolkit resource.
|
||||||
|
each:
|
||||||
|
type: Info
|
||||||
|
info:
|
||||||
|
labelsFromPath:
|
||||||
|
name:
|
||||||
|
- metadata
|
||||||
|
- name
|
||||||
|
labelsFromPath:
|
||||||
|
resource_namespace:
|
||||||
|
- metadata
|
||||||
|
- namespace
|
||||||
|
suspended:
|
||||||
|
- spec
|
||||||
|
- suspend
|
||||||
|
ready:
|
||||||
|
- status
|
||||||
|
- conditions
|
||||||
|
- '[type=Ready]'
|
||||||
|
- status
|
||||||
|
- groupVersionKind:
|
||||||
|
group: helm.toolkit.fluxcd.io
|
||||||
|
version: v2
|
||||||
|
kind: HelmRelease
|
||||||
|
metricNamePrefix: gotk
|
||||||
|
metrics:
|
||||||
|
- name: resource_info
|
||||||
|
help: The current state of a GitOps Toolkit resource.
|
||||||
|
each:
|
||||||
|
type: Info
|
||||||
|
info:
|
||||||
|
labelsFromPath:
|
||||||
|
name:
|
||||||
|
- metadata
|
||||||
|
- name
|
||||||
|
labelsFromPath:
|
||||||
|
resource_namespace:
|
||||||
|
- metadata
|
||||||
|
- namespace
|
||||||
|
suspended:
|
||||||
|
- spec
|
||||||
|
- suspend
|
||||||
|
ready:
|
||||||
|
- status
|
||||||
|
- conditions
|
||||||
|
- '[type=Ready]'
|
||||||
|
- status
|
||||||
|
- groupVersionKind:
|
||||||
|
group: source.toolkit.fluxcd.io
|
||||||
|
version: v1
|
||||||
|
kind: GitRepository
|
||||||
|
metricNamePrefix: gotk
|
||||||
|
metrics:
|
||||||
|
- name: resource_info
|
||||||
|
help: The current state of a GitOps Toolkit resource.
|
||||||
|
each:
|
||||||
|
type: Info
|
||||||
|
info:
|
||||||
|
labelsFromPath:
|
||||||
|
name:
|
||||||
|
- metadata
|
||||||
|
- name
|
||||||
|
labelsFromPath:
|
||||||
|
resource_namespace:
|
||||||
|
- metadata
|
||||||
|
- namespace
|
||||||
|
suspended:
|
||||||
|
- spec
|
||||||
|
- suspend
|
||||||
|
ready:
|
||||||
|
- status
|
||||||
|
- conditions
|
||||||
|
- '[type=Ready]'
|
||||||
|
- status
|
||||||
|
- groupVersionKind:
|
||||||
|
group: source.toolkit.fluxcd.io
|
||||||
|
version: v1
|
||||||
|
kind: HelmRepository
|
||||||
|
metricNamePrefix: gotk
|
||||||
|
metrics:
|
||||||
|
- name: resource_info
|
||||||
|
help: The current state of a GitOps Toolkit resource.
|
||||||
|
each:
|
||||||
|
type: Info
|
||||||
|
info:
|
||||||
|
labelsFromPath:
|
||||||
|
name:
|
||||||
|
- metadata
|
||||||
|
- name
|
||||||
|
labelsFromPath:
|
||||||
|
resource_namespace:
|
||||||
|
- metadata
|
||||||
|
- namespace
|
||||||
|
suspended:
|
||||||
|
- spec
|
||||||
|
- suspend
|
||||||
|
ready:
|
||||||
|
- status
|
||||||
|
- conditions
|
||||||
|
- '[type=Ready]'
|
||||||
|
- status
|
||||||
|
- groupVersionKind:
|
||||||
|
group: source.toolkit.fluxcd.io
|
||||||
|
version: v1
|
||||||
|
kind: HelmChart
|
||||||
|
metricNamePrefix: gotk
|
||||||
|
metrics:
|
||||||
|
- name: resource_info
|
||||||
|
help: The current state of a GitOps Toolkit resource.
|
||||||
|
each:
|
||||||
|
type: Info
|
||||||
|
info:
|
||||||
|
labelsFromPath:
|
||||||
|
name:
|
||||||
|
- metadata
|
||||||
|
- name
|
||||||
|
labelsFromPath:
|
||||||
|
resource_namespace:
|
||||||
|
- metadata
|
||||||
|
- namespace
|
||||||
|
suspended:
|
||||||
|
- spec
|
||||||
|
- suspend
|
||||||
|
ready:
|
||||||
|
- status
|
||||||
|
- conditions
|
||||||
|
- '[type=Ready]'
|
||||||
|
- status
|
||||||
|
- groupVersionKind:
|
||||||
|
group: source.toolkit.fluxcd.io
|
||||||
|
version: v1
|
||||||
|
kind: OCIRepository
|
||||||
|
metricNamePrefix: gotk
|
||||||
|
metrics:
|
||||||
|
- name: resource_info
|
||||||
|
help: The current state of a GitOps Toolkit resource.
|
||||||
|
each:
|
||||||
|
type: Info
|
||||||
|
info:
|
||||||
|
labelsFromPath:
|
||||||
|
name:
|
||||||
|
- metadata
|
||||||
|
- name
|
||||||
|
labelsFromPath:
|
||||||
|
resource_namespace:
|
||||||
|
- metadata
|
||||||
|
- namespace
|
||||||
|
suspended:
|
||||||
|
- spec
|
||||||
|
- suspend
|
||||||
|
ready:
|
||||||
|
- status
|
||||||
|
- conditions
|
||||||
|
- '[type=Ready]'
|
||||||
|
- status
|
||||||
|
|||||||
@@ -21,3 +21,8 @@ resources:
|
|||||||
- rules/kubernetes-health.yaml
|
- rules/kubernetes-health.yaml
|
||||||
- rules/host-health.yaml
|
- rules/host-health.yaml
|
||||||
- rules/monitoring-delivery.yaml
|
- rules/monitoring-delivery.yaml
|
||||||
|
- scrapes/platform-controllers.yaml
|
||||||
|
- rules/platform-controllers.yaml
|
||||||
|
- scrapes/cnpg.yaml
|
||||||
|
- scrapes/seaweedfs.yaml
|
||||||
|
- rules/data-services.yaml
|
||||||
|
|||||||
@@ -0,0 +1,148 @@
|
|||||||
|
apiVersion: monitoring.coreos.com/v1
|
||||||
|
kind: PrometheusRule
|
||||||
|
metadata:
|
||||||
|
name: data-services
|
||||||
|
namespace: monitoring
|
||||||
|
spec:
|
||||||
|
groups:
|
||||||
|
- name: data-services
|
||||||
|
interval: 30s
|
||||||
|
rules:
|
||||||
|
- alert: DataServiceMetricsUnavailable
|
||||||
|
expr: up{job="cnpg"} == 0 or absent(up{job="cnpg"})
|
||||||
|
for: 5m
|
||||||
|
labels:
|
||||||
|
severity: critical
|
||||||
|
job: cnpg
|
||||||
|
annotations:
|
||||||
|
summary: 数据服务 {{ $labels.job }} 指标不可用
|
||||||
|
description: 检查监控对象、Pod 与 exporter；目标消失同样触发。
|
||||||
|
runbook_url: https://git.ddupan.top/panxiao81/homelab-wiki/src/branch/main/guides/monitoring-data-services.md
|
||||||
|
- alert: DataServiceMetricsUnavailable
|
||||||
|
expr: up{job="seaweedfs-master"} == 0 or absent(up{job="seaweedfs-master"})
|
||||||
|
for: 5m
|
||||||
|
labels:
|
||||||
|
severity: critical
|
||||||
|
job: seaweedfs-master
|
||||||
|
annotations:
|
||||||
|
summary: 数据服务 {{ $labels.job }} 指标不可用
|
||||||
|
description: 检查监控对象、Pod 与 exporter；目标消失同样触发。
|
||||||
|
runbook_url: https://git.ddupan.top/panxiao81/homelab-wiki/src/branch/main/guides/monitoring-data-services.md
|
||||||
|
- alert: DataServiceMetricsUnavailable
|
||||||
|
expr: up{job="seaweedfs-volume"} == 0 or absent(up{job="seaweedfs-volume"})
|
||||||
|
for: 5m
|
||||||
|
labels:
|
||||||
|
severity: critical
|
||||||
|
job: seaweedfs-volume
|
||||||
|
annotations:
|
||||||
|
summary: 数据服务 {{ $labels.job }} 指标不可用
|
||||||
|
description: 检查监控对象、Pod 与 exporter；目标消失同样触发。
|
||||||
|
runbook_url: https://git.ddupan.top/panxiao81/homelab-wiki/src/branch/main/guides/monitoring-data-services.md
|
||||||
|
- alert: DataServiceMetricsUnavailable
|
||||||
|
expr: up{job="seaweedfs-filer"} == 0 or absent(up{job="seaweedfs-filer"})
|
||||||
|
for: 5m
|
||||||
|
labels:
|
||||||
|
severity: critical
|
||||||
|
job: seaweedfs-filer
|
||||||
|
annotations:
|
||||||
|
summary: 数据服务 {{ $labels.job }} 指标不可用
|
||||||
|
description: 检查监控对象、Pod 与 exporter；目标消失同样触发。
|
||||||
|
runbook_url: https://git.ddupan.top/panxiao81/homelab-wiki/src/branch/main/guides/monitoring-data-services.md
|
||||||
|
- alert: CnpgPostgresDown
|
||||||
|
expr: cnpg_collector_up{job="cnpg"} == 0
|
||||||
|
for: 2m
|
||||||
|
labels:
|
||||||
|
severity: critical
|
||||||
|
annotations:
|
||||||
|
summary: CNPG {{ $labels.pod }} PostgreSQL 不可用
|
||||||
|
description: 检查数据库 Pod 状态、实例日志和存储；exporter up 不代表数据库可用。
|
||||||
|
runbook_url: https://git.ddupan.top/panxiao81/homelab-wiki/src/branch/main/guides/monitoring-data-services.md
|
||||||
|
- alert: CnpgMetricsCollectionFailed
|
||||||
|
expr: cnpg_collector_last_collection_error{job="cnpg"} == 1
|
||||||
|
for: 5m
|
||||||
|
labels:
|
||||||
|
severity: warning
|
||||||
|
annotations:
|
||||||
|
summary: CNPG {{ $labels.pod }} SQL 指标采集失败
|
||||||
|
description: 检查内置 exporter 日志、数据库连接与查询兼容性。
|
||||||
|
runbook_url: https://git.ddupan.top/panxiao81/homelab-wiki/src/branch/main/guides/monitoring-data-services.md
|
||||||
|
- alert: CnpgConnectionsHigh
|
||||||
|
expr: sum by (job,namespace,pod) (cnpg_backends_total{job="cnpg"}) / on(job,namespace,pod)
|
||||||
|
max by(job,namespace,pod) (cnpg_pg_settings_setting{job="cnpg",name="max_connections"})
|
||||||
|
> 0.8
|
||||||
|
for: 10m
|
||||||
|
labels:
|
||||||
|
severity: warning
|
||||||
|
annotations:
|
||||||
|
summary: CNPG {{ $labels.pod }} 连接使用率超过 80%
|
||||||
|
description: 检查连接池、空闲连接和连接泄漏，不直接提高 max_connections。
|
||||||
|
runbook_url: https://git.ddupan.top/panxiao81/homelab-wiki/src/branch/main/guides/monitoring-data-services.md
|
||||||
|
- alert: CnpgLongTransaction
|
||||||
|
expr: cnpg_backends_max_tx_duration_seconds{job="cnpg",state!="idle"} > 300
|
||||||
|
for: 5m
|
||||||
|
labels:
|
||||||
|
severity: warning
|
||||||
|
annotations:
|
||||||
|
summary: CNPG {{ $labels.pod }} 存在长事务
|
||||||
|
description: 事务持续超过五分钟并维持五分钟，检查阻塞与 idle in transaction；先确认业务再处理。
|
||||||
|
runbook_url: https://git.ddupan.top/panxiao81/homelab-wiki/src/branch/main/guides/monitoring-data-services.md
|
||||||
|
- alert: NatsJetStreamStorageHigh
|
||||||
|
expr: nats_varz_jetstream_stats_storage{job="nats/nats"} / nats_varz_jetstream_config_max_storage{job="nats/nats"}
|
||||||
|
> 0.8
|
||||||
|
for: 10m
|
||||||
|
labels:
|
||||||
|
severity: warning
|
||||||
|
annotations:
|
||||||
|
summary: NATS JetStream 文件容量超过 80%
|
||||||
|
description: 检查服务端配额与消息保留策略；该指标不代表单个 Account 或 stream 配额。
|
||||||
|
runbook_url: https://git.ddupan.top/panxiao81/homelab-wiki/src/branch/main/guides/monitoring-data-services.md
|
||||||
|
- alert: NatsJetStreamMemoryHigh
|
||||||
|
expr: nats_varz_jetstream_stats_memory{job="nats/nats"} / nats_varz_jetstream_config_max_memory{job="nats/nats"}
|
||||||
|
> 0.8
|
||||||
|
for: 10m
|
||||||
|
labels:
|
||||||
|
severity: warning
|
||||||
|
annotations:
|
||||||
|
summary: NATS JetStream 内存容量超过 80%
|
||||||
|
description: 检查 memory store 消息保留与服务端内存配额。
|
||||||
|
runbook_url: https://git.ddupan.top/panxiao81/homelab-wiki/src/branch/main/guides/monitoring-data-services.md
|
||||||
|
- alert: NatsSlowConsumers
|
||||||
|
expr: increase(nats_varz_slow_consumers{job="nats/nats"}[10m]) > 0
|
||||||
|
for: 5m
|
||||||
|
labels:
|
||||||
|
severity: warning
|
||||||
|
annotations:
|
||||||
|
summary: NATS 检测到慢消费者
|
||||||
|
description: 检查消费者处理速率与网络；此规则不能代替 JetStream consumer 积压监控。
|
||||||
|
runbook_url: https://git.ddupan.top/panxiao81/homelab-wiki/src/branch/main/guides/monitoring-data-services.md
|
||||||
|
- alert: SeaweedVolumeDiskLow
|
||||||
|
expr: SeaweedFS_volumeServer_resource{job="seaweedfs-volume",type="avail"} /
|
||||||
|
ignoring(type) SeaweedFS_volumeServer_resource{job="seaweedfs-volume",type="all"}
|
||||||
|
< 0.1
|
||||||
|
for: 10m
|
||||||
|
labels:
|
||||||
|
severity: warning
|
||||||
|
annotations:
|
||||||
|
summary: SeaweedFS {{ $labels.name }} 可用空间不足 10%
|
||||||
|
description: 检查 volume 文件系统容量、保留策略与底层 ZFS；不要直接删除卷文件。
|
||||||
|
runbook_url: https://git.ddupan.top/panxiao81/homelab-wiki/src/branch/main/guides/monitoring-data-services.md
|
||||||
|
- alert: SeaweedVolumeDiskReadOnly
|
||||||
|
expr: SeaweedFS_volumeServer_read_only_volumes{job="seaweedfs-volume",type="isDiskSpaceLow"}
|
||||||
|
> 0
|
||||||
|
for: 5m
|
||||||
|
labels:
|
||||||
|
severity: critical
|
||||||
|
annotations:
|
||||||
|
summary: SeaweedFS 因磁盘空间不足限制卷写入
|
||||||
|
description: 检查磁盘容量和 volume 日志；主动只读或满卷切换不按此故障处理。
|
||||||
|
runbook_url: https://git.ddupan.top/panxiao81/homelab-wiki/src/branch/main/guides/monitoring-data-services.md
|
||||||
|
- alert: SeaweedVolumeWriteFailures
|
||||||
|
expr: increase(SeaweedFS_volumeServer_file_write_failures{job="seaweedfs-volume"}[5m])
|
||||||
|
> 0
|
||||||
|
for: 2m
|
||||||
|
labels:
|
||||||
|
severity: warning
|
||||||
|
annotations:
|
||||||
|
summary: SeaweedFS 卷写入失败
|
||||||
|
description: 检查 volume 日志、磁盘状态及上游请求；不代表已经验证 S3 端到端可用。
|
||||||
|
runbook_url: https://git.ddupan.top/panxiao81/homelab-wiki/src/branch/main/guides/monitoring-data-services.md
|
||||||
@@ -0,0 +1,161 @@
|
|||||||
|
apiVersion: monitoring.coreos.com/v1
|
||||||
|
kind: PrometheusRule
|
||||||
|
metadata:
|
||||||
|
name: platform-controllers
|
||||||
|
namespace: monitoring
|
||||||
|
spec:
|
||||||
|
groups:
|
||||||
|
- name: platform-controllers
|
||||||
|
interval: 30s
|
||||||
|
rules:
|
||||||
|
- alert: PlatformControllerMetricsUnavailable
|
||||||
|
expr: up{job="cert-manager"} == 0 or absent(up{job="cert-manager"})
|
||||||
|
for: 5m
|
||||||
|
labels:
|
||||||
|
severity: warning
|
||||||
|
job: cert-manager
|
||||||
|
annotations:
|
||||||
|
summary: 控制器 {{ $labels.job }} 指标不可用
|
||||||
|
description: 检查对应 Pod、监控 CR 转换与 targets；目标消失同样触发。
|
||||||
|
runbook_url: https://git.ddupan.top/panxiao81/homelab-wiki/src/branch/main/guides/monitoring-controllers.md
|
||||||
|
- alert: PlatformControllerMetricsUnavailable
|
||||||
|
expr: up{job="external-secrets"} == 0 or absent(up{job="external-secrets"})
|
||||||
|
for: 5m
|
||||||
|
labels:
|
||||||
|
severity: warning
|
||||||
|
job: external-secrets
|
||||||
|
annotations:
|
||||||
|
summary: 控制器 {{ $labels.job }} 指标不可用
|
||||||
|
description: 检查对应 Pod、监控 CR 转换与 targets；目标消失同样触发。
|
||||||
|
runbook_url: https://git.ddupan.top/panxiao81/homelab-wiki/src/branch/main/guides/monitoring-controllers.md
|
||||||
|
- alert: PlatformControllerMetricsUnavailable
|
||||||
|
expr: up{job="external-secrets-cert-controller"} == 0 or absent(up{job="external-secrets-cert-controller"})
|
||||||
|
for: 5m
|
||||||
|
labels:
|
||||||
|
severity: warning
|
||||||
|
job: external-secrets-cert-controller
|
||||||
|
annotations:
|
||||||
|
summary: 控制器 {{ $labels.job }} 指标不可用
|
||||||
|
description: 检查对应 Pod、监控 CR 转换与 targets；目标消失同样触发。
|
||||||
|
runbook_url: https://git.ddupan.top/panxiao81/homelab-wiki/src/branch/main/guides/monitoring-controllers.md
|
||||||
|
- alert: PlatformControllerMetricsUnavailable
|
||||||
|
expr: up{job="external-secrets-webhook"} == 0 or absent(up{job="external-secrets-webhook"})
|
||||||
|
for: 5m
|
||||||
|
labels:
|
||||||
|
severity: warning
|
||||||
|
job: external-secrets-webhook
|
||||||
|
annotations:
|
||||||
|
summary: 控制器 {{ $labels.job }} 指标不可用
|
||||||
|
description: 检查对应 Pod、监控 CR 转换与 targets；目标消失同样触发。
|
||||||
|
runbook_url: https://git.ddupan.top/panxiao81/homelab-wiki/src/branch/main/guides/monitoring-controllers.md
|
||||||
|
- alert: PlatformControllerMetricsUnavailable
|
||||||
|
expr: up{job="helm-controller"} == 0 or absent(up{job="helm-controller"})
|
||||||
|
for: 5m
|
||||||
|
labels:
|
||||||
|
severity: warning
|
||||||
|
job: helm-controller
|
||||||
|
annotations:
|
||||||
|
summary: 控制器 {{ $labels.job }} 指标不可用
|
||||||
|
description: 检查对应 Pod、监控 CR 转换与 targets；目标消失同样触发。
|
||||||
|
runbook_url: https://git.ddupan.top/panxiao81/homelab-wiki/src/branch/main/guides/monitoring-controllers.md
|
||||||
|
- alert: PlatformControllerMetricsUnavailable
|
||||||
|
expr: up{job="kustomize-controller"} == 0 or absent(up{job="kustomize-controller"})
|
||||||
|
for: 5m
|
||||||
|
labels:
|
||||||
|
severity: warning
|
||||||
|
job: kustomize-controller
|
||||||
|
annotations:
|
||||||
|
summary: 控制器 {{ $labels.job }} 指标不可用
|
||||||
|
description: 检查对应 Pod、监控 CR 转换与 targets；目标消失同样触发。
|
||||||
|
runbook_url: https://git.ddupan.top/panxiao81/homelab-wiki/src/branch/main/guides/monitoring-controllers.md
|
||||||
|
- alert: PlatformControllerMetricsUnavailable
|
||||||
|
expr: up{job="source-controller"} == 0 or absent(up{job="source-controller"})
|
||||||
|
for: 5m
|
||||||
|
labels:
|
||||||
|
severity: warning
|
||||||
|
job: source-controller
|
||||||
|
annotations:
|
||||||
|
summary: 控制器 {{ $labels.job }} 指标不可用
|
||||||
|
description: 检查对应 Pod、监控 CR 转换与 targets；目标消失同样触发。
|
||||||
|
runbook_url: https://git.ddupan.top/panxiao81/homelab-wiki/src/branch/main/guides/monitoring-controllers.md
|
||||||
|
- alert: PlatformControllerMetricsUnavailable
|
||||||
|
expr: up{job="notification-controller"} == 0 or absent(up{job="notification-controller"})
|
||||||
|
for: 5m
|
||||||
|
labels:
|
||||||
|
severity: warning
|
||||||
|
job: notification-controller
|
||||||
|
annotations:
|
||||||
|
summary: 控制器 {{ $labels.job }} 指标不可用
|
||||||
|
description: 检查对应 Pod、监控 CR 转换与 targets；目标消失同样触发。
|
||||||
|
runbook_url: https://git.ddupan.top/panxiao81/homelab-wiki/src/branch/main/guides/monitoring-controllers.md
|
||||||
|
- alert: CertificateNotReady
|
||||||
|
expr: certmanager_certificate_ready_status{job="cert-manager",condition="True"}
|
||||||
|
== 0
|
||||||
|
for: 15m
|
||||||
|
labels:
|
||||||
|
severity: warning
|
||||||
|
annotations:
|
||||||
|
summary: 证书 {{ $labels.namespace }}/{{ $labels.name }} 未就绪
|
||||||
|
description: 检查 Certificate、CertificateRequest、Issuer、Order 与 Challenge；允许短暂签发过程。
|
||||||
|
runbook_url: https://git.ddupan.top/panxiao81/homelab-wiki/src/branch/main/guides/monitoring-controllers.md
|
||||||
|
- alert: CertificateExpiringSoon
|
||||||
|
expr: (certmanager_certificate_expiration_timestamp_seconds{job="cert-manager"}
|
||||||
|
- time() < 7 * 86400) and (certmanager_certificate_expiration_timestamp_seconds{job="cert-manager"}
|
||||||
|
- time() >= 86400)
|
||||||
|
for: 15m
|
||||||
|
labels:
|
||||||
|
severity: warning
|
||||||
|
annotations:
|
||||||
|
summary: 证书 {{ $labels.namespace }}/{{ $labels.name }} 将在七天内到期
|
||||||
|
description: 检查自动续期链路及实际入口证书；一天内到期由 critical 规则接替。
|
||||||
|
runbook_url: https://git.ddupan.top/panxiao81/homelab-wiki/src/branch/main/guides/monitoring-controllers.md
|
||||||
|
- alert: CertificateExpiryCritical
|
||||||
|
expr: certmanager_certificate_expiration_timestamp_seconds{job="cert-manager"}
|
||||||
|
- time() < 86400
|
||||||
|
for: 5m
|
||||||
|
labels:
|
||||||
|
severity: critical
|
||||||
|
annotations:
|
||||||
|
summary: 证书 {{ $labels.namespace }}/{{ $labels.name }} 即将到期或已过期
|
||||||
|
description: 不足一天或已经过期；立即检查续期与入口证书。
|
||||||
|
runbook_url: https://git.ddupan.top/panxiao81/homelab-wiki/src/branch/main/guides/monitoring-controllers.md
|
||||||
|
- alert: ExternalSecretNotReady
|
||||||
|
expr: externalsecret_status_condition{job="external-secrets",condition="Ready",status="True"}
|
||||||
|
== 0
|
||||||
|
for: 10m
|
||||||
|
labels:
|
||||||
|
severity: warning
|
||||||
|
annotations:
|
||||||
|
summary: Secret 同步 {{ $labels.namespace }}/{{ $labels.name }} 未就绪
|
||||||
|
description: 检查 ExternalSecret 状态及 provider 权限、网络；不要输出 Secret 内容。
|
||||||
|
runbook_url: https://git.ddupan.top/panxiao81/homelab-wiki/src/branch/main/guides/monitoring-controllers.md
|
||||||
|
- alert: ClusterSecretStoreNotReady
|
||||||
|
expr: clustersecretstore_status_condition{job="external-secrets",condition="Ready",status="True"}
|
||||||
|
== 0
|
||||||
|
for: 5m
|
||||||
|
labels:
|
||||||
|
severity: critical
|
||||||
|
annotations:
|
||||||
|
summary: ClusterSecretStore {{ $labels.name }} 未就绪
|
||||||
|
description: 检查 OpenBao 可达性与 ESO 身份鉴权；可能影响多个命名空间。
|
||||||
|
runbook_url: https://git.ddupan.top/panxiao81/homelab-wiki/src/branch/main/guides/monitoring-controllers.md
|
||||||
|
- alert: FluxResourceNotReady
|
||||||
|
expr: gotk_resource_info{job="kube-state-metrics",ready!="True",suspended!="true"}
|
||||||
|
== 1
|
||||||
|
for: 15m
|
||||||
|
labels:
|
||||||
|
severity: warning
|
||||||
|
annotations:
|
||||||
|
summary: Flux {{ $labels.customresource_kind }} {{ $labels.resource_namespace
|
||||||
|
}}/{{ $labels.name }} 未就绪
|
||||||
|
description: 检查对象 conditions、依赖和 controller 日志；主动 suspend 不触发，不自动解除暂停。
|
||||||
|
runbook_url: https://git.ddupan.top/panxiao81/homelab-wiki/src/branch/main/guides/monitoring-controllers.md
|
||||||
|
- alert: FluxStateMetricsMissing
|
||||||
|
expr: absent(gotk_resource_info{job="kube-state-metrics",customresource_kind="Kustomization",resource_namespace="flux-system",name="flux-system"})
|
||||||
|
for: 5m
|
||||||
|
labels:
|
||||||
|
severity: warning
|
||||||
|
annotations:
|
||||||
|
summary: Flux 对象状态指标缺失
|
||||||
|
description: 检查 kube-state-metrics 自定义资源配置、RBAC 与采集；不能把空结果当作健康。
|
||||||
|
runbook_url: https://git.ddupan.top/panxiao81/homelab-wiki/src/branch/main/guides/monitoring-controllers.md
|
||||||
@@ -0,0 +1,22 @@
|
|||||||
|
apiVersion: monitoring.coreos.com/v1
|
||||||
|
kind: PodMonitor
|
||||||
|
metadata:
|
||||||
|
name: cnpg
|
||||||
|
namespace: monitoring
|
||||||
|
spec:
|
||||||
|
namespaceSelector:
|
||||||
|
matchNames:
|
||||||
|
- shared-db
|
||||||
|
selector:
|
||||||
|
matchLabels:
|
||||||
|
cnpg.io/cluster: shared-postgresql
|
||||||
|
cnpg.io/podRole: instance
|
||||||
|
podTargetLabels:
|
||||||
|
- cnpg.io/cluster
|
||||||
|
podMetricsEndpoints:
|
||||||
|
- port: metrics
|
||||||
|
interval: 30s
|
||||||
|
honorLabels: true
|
||||||
|
relabelings:
|
||||||
|
- targetLabel: job
|
||||||
|
replacement: cnpg
|
||||||
@@ -0,0 +1,57 @@
|
|||||||
|
apiVersion: monitoring.coreos.com/v1
|
||||||
|
kind: ServiceMonitor
|
||||||
|
metadata:
|
||||||
|
name: cert-manager
|
||||||
|
namespace: monitoring
|
||||||
|
spec:
|
||||||
|
namespaceSelector:
|
||||||
|
matchNames:
|
||||||
|
- cert-manager
|
||||||
|
selector:
|
||||||
|
matchLabels:
|
||||||
|
app.kubernetes.io/name: cert-manager
|
||||||
|
app.kubernetes.io/component: controller
|
||||||
|
jobLabel: app.kubernetes.io/name
|
||||||
|
endpoints:
|
||||||
|
- port: http-metrics
|
||||||
|
interval: 30s
|
||||||
|
scrapeTimeout: 10s
|
||||||
|
honorLabels: true
|
||||||
|
---
|
||||||
|
apiVersion: monitoring.coreos.com/v1
|
||||||
|
kind: PodMonitor
|
||||||
|
metadata:
|
||||||
|
name: external-secrets
|
||||||
|
namespace: monitoring
|
||||||
|
spec:
|
||||||
|
namespaceSelector:
|
||||||
|
matchNames:
|
||||||
|
- external-secrets
|
||||||
|
selector:
|
||||||
|
matchLabels:
|
||||||
|
app.kubernetes.io/instance: external-secrets
|
||||||
|
jobLabel: app.kubernetes.io/name
|
||||||
|
podMetricsEndpoints:
|
||||||
|
- port: metrics
|
||||||
|
interval: 30s
|
||||||
|
scrapeTimeout: 10s
|
||||||
|
honorLabels: true
|
||||||
|
---
|
||||||
|
apiVersion: monitoring.coreos.com/v1
|
||||||
|
kind: PodMonitor
|
||||||
|
metadata:
|
||||||
|
name: flux-controllers
|
||||||
|
namespace: monitoring
|
||||||
|
spec:
|
||||||
|
namespaceSelector:
|
||||||
|
matchNames:
|
||||||
|
- flux-system
|
||||||
|
selector:
|
||||||
|
matchLabels:
|
||||||
|
app.kubernetes.io/part-of: flux
|
||||||
|
jobLabel: app
|
||||||
|
podMetricsEndpoints:
|
||||||
|
- port: http-prom
|
||||||
|
interval: 30s
|
||||||
|
scrapeTimeout: 10s
|
||||||
|
honorLabels: true
|
||||||
@@ -0,0 +1,25 @@
|
|||||||
|
apiVersion: monitoring.coreos.com/v1
|
||||||
|
kind: ServiceMonitor
|
||||||
|
metadata:
|
||||||
|
name: seaweedfs
|
||||||
|
namespace: monitoring
|
||||||
|
spec:
|
||||||
|
namespaceSelector:
|
||||||
|
matchNames:
|
||||||
|
- seaweedfs
|
||||||
|
selector:
|
||||||
|
matchLabels:
|
||||||
|
app.kubernetes.io/name: seaweedfs
|
||||||
|
app.kubernetes.io/instance: seaweedfs
|
||||||
|
endpoints:
|
||||||
|
- port: metrics
|
||||||
|
interval: 30s
|
||||||
|
honorLabels: true
|
||||||
|
relabelings:
|
||||||
|
- sourceLabels:
|
||||||
|
- __meta_kubernetes_service_name
|
||||||
|
regex: seaweedfs-(master|volume|filer)
|
||||||
|
action: keep
|
||||||
|
- sourceLabels:
|
||||||
|
- __meta_kubernetes_service_name
|
||||||
|
targetLabel: job
|
||||||
@@ -10,11 +10,11 @@ import sys
|
|||||||
import yaml
|
import yaml
|
||||||
|
|
||||||
groups = []
|
groups = []
|
||||||
for name in ("kubernetes-health", "host-health", "monitoring-delivery"):
|
for name in ("kubernetes-health", "host-health", "monitoring-delivery", "platform-controllers", "data-services"):
|
||||||
path = pathlib.Path(sys.argv[1]) / f"{name}.yaml"
|
path = pathlib.Path(sys.argv[1]) / f"{name}.yaml"
|
||||||
groups.extend(yaml.safe_load(path.read_text())["spec"]["groups"])
|
groups.extend(yaml.safe_load(path.read_text())["spec"]["groups"])
|
||||||
pathlib.Path(sys.argv[2]).write_text(yaml.safe_dump({"groups": groups}, allow_unicode=True))
|
pathlib.Path(sys.argv[2]).write_text(yaml.safe_dump({"groups": groups}, allow_unicode=True))
|
||||||
PY
|
PY
|
||||||
cp "$test_dir/foundation.test.yaml" "$check_dir/foundation.test.yaml"
|
cp "$test_dir/"*.test.yaml "$check_dir/"
|
||||||
promtool check rules "$check_dir/rules.yaml"
|
promtool check rules "$check_dir/rules.yaml"
|
||||||
promtool test rules "$check_dir/foundation.test.yaml"
|
promtool test rules "$check_dir/"*.test.yaml
|
||||||
|
|||||||
@@ -0,0 +1,176 @@
|
|||||||
|
rule_files:
|
||||||
|
- rules.yaml
|
||||||
|
evaluation_interval: 1m
|
||||||
|
tests:
|
||||||
|
- name: Flux 暂停的失败对象不报警
|
||||||
|
interval: 1m
|
||||||
|
input_series:
|
||||||
|
- series: gotk_resource_info{job="kube-state-metrics",ready="False",suspended="true",customresource_kind="HelmRelease",resource_namespace="app",name="paused"}
|
||||||
|
values: 1x30
|
||||||
|
alert_rule_test:
|
||||||
|
- eval_time: 20m
|
||||||
|
alertname: FluxResourceNotReady
|
||||||
|
exp_alerts: []
|
||||||
|
- name: Flux 缺省 suspend 仍检测失败
|
||||||
|
interval: 1m
|
||||||
|
input_series:
|
||||||
|
- series: gotk_resource_info{job="kube-state-metrics",ready="False",customresource_kind="HelmRelease",resource_namespace="app",name="broken"}
|
||||||
|
values: 1x30
|
||||||
|
alert_rule_test:
|
||||||
|
- eval_time: 10m
|
||||||
|
alertname: FluxResourceNotReady
|
||||||
|
exp_alerts: []
|
||||||
|
- eval_time: 16m
|
||||||
|
alertname: FluxResourceNotReady
|
||||||
|
exp_alerts:
|
||||||
|
- exp_labels:
|
||||||
|
job: kube-state-metrics
|
||||||
|
ready: 'False'
|
||||||
|
customresource_kind: HelmRelease
|
||||||
|
resource_namespace: app
|
||||||
|
name: broken
|
||||||
|
severity: warning
|
||||||
|
exp_annotations:
|
||||||
|
summary: Flux HelmRelease app/broken 未就绪
|
||||||
|
description: 检查对象 conditions、依赖和 controller 日志；主动 suspend 不触发，不自动解除暂停。
|
||||||
|
runbook_url: https://git.ddupan.top/panxiao81/homelab-wiki/src/branch/main/guides/monitoring-controllers.md
|
||||||
|
- name: 证书一天内到期只由 critical 接替
|
||||||
|
interval: 1m
|
||||||
|
input_series:
|
||||||
|
- series: certmanager_certificate_expiration_timestamp_seconds{job="cert-manager",namespace="app",name="tls"}
|
||||||
|
values: 3600x30
|
||||||
|
alert_rule_test:
|
||||||
|
- eval_time: 20m
|
||||||
|
alertname: CertificateExpiringSoon
|
||||||
|
exp_alerts: []
|
||||||
|
- name: 过期证书仍持续 critical
|
||||||
|
interval: 1m
|
||||||
|
input_series:
|
||||||
|
- series: certmanager_certificate_expiration_timestamp_seconds{job="cert-manager",namespace="app",name="tls"}
|
||||||
|
values: '0x30'
|
||||||
|
alert_rule_test:
|
||||||
|
- eval_time: 4m
|
||||||
|
alertname: CertificateExpiryCritical
|
||||||
|
exp_alerts: []
|
||||||
|
- eval_time: 6m
|
||||||
|
alertname: CertificateExpiryCritical
|
||||||
|
exp_alerts:
|
||||||
|
- exp_labels:
|
||||||
|
job: cert-manager
|
||||||
|
namespace: app
|
||||||
|
name: tls
|
||||||
|
severity: critical
|
||||||
|
exp_annotations:
|
||||||
|
summary: 证书 app/tls 即将到期或已过期
|
||||||
|
description: 不足一天或已经过期；立即检查续期与入口证书。
|
||||||
|
runbook_url: https://git.ddupan.top/panxiao81/homelab-wiki/src/branch/main/guides/monitoring-controllers.md
|
||||||
|
- name: ESO False 零值不能导致健康对象误报
|
||||||
|
interval: 1m
|
||||||
|
input_series:
|
||||||
|
- series: externalsecret_status_condition{job="external-secrets",condition="Ready",status="False",namespace="app",name="credentials"}
|
||||||
|
values: '0x30'
|
||||||
|
- series: externalsecret_status_condition{job="external-secrets",condition="Ready",status="True",namespace="app",name="credentials"}
|
||||||
|
values: 1x30
|
||||||
|
alert_rule_test:
|
||||||
|
- eval_time: 20m
|
||||||
|
alertname: ExternalSecretNotReady
|
||||||
|
exp_alerts: []
|
||||||
|
- name: ESO 失败持续十分钟报警
|
||||||
|
interval: 1m
|
||||||
|
input_series:
|
||||||
|
- series: externalsecret_status_condition{job="external-secrets",condition="Ready",status="True",namespace="app",name="credentials"}
|
||||||
|
values: '0x30'
|
||||||
|
alert_rule_test:
|
||||||
|
- eval_time: 9m
|
||||||
|
alertname: ExternalSecretNotReady
|
||||||
|
exp_alerts: []
|
||||||
|
- eval_time: 11m
|
||||||
|
alertname: ExternalSecretNotReady
|
||||||
|
exp_alerts:
|
||||||
|
- exp_labels:
|
||||||
|
job: external-secrets
|
||||||
|
condition: Ready
|
||||||
|
status: 'True'
|
||||||
|
namespace: app
|
||||||
|
name: credentials
|
||||||
|
severity: warning
|
||||||
|
exp_annotations:
|
||||||
|
summary: Secret 同步 app/credentials 未就绪
|
||||||
|
description: 检查 ExternalSecret 状态及 provider 权限、网络；不要输出 Secret 内容。
|
||||||
|
runbook_url: https://git.ddupan.top/panxiao81/homelab-wiki/src/branch/main/guides/monitoring-controllers.md
|
||||||
|
- name: CNPG 聚合数据库连接并对齐实例限额
|
||||||
|
interval: 1m
|
||||||
|
input_series:
|
||||||
|
- series: cnpg_backends_total{job="cnpg",namespace="db",pod="pg-1",datname="a"}
|
||||||
|
values: 50x30
|
||||||
|
- series: cnpg_backends_total{job="cnpg",namespace="db",pod="pg-1",datname="b"}
|
||||||
|
values: 40x30
|
||||||
|
- series: cnpg_pg_settings_setting{job="cnpg",namespace="db",pod="pg-1",name="max_connections"}
|
||||||
|
values: 100x30
|
||||||
|
alert_rule_test:
|
||||||
|
- eval_time: 9m
|
||||||
|
alertname: CnpgConnectionsHigh
|
||||||
|
exp_alerts: []
|
||||||
|
- eval_time: 11m
|
||||||
|
alertname: CnpgConnectionsHigh
|
||||||
|
exp_alerts:
|
||||||
|
- exp_labels:
|
||||||
|
job: cnpg
|
||||||
|
namespace: db
|
||||||
|
pod: pg-1
|
||||||
|
severity: warning
|
||||||
|
exp_annotations:
|
||||||
|
summary: CNPG pg-1 连接使用率超过 80%
|
||||||
|
description: 检查连接池、空闲连接和连接泄漏，不直接提高 max_connections。
|
||||||
|
runbook_url: https://git.ddupan.top/panxiao81/homelab-wiki/src/branch/main/guides/monitoring-data-services.md
|
||||||
|
- name: SeaweedFS 磁盘分母忽略 type 标签
|
||||||
|
interval: 1m
|
||||||
|
input_series:
|
||||||
|
- series: SeaweedFS_volumeServer_resource{job="seaweedfs-volume",name="/data",type="avail"}
|
||||||
|
values: 5x30
|
||||||
|
- series: SeaweedFS_volumeServer_resource{job="seaweedfs-volume",name="/data",type="all"}
|
||||||
|
values: 100x30
|
||||||
|
alert_rule_test:
|
||||||
|
- eval_time: 9m
|
||||||
|
alertname: SeaweedVolumeDiskLow
|
||||||
|
exp_alerts: []
|
||||||
|
- eval_time: 11m
|
||||||
|
alertname: SeaweedVolumeDiskLow
|
||||||
|
exp_alerts:
|
||||||
|
- exp_labels:
|
||||||
|
job: seaweedfs-volume
|
||||||
|
name: /data
|
||||||
|
severity: warning
|
||||||
|
exp_annotations:
|
||||||
|
summary: SeaweedFS /data 可用空间不足 10%
|
||||||
|
description: 检查 volume 文件系统容量、保留策略与底层 ZFS；不要直接删除卷文件。
|
||||||
|
runbook_url: https://git.ddupan.top/panxiao81/homelab-wiki/src/branch/main/guides/monitoring-data-services.md
|
||||||
|
- name: NATS 历史慢消费者计数不持续报警
|
||||||
|
interval: 1m
|
||||||
|
input_series:
|
||||||
|
- series: nats_varz_slow_consumers{job="nats/nats"}
|
||||||
|
values: 10x30
|
||||||
|
alert_rule_test:
|
||||||
|
- eval_time: 20m
|
||||||
|
alertname: NatsSlowConsumers
|
||||||
|
exp_alerts: []
|
||||||
|
- name: Flux 根对象状态指标缺失报警
|
||||||
|
interval: 1m
|
||||||
|
input_series: []
|
||||||
|
alert_rule_test:
|
||||||
|
- eval_time: 4m
|
||||||
|
alertname: FluxStateMetricsMissing
|
||||||
|
exp_alerts: []
|
||||||
|
- eval_time: 6m
|
||||||
|
alertname: FluxStateMetricsMissing
|
||||||
|
exp_alerts:
|
||||||
|
- exp_labels:
|
||||||
|
job: kube-state-metrics
|
||||||
|
customresource_kind: Kustomization
|
||||||
|
resource_namespace: flux-system
|
||||||
|
name: flux-system
|
||||||
|
severity: warning
|
||||||
|
exp_annotations:
|
||||||
|
summary: Flux 对象状态指标缺失
|
||||||
|
description: 检查 kube-state-metrics 自定义资源配置、RBAC 与采集；不能把空结果当作健康。
|
||||||
|
runbook_url: https://git.ddupan.top/panxiao81/homelab-wiki/src/branch/main/guides/monitoring-controllers.md
|
||||||
Reference in New Issue
Block a user