补齐集群状态、主机与通知链路基础监控 (#149)

Co-authored-by: panxiao81 <[email protected]>
2026-09-25 17:45:56 +00:00
parent a43b7d3c26
commit 39aeac25ee
9 changed files with 474 additions and 0 deletions
@@ -0,0 +1,66 @@
+apiVersion: operator.victoriametrics.com/v1beta1
+kind: VMRule
+metadata:
+  name: host-health
+  namespace: monitoring
+  labels:
+    app.kubernetes.io/part-of: victoria-metrics
+spec:
+  groups:
+  - name: host-health
+    interval: 30s
+    rules:
+    - alert: HostMemoryLow
+      expr: (node_memory_MemAvailable_bytes{job="node-exporter"} / node_memory_MemTotal_bytes{job="node-exporter"})
+        < 0.10
+      for: 10m
+      labels:
+        severity: warning
+      annotations:
+        summary: 主机 {{ $labels.instance }} 可用内存不足 10%
+        description: 检查主机内存、Swap、进程 PSS；仅使用 node-exporter job，避免重复采集统计。
+        runbook_url: https://git.ddupan.top/panxiao81/homelab-wiki/src/branch/main/guides/monitoring-foundation.md
+    - alert: HostFilesystemSpaceLow
+      expr: (node_filesystem_avail_bytes{job="node-exporter",fstype!~"tmpfs|devtmpfs|overlay|squashfs|nsfs|fuse.*",mountpoint!~"/run(/.*)?|/var/lib/kubelet/.*|/var/lib/docker/.*"}
+        / node_filesystem_size_bytes{job="node-exporter",fstype!~"tmpfs|devtmpfs|overlay|squashfs|nsfs|fuse.*",mountpoint!~"/run(/.*)?|/var/lib/kubelet/.*|/var/lib/docker/.*"}
+        < 0.10) and on(instance,device,mountpoint) (node_filesystem_readonly{job="node-exporter",fstype!~"tmpfs|devtmpfs|overlay|squashfs|nsfs|fuse.*",mountpoint!~"/run(/.*)?|/var/lib/kubelet/.*|/var/lib/docker/.*"}
+        == 0)
+      for: 15m
+      labels:
+        severity: warning
+      annotations:
+        summary: 主机 {{ $labels.instance }} {{ $labels.mountpoint }} 可用空间不足 10%
+        description: 检查文件系统和 ZFS dataset 容量，清理前确认数据归属。
+        runbook_url: https://git.ddupan.top/panxiao81/homelab-wiki/src/branch/main/guides/monitoring-foundation.md
+    - alert: HostFilesystemInodesLow
+      expr: (node_filesystem_files_free{job="node-exporter",fstype!~"tmpfs|devtmpfs|overlay|squashfs|nsfs|fuse.*",mountpoint!~"/run(/.*)?|/var/lib/kubelet/.*|/var/lib/docker/.*"}
+        / node_filesystem_files{job="node-exporter",fstype!~"tmpfs|devtmpfs|overlay|squashfs|nsfs|fuse.*",mountpoint!~"/run(/.*)?|/var/lib/kubelet/.*|/var/lib/docker/.*"}
+        < 0.10) and on(instance,device,mountpoint) (node_filesystem_files{job="node-exporter",fstype!~"tmpfs|devtmpfs|overlay|squashfs|nsfs|fuse.*",mountpoint!~"/run(/.*)?|/var/lib/kubelet/.*|/var/lib/docker/.*"}
+        > 0) and on(instance,device,mountpoint) (node_filesystem_readonly{job="node-exporter",fstype!~"tmpfs|devtmpfs|overlay|squashfs|nsfs|fuse.*",mountpoint!~"/run(/.*)?|/var/lib/kubelet/.*|/var/lib/docker/.*"}
+        == 0)
+      for: 15m
+      labels:
+        severity: warning
+      annotations:
+        summary: 主机 {{ $labels.instance }} {{ $labels.mountpoint }} inode 不足 10%
+        description: 检查小文件数量，确认 filesystem 是否耗尽 inode。
+        runbook_url: https://git.ddupan.top/panxiao81/homelab-wiki/src/branch/main/guides/monitoring-foundation.md
+    - alert: HostFilesystemReadOnly
+      expr: node_filesystem_readonly{job="node-exporter",fstype!~"tmpfs|devtmpfs|overlay|squashfs|nsfs|fuse.*",mountpoint!~"/run(/.*)?|/var/lib/kubelet/.*|/var/lib/docker/.*"}
+        == 1
+      for: 5m
+      labels:
+        severity: critical
+      annotations:
+        summary: 主机 {{ $labels.instance }} {{ $labels.mountpoint }} 变为只读
+        description: 排查磁盘和文件系统错误；不要直接强制重新挂载。
+        runbook_url: https://git.ddupan.top/panxiao81/homelab-wiki/src/branch/main/guides/monitoring-foundation.md
+    - alert: HostZpoolUnhealthy
+      expr: node_zfs_zpool_state{job="node-exporter",state!="online"} == 1
+      for: 5m
+      labels:
+        severity: critical
+      annotations:
+        summary: 主机 {{ $labels.instance }} ZFS 池 {{ $labels.zpool }} 状态 {{ $labels.state }}
+        description: 检查 zpool status、磁盘和冗余；零值状态不触发。
+        runbook_url: https://git.ddupan.top/panxiao81/homelab-wiki/src/branch/main/guides/monitoring-foundation.md
@@ -0,0 +1,121 @@
+apiVersion: operator.victoriametrics.com/v1beta1
+kind: VMRule
+metadata:
+  name: kubernetes-health
+  namespace: monitoring
+  labels:
+    app.kubernetes.io/part-of: victoria-metrics
+spec:
+  groups:
+  - name: kubernetes-health
+    interval: 30s
+    rules:
+    - alert: KubeNodeNotReady
+      expr: max by (node) (kube_node_status_condition{job="kube-state-metrics",condition="Ready",status="true"})
+        == 0
+      for: 5m
+      labels:
+        severity: critical
+      annotations:
+        summary: 节点 {{ $labels.node }} 未就绪
+        description: 检查节点、kubelet 和网络；维护时按 node 精确静默。
+        runbook_url: https://git.ddupan.top/panxiao81/homelab-wiki/src/branch/main/guides/monitoring-foundation.md
+    - alert: KubeNodePressure
+      expr: max by (node, condition) (kube_node_status_condition{job="kube-state-metrics",condition=~"MemoryPressure|DiskPressure|PIDPressure",status="true"})
+        == 1
+      for: 5m
+      labels:
+        severity: warning
+      annotations:
+        summary: 节点 {{ $labels.node }} 出现 {{ $labels.condition }}
+        description: 检查内存、磁盘/inode 或 PID 资源压力。
+        runbook_url: https://git.ddupan.top/panxiao81/homelab-wiki/src/branch/main/guides/monitoring-foundation.md
+    - alert: KubePodCrashLooping
+      expr: max by (namespace,pod,container) (kube_pod_container_status_waiting_reason{job="kube-state-metrics",reason="CrashLoopBackOff"})
+        == 1
+      for: 10m
+      labels:
+        severity: warning
+      annotations:
+        summary: 容器 {{ $labels.namespace }}/{{ $labels.pod }}/{{ $labels.container }} 持续崩溃
+        description: 检查容器退出原因和日志；持续 10 分钟后通知，避免短暂重启噪声。
+        runbook_url: https://git.ddupan.top/panxiao81/homelab-wiki/src/branch/main/guides/monitoring-foundation.md
+    - alert: KubePodFrequentRestarts
+      expr: sum by (namespace,pod,container) (increase(kube_pod_container_status_restarts_total{job="kube-state-metrics"}[15m]))
+        > 3
+      for: 5m
+      labels:
+        severity: warning
+      annotations:
+        summary: 容器 {{ $labels.namespace }}/{{ $labels.pod }}/{{ $labels.container }} 频繁重启
+        description: 15 分钟内重启超过 3 次，检查退出原因和资源限制。
+        runbook_url: https://git.ddupan.top/panxiao81/homelab-wiki/src/branch/main/guides/monitoring-foundation.md
+    - alert: KubePodOOMKilled
+      expr: (max by (namespace,pod,container) (kube_pod_container_status_last_terminated_reason{job="kube-state-metrics",reason="OOMKilled"})
+        == 1) and on(namespace,pod,container) (sum by (namespace,pod,container) (increase(kube_pod_container_status_restarts_total{job="kube-state-metrics"}[10m]))
+        > 0)
+      for: 1m
+      labels:
+        severity: warning
+      annotations:
+        summary: 容器 {{ $labels.namespace }}/{{ $labels.pod }}/{{ $labels.container }} 发生 OOM 重启
+        description: 仅在近期有重启且最近终止原因为 OOMKilled 时触发，检查内存峰值和限制。
+        runbook_url: https://git.ddupan.top/panxiao81/homelab-wiki/src/branch/main/guides/monitoring-foundation.md
+    - alert: KubePodPending
+      expr: max by (namespace,pod) (kube_pod_status_phase{job="kube-state-metrics",phase="Pending"}) == 1
+      for: 15m
+      labels:
+        severity: warning
+      annotations:
+        summary: Pod {{ $labels.namespace }}/{{ $labels.pod }} 长时间 Pending
+        description: 检查调度事件、资源、卷和镜像拉取。
+        runbook_url: https://git.ddupan.top/panxiao81/homelab-wiki/src/branch/main/guides/monitoring-foundation.md
+    - alert: KubeDeploymentUnavailable
+      expr: (max by(namespace,deployment) (kube_deployment_spec_replicas{job="kube-state-metrics"}) - max by(namespace,deployment)
+        (kube_deployment_status_replicas_available{job="kube-state-metrics"})) > 0
+      for: 10m
+      labels:
+        severity: warning
+      annotations:
+        summary: Deployment {{ $labels.namespace }}/{{ $labels.deployment }} 副本不足
+        description: 可用副本持续 10 分钟低于声明值；缩容至零不会触发。
+        runbook_url: https://git.ddupan.top/panxiao81/homelab-wiki/src/branch/main/guides/monitoring-foundation.md
+    - alert: KubeStatefulSetUnavailable
+      expr: (max by(namespace,statefulset) (kube_statefulset_replicas{job="kube-state-metrics"}) - max by(namespace,statefulset)
+        (kube_statefulset_status_replicas_ready{job="kube-state-metrics"})) > 0
+      for: 10m
+      labels:
+        severity: warning
+      annotations:
+        summary: StatefulSet {{ $labels.namespace }}/{{ $labels.statefulset }} 副本不足
+        description: 检查 Pod 就绪、存储和依赖；已知维护按工作负载精确静默。
+        runbook_url: https://git.ddupan.top/panxiao81/homelab-wiki/src/branch/main/guides/monitoring-foundation.md
+    - alert: KubeDaemonSetUnavailable
+      expr: (max by(namespace,daemonset) (kube_daemonset_status_desired_number_scheduled{job="kube-state-metrics"})
+        - max by(namespace,daemonset) (kube_daemonset_status_number_ready{job="kube-state-metrics"})) > 0
+      for: 10m
+      labels:
+        severity: warning
+      annotations:
+        summary: DaemonSet {{ $labels.namespace }}/{{ $labels.daemonset }} 副本不足
+        description: 检查节点和 Pod 就绪；不按整个 namespace 屏蔽。
+        runbook_url: https://git.ddupan.top/panxiao81/homelab-wiki/src/branch/main/guides/monitoring-foundation.md
+    - alert: KubePVCPending
+      expr: max by(namespace,persistentvolumeclaim) (kube_persistentvolumeclaim_status_phase{job="kube-state-metrics",phase="Pending"})
+        == 1
+      for: 15m
+      labels:
+        severity: warning
+      annotations:
+        summary: PVC {{ $labels.namespace }}/{{ $labels.persistentvolumeclaim }} 长时间 Pending
+        description: 检查 StorageClass、调度拓扑、容量与 provisioner。
+        runbook_url: https://git.ddupan.top/panxiao81/homelab-wiki/src/branch/main/guides/monitoring-foundation.md
+    - alert: KubeJobFailed
+      expr: max by(namespace,job_name) (kube_job_failed{job="kube-state-metrics",condition="true"}) == 1
+      for: 5m
+      labels:
+        severity: warning
+      annotations:
+        summary: Job {{ $labels.namespace }}/{{ $labels.job_name }} 已失败
+        description: 检查失败 Job 的 Pod 和任务日志；成功重试中的失败 Pod 计数不作为失败 Job。
+        runbook_url: https://git.ddupan.top/panxiao81/homelab-wiki/src/branch/main/guides/monitoring-foundation.md
@@ -0,0 +1,58 @@
+apiVersion: operator.victoriametrics.com/v1beta1
+kind: VMRule
+metadata:
+  name: monitoring-delivery
+  namespace: monitoring
+  labels:
+    app.kubernetes.io/part-of: victoria-metrics
+spec:
+  groups:
+  - name: monitoring-delivery
+    interval: 30s
+    rules:
+    - alert: KubeStateMetricsUnavailable
+      expr: up{job="kube-state-metrics"} == 0 or absent(up{job="kube-state-metrics"})
+      for: 5m
+      labels:
+        severity: critical
+      annotations:
+        summary: kube-state-metrics 采集不可用
+        description: 检查 HelmRelease、ServiceMonitor 转换、targets 与 exporter；目标消失也会触发。
+        runbook_url: https://git.ddupan.top/panxiao81/homelab-wiki/src/branch/main/guides/monitoring-foundation.md
+    - alert: NodeExporterUnavailable
+      expr: up{job="node-exporter"} == 0 or absent(up{job="node-exporter"})
+      for: 5m
+      labels:
+        severity: critical
+      annotations:
+        summary: node-exporter 采集不可用
+        description: 检查节点与 exporter，主机健康规则依赖此采集。
+        runbook_url: https://git.ddupan.top/panxiao81/homelab-wiki/src/branch/main/guides/monitoring-foundation.md
+    - alert: NatsMetricsUnavailable
+      expr: up{job="nats/nats"} == 0 or absent(up{job="nats/nats"})
+      for: 5m
+      labels:
+        severity: warning
+      annotations:
+        summary: NATS 指标采集不可用
+        description: 依次检查 PodMonitor、转换后的 VMPodScrape、:7777 exporter 及 targets。
+        runbook_url: https://git.ddupan.top/panxiao81/homelab-wiki/src/branch/main/guides/monitoring-foundation.md
+    - alert: AlertmanagerTelegramDeliveryFailed
+      expr: sum by(job,instance,integration) (increase(alertmanager_notifications_failed_total{job="vmalertmanager-main",integration="telegram"}[5m]))
+        > 0
+      for: 1m
+      labels:
+        severity: critical
+      annotations:
+        summary: Alertmanager 向 Telegram 发送失败
+        description: 检查网络、bot token 和频道权限。若 Telegram 完全不可达，本告警也无法经同一渠道送达；需在 Alertmanager/Grafana 排查，集群外心跳另行建设。
+        runbook_url: https://git.ddupan.top/panxiao81/homelab-wiki/src/branch/main/guides/monitoring-foundation.md
+    - alert: AlertmanagerConfigurationReloadFailed
+      expr: alertmanager_config_last_reload_successful{job="vmalertmanager-main"} == 0
+      for: 5m
+      labels:
+        severity: critical
+      annotations:
+        summary: Alertmanager 配置加载失败
+        description: 检查 operator 和 Alertmanager 日志、配置格式及 Secret 引用。
+        runbook_url: https://git.ddupan.top/panxiao81/homelab-wiki/src/branch/main/guides/monitoring-foundation.md