homelab-infra/platform/observability/metrics/rules/kubernetes-health.yaml

apiVersion: monitoring.coreos.com/v1
kind: PrometheusRule
metadata:
  name: kubernetes-health
  namespace: monitoring
  labels:
    app.kubernetes.io/part-of: victoria-metrics
spec:
  groups:
  - name: kubernetes-health
    interval: 30s
    rules:
    - alert: KubeNodeNotReady
      expr: max by (node) (kube_node_status_condition{job="kube-state-metrics",condition="Ready",status="true"})
        == 0
      for: 5m
      labels:
        severity: critical
      annotations:
        summary: 节点 {{ $labels.node }} 未就绪
        description: 检查节点、kubelet 和网络；维护时按 node 精确静默。
        runbook_url: https://git.ddupan.top/panxiao81/homelab-wiki/src/branch/main/guides/monitoring-foundation.md
    - alert: KubeNodePressure
      expr: max by (node, condition) (kube_node_status_condition{job="kube-state-metrics",condition=~"MemoryPressure|DiskPressure|PIDPressure",status="true"})
        == 1
      for: 5m
      labels:
        severity: warning
      annotations:
        summary: 节点 {{ $labels.node }} 出现 {{ $labels.condition }}
        description: 检查内存、磁盘/inode 或 PID 资源压力。
        runbook_url: https://git.ddupan.top/panxiao81/homelab-wiki/src/branch/main/guides/monitoring-foundation.md
    - alert: KubePodCrashLooping
      expr: max by (namespace,pod,container) (kube_pod_container_status_waiting_reason{job="kube-state-metrics",reason="CrashLoopBackOff"})
        == 1
      for: 10m
      labels:
        severity: warning
      annotations:
        summary: 容器 {{ $labels.namespace }}/{{ $labels.pod }}/{{ $labels.container }} 持续崩溃
        description: 检查容器退出原因和日志；持续 10 分钟后通知，避免短暂重启噪声。
        runbook_url: https://git.ddupan.top/panxiao81/homelab-wiki/src/branch/main/guides/monitoring-foundation.md
    - alert: KubePodFrequentRestarts
      expr: sum by (namespace,pod,container) (increase(kube_pod_container_status_restarts_total{job="kube-state-metrics"}[15m]))
        > 3
      for: 5m
      labels:
        severity: warning
      annotations:
        summary: 容器 {{ $labels.namespace }}/{{ $labels.pod }}/{{ $labels.container }} 频繁重启
        description: 15 分钟内重启超过 3 次，检查退出原因和资源限制。
        runbook_url: https://git.ddupan.top/panxiao81/homelab-wiki/src/branch/main/guides/monitoring-foundation.md
    - alert: KubePodOOMKilled
      expr: (max by (namespace,pod,container) (kube_pod_container_status_last_terminated_reason{job="kube-state-metrics",reason="OOMKilled"})
        == 1) and on(namespace,pod,container) (sum by (namespace,pod,container) (increase(kube_pod_container_status_restarts_total{job="kube-state-metrics"}[10m]))
        > 0)
      for: 1m
      labels:
        severity: warning
      annotations:
        summary: 容器 {{ $labels.namespace }}/{{ $labels.pod }}/{{ $labels.container }} 发生 OOM 重启
        description: 仅在近期有重启且最近终止原因为 OOMKilled 时触发，检查内存峰值和限制。
        runbook_url: https://git.ddupan.top/panxiao81/homelab-wiki/src/branch/main/guides/monitoring-foundation.md
    - alert: KubePodPending
      expr: max by (namespace,pod) (kube_pod_status_phase{job="kube-state-metrics",phase="Pending"}) == 1
      for: 15m
      labels:
        severity: warning
      annotations:
        summary: Pod {{ $labels.namespace }}/{{ $labels.pod }} 长时间 Pending
        description: 检查调度事件、资源、卷和镜像拉取。
        runbook_url: https://git.ddupan.top/panxiao81/homelab-wiki/src/branch/main/guides/monitoring-foundation.md
    - alert: KubeDeploymentUnavailable
      expr: (max by(namespace,deployment) (kube_deployment_spec_replicas{job="kube-state-metrics"}) - max by(namespace,deployment)
        (kube_deployment_status_replicas_available{job="kube-state-metrics"})) > 0
      for: 10m
      labels:
        severity: warning
      annotations:
        summary: Deployment {{ $labels.namespace }}/{{ $labels.deployment }} 副本不足
        description: 可用副本持续 10 分钟低于声明值；缩容至零不会触发。
        runbook_url: https://git.ddupan.top/panxiao81/homelab-wiki/src/branch/main/guides/monitoring-foundation.md
    - alert: KubeStatefulSetUnavailable
      expr: (max by(namespace,statefulset) (kube_statefulset_replicas{job="kube-state-metrics"}) - max by(namespace,statefulset)
        (kube_statefulset_status_replicas_ready{job="kube-state-metrics"})) > 0
      for: 10m
      labels:
        severity: warning
      annotations:
        summary: StatefulSet {{ $labels.namespace }}/{{ $labels.statefulset }} 副本不足
        description: 检查 Pod 就绪、存储和依赖；已知维护按工作负载精确静默。
        runbook_url: https://git.ddupan.top/panxiao81/homelab-wiki/src/branch/main/guides/monitoring-foundation.md
    - alert: KubeDaemonSetUnavailable
      expr: (max by(namespace,daemonset) (kube_daemonset_status_desired_number_scheduled{job="kube-state-metrics"})
        - max by(namespace,daemonset) (kube_daemonset_status_number_ready{job="kube-state-metrics"})) > 0
      for: 10m
      labels:
        severity: warning
      annotations:
        summary: DaemonSet {{ $labels.namespace }}/{{ $labels.daemonset }} 副本不足
        description: 检查节点和 Pod 就绪；不按整个 namespace 屏蔽。
        runbook_url: https://git.ddupan.top/panxiao81/homelab-wiki/src/branch/main/guides/monitoring-foundation.md
    - alert: KubePVCPending
      expr: max by(namespace,persistentvolumeclaim) (kube_persistentvolumeclaim_status_phase{job="kube-state-metrics",phase="Pending"})
        == 1
      for: 15m
      labels:
        severity: warning
      annotations:
        summary: PVC {{ $labels.namespace }}/{{ $labels.persistentvolumeclaim }} 长时间 Pending
        description: 检查 StorageClass、调度拓扑、容量与 provisioner。
        runbook_url: https://git.ddupan.top/panxiao81/homelab-wiki/src/branch/main/guides/monitoring-foundation.md
    - alert: KubeJobFailed
      expr: max by(namespace,job_name) (kube_job_failed{job="kube-state-metrics",condition="true"}) == 1
      for: 5m
      labels:
        severity: warning
      annotations:
        summary: Job {{ $labels.namespace }}/{{ $labels.job_name }} 已失败
        description: 检查失败 Job 的 Pod 和任务日志；成功重试中的失败 Pod 计数不作为失败 Job。
        runbook_url: https://git.ddupan.top/panxiao81/homelab-wiki/src/branch/main/guides/monitoring-foundation.md