Co-authored-by: panxiao81 <[email protected]>
This commit was merged in pull request #149.
This commit is contained in:
@@ -0,0 +1,60 @@
|
||||
apiVersion: helm.toolkit.fluxcd.io/v2
|
||||
kind: HelmRelease
|
||||
metadata:
|
||||
name: kube-state-metrics
|
||||
namespace: monitoring
|
||||
spec:
|
||||
interval: 30m
|
||||
timeout: 5m
|
||||
dependsOn:
|
||||
- name: vm-operator
|
||||
- name: prometheus-operator-crds
|
||||
chart:
|
||||
spec:
|
||||
chart: kube-state-metrics
|
||||
version: 8.5.0
|
||||
sourceRef:
|
||||
kind: HelmRepository
|
||||
name: prometheus-community
|
||||
namespace: monitoring
|
||||
interval: 1h
|
||||
driftDetection:
|
||||
mode: enabled
|
||||
install:
|
||||
remediation:
|
||||
retries: 3
|
||||
upgrade:
|
||||
remediation:
|
||||
retries: 3
|
||||
values:
|
||||
fullnameOverride: kube-state-metrics
|
||||
collectors:
|
||||
- cronjobs
|
||||
- daemonsets
|
||||
- deployments
|
||||
- jobs
|
||||
- namespaces
|
||||
- nodes
|
||||
- persistentvolumeclaims
|
||||
- persistentvolumes
|
||||
- pods
|
||||
- statefulsets
|
||||
resources:
|
||||
requests:
|
||||
cpu: 100m
|
||||
memory: 128Mi
|
||||
limits:
|
||||
cpu: 500m
|
||||
memory: 384Mi
|
||||
prometheus:
|
||||
monitor:
|
||||
enabled: true
|
||||
jobLabel: app.kubernetes.io/name
|
||||
http:
|
||||
interval: 30s
|
||||
honorLabels: true
|
||||
metrics:
|
||||
interval: 30s
|
||||
honorLabels: true
|
||||
selfMonitor:
|
||||
enabled: true
|
||||
@@ -17,3 +17,7 @@ resources:
|
||||
- scrapes/kubelet.yaml
|
||||
- exporters/node-exporter.yaml
|
||||
- exporters/process-exporter.yaml
|
||||
- exporters/kube-state-metrics-helmrelease.yaml
|
||||
- rules/kubernetes-health.yaml
|
||||
- rules/host-health.yaml
|
||||
- rules/monitoring-delivery.yaml
|
||||
|
||||
@@ -0,0 +1,66 @@
|
||||
apiVersion: operator.victoriametrics.com/v1beta1
|
||||
kind: VMRule
|
||||
metadata:
|
||||
name: host-health
|
||||
namespace: monitoring
|
||||
labels:
|
||||
app.kubernetes.io/part-of: victoria-metrics
|
||||
spec:
|
||||
groups:
|
||||
- name: host-health
|
||||
interval: 30s
|
||||
rules:
|
||||
- alert: HostMemoryLow
|
||||
expr: (node_memory_MemAvailable_bytes{job="node-exporter"} / node_memory_MemTotal_bytes{job="node-exporter"})
|
||||
< 0.10
|
||||
for: 10m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: 主机 {{ $labels.instance }} 可用内存不足 10%
|
||||
description: 检查主机内存、Swap、进程 PSS；仅使用 node-exporter job，避免重复采集统计。
|
||||
runbook_url: https://git.ddupan.top/panxiao81/homelab-wiki/src/branch/main/guides/monitoring-foundation.md
|
||||
- alert: HostFilesystemSpaceLow
|
||||
expr: (node_filesystem_avail_bytes{job="node-exporter",fstype!~"tmpfs|devtmpfs|overlay|squashfs|nsfs|fuse.*",mountpoint!~"/run(/.*)?|/var/lib/kubelet/.*|/var/lib/docker/.*"}
|
||||
/ node_filesystem_size_bytes{job="node-exporter",fstype!~"tmpfs|devtmpfs|overlay|squashfs|nsfs|fuse.*",mountpoint!~"/run(/.*)?|/var/lib/kubelet/.*|/var/lib/docker/.*"}
|
||||
< 0.10) and on(instance,device,mountpoint) (node_filesystem_readonly{job="node-exporter",fstype!~"tmpfs|devtmpfs|overlay|squashfs|nsfs|fuse.*",mountpoint!~"/run(/.*)?|/var/lib/kubelet/.*|/var/lib/docker/.*"}
|
||||
== 0)
|
||||
for: 15m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: 主机 {{ $labels.instance }} {{ $labels.mountpoint }} 可用空间不足 10%
|
||||
description: 检查文件系统和 ZFS dataset 容量，清理前确认数据归属。
|
||||
runbook_url: https://git.ddupan.top/panxiao81/homelab-wiki/src/branch/main/guides/monitoring-foundation.md
|
||||
- alert: HostFilesystemInodesLow
|
||||
expr: (node_filesystem_files_free{job="node-exporter",fstype!~"tmpfs|devtmpfs|overlay|squashfs|nsfs|fuse.*",mountpoint!~"/run(/.*)?|/var/lib/kubelet/.*|/var/lib/docker/.*"}
|
||||
/ node_filesystem_files{job="node-exporter",fstype!~"tmpfs|devtmpfs|overlay|squashfs|nsfs|fuse.*",mountpoint!~"/run(/.*)?|/var/lib/kubelet/.*|/var/lib/docker/.*"}
|
||||
< 0.10) and on(instance,device,mountpoint) (node_filesystem_files{job="node-exporter",fstype!~"tmpfs|devtmpfs|overlay|squashfs|nsfs|fuse.*",mountpoint!~"/run(/.*)?|/var/lib/kubelet/.*|/var/lib/docker/.*"}
|
||||
> 0) and on(instance,device,mountpoint) (node_filesystem_readonly{job="node-exporter",fstype!~"tmpfs|devtmpfs|overlay|squashfs|nsfs|fuse.*",mountpoint!~"/run(/.*)?|/var/lib/kubelet/.*|/var/lib/docker/.*"}
|
||||
== 0)
|
||||
for: 15m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: 主机 {{ $labels.instance }} {{ $labels.mountpoint }} inode 不足 10%
|
||||
description: 检查小文件数量，确认 filesystem 是否耗尽 inode。
|
||||
runbook_url: https://git.ddupan.top/panxiao81/homelab-wiki/src/branch/main/guides/monitoring-foundation.md
|
||||
- alert: HostFilesystemReadOnly
|
||||
expr: node_filesystem_readonly{job="node-exporter",fstype!~"tmpfs|devtmpfs|overlay|squashfs|nsfs|fuse.*",mountpoint!~"/run(/.*)?|/var/lib/kubelet/.*|/var/lib/docker/.*"}
|
||||
== 1
|
||||
for: 5m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: 主机 {{ $labels.instance }} {{ $labels.mountpoint }} 变为只读
|
||||
description: 排查磁盘和文件系统错误；不要直接强制重新挂载。
|
||||
runbook_url: https://git.ddupan.top/panxiao81/homelab-wiki/src/branch/main/guides/monitoring-foundation.md
|
||||
- alert: HostZpoolUnhealthy
|
||||
expr: node_zfs_zpool_state{job="node-exporter",state!="online"} == 1
|
||||
for: 5m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: 主机 {{ $labels.instance }} ZFS 池 {{ $labels.zpool }} 状态 {{ $labels.state }}
|
||||
description: 检查 zpool status、磁盘和冗余；零值状态不触发。
|
||||
runbook_url: https://git.ddupan.top/panxiao81/homelab-wiki/src/branch/main/guides/monitoring-foundation.md
|
||||
@@ -0,0 +1,121 @@
|
||||
apiVersion: operator.victoriametrics.com/v1beta1
|
||||
kind: VMRule
|
||||
metadata:
|
||||
name: kubernetes-health
|
||||
namespace: monitoring
|
||||
labels:
|
||||
app.kubernetes.io/part-of: victoria-metrics
|
||||
spec:
|
||||
groups:
|
||||
- name: kubernetes-health
|
||||
interval: 30s
|
||||
rules:
|
||||
- alert: KubeNodeNotReady
|
||||
expr: max by (node) (kube_node_status_condition{job="kube-state-metrics",condition="Ready",status="true"})
|
||||
== 0
|
||||
for: 5m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: 节点 {{ $labels.node }} 未就绪
|
||||
description: 检查节点、kubelet 和网络；维护时按 node 精确静默。
|
||||
runbook_url: https://git.ddupan.top/panxiao81/homelab-wiki/src/branch/main/guides/monitoring-foundation.md
|
||||
- alert: KubeNodePressure
|
||||
expr: max by (node, condition) (kube_node_status_condition{job="kube-state-metrics",condition=~"MemoryPressure|DiskPressure|PIDPressure",status="true"})
|
||||
== 1
|
||||
for: 5m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: 节点 {{ $labels.node }} 出现 {{ $labels.condition }}
|
||||
description: 检查内存、磁盘/inode 或 PID 资源压力。
|
||||
runbook_url: https://git.ddupan.top/panxiao81/homelab-wiki/src/branch/main/guides/monitoring-foundation.md
|
||||
- alert: KubePodCrashLooping
|
||||
expr: max by (namespace,pod,container) (kube_pod_container_status_waiting_reason{job="kube-state-metrics",reason="CrashLoopBackOff"})
|
||||
== 1
|
||||
for: 10m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: 容器 {{ $labels.namespace }}/{{ $labels.pod }}/{{ $labels.container }} 持续崩溃
|
||||
description: 检查容器退出原因和日志；持续 10 分钟后通知，避免短暂重启噪声。
|
||||
runbook_url: https://git.ddupan.top/panxiao81/homelab-wiki/src/branch/main/guides/monitoring-foundation.md
|
||||
- alert: KubePodFrequentRestarts
|
||||
expr: sum by (namespace,pod,container) (increase(kube_pod_container_status_restarts_total{job="kube-state-metrics"}[15m]))
|
||||
> 3
|
||||
for: 5m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: 容器 {{ $labels.namespace }}/{{ $labels.pod }}/{{ $labels.container }} 频繁重启
|
||||
description: 15 分钟内重启超过 3 次，检查退出原因和资源限制。
|
||||
runbook_url: https://git.ddupan.top/panxiao81/homelab-wiki/src/branch/main/guides/monitoring-foundation.md
|
||||
- alert: KubePodOOMKilled
|
||||
expr: (max by (namespace,pod,container) (kube_pod_container_status_last_terminated_reason{job="kube-state-metrics",reason="OOMKilled"})
|
||||
== 1) and on(namespace,pod,container) (sum by (namespace,pod,container) (increase(kube_pod_container_status_restarts_total{job="kube-state-metrics"}[10m]))
|
||||
> 0)
|
||||
for: 1m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: 容器 {{ $labels.namespace }}/{{ $labels.pod }}/{{ $labels.container }} 发生 OOM 重启
|
||||
description: 仅在近期有重启且最近终止原因为 OOMKilled 时触发，检查内存峰值和限制。
|
||||
runbook_url: https://git.ddupan.top/panxiao81/homelab-wiki/src/branch/main/guides/monitoring-foundation.md
|
||||
- alert: KubePodPending
|
||||
expr: max by (namespace,pod) (kube_pod_status_phase{job="kube-state-metrics",phase="Pending"}) == 1
|
||||
for: 15m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: Pod {{ $labels.namespace }}/{{ $labels.pod }} 长时间 Pending
|
||||
description: 检查调度事件、资源、卷和镜像拉取。
|
||||
runbook_url: https://git.ddupan.top/panxiao81/homelab-wiki/src/branch/main/guides/monitoring-foundation.md
|
||||
- alert: KubeDeploymentUnavailable
|
||||
expr: (max by(namespace,deployment) (kube_deployment_spec_replicas{job="kube-state-metrics"}) - max by(namespace,deployment)
|
||||
(kube_deployment_status_replicas_available{job="kube-state-metrics"})) > 0
|
||||
for: 10m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: Deployment {{ $labels.namespace }}/{{ $labels.deployment }} 副本不足
|
||||
description: 可用副本持续 10 分钟低于声明值；缩容至零不会触发。
|
||||
runbook_url: https://git.ddupan.top/panxiao81/homelab-wiki/src/branch/main/guides/monitoring-foundation.md
|
||||
- alert: KubeStatefulSetUnavailable
|
||||
expr: (max by(namespace,statefulset) (kube_statefulset_replicas{job="kube-state-metrics"}) - max by(namespace,statefulset)
|
||||
(kube_statefulset_status_replicas_ready{job="kube-state-metrics"})) > 0
|
||||
for: 10m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: StatefulSet {{ $labels.namespace }}/{{ $labels.statefulset }} 副本不足
|
||||
description: 检查 Pod 就绪、存储和依赖；已知维护按工作负载精确静默。
|
||||
runbook_url: https://git.ddupan.top/panxiao81/homelab-wiki/src/branch/main/guides/monitoring-foundation.md
|
||||
- alert: KubeDaemonSetUnavailable
|
||||
expr: (max by(namespace,daemonset) (kube_daemonset_status_desired_number_scheduled{job="kube-state-metrics"})
|
||||
- max by(namespace,daemonset) (kube_daemonset_status_number_ready{job="kube-state-metrics"})) > 0
|
||||
for: 10m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: DaemonSet {{ $labels.namespace }}/{{ $labels.daemonset }} 副本不足
|
||||
description: 检查节点和 Pod 就绪；不按整个 namespace 屏蔽。
|
||||
runbook_url: https://git.ddupan.top/panxiao81/homelab-wiki/src/branch/main/guides/monitoring-foundation.md
|
||||
- alert: KubePVCPending
|
||||
expr: max by(namespace,persistentvolumeclaim) (kube_persistentvolumeclaim_status_phase{job="kube-state-metrics",phase="Pending"})
|
||||
== 1
|
||||
for: 15m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: PVC {{ $labels.namespace }}/{{ $labels.persistentvolumeclaim }} 长时间 Pending
|
||||
description: 检查 StorageClass、调度拓扑、容量与 provisioner。
|
||||
runbook_url: https://git.ddupan.top/panxiao81/homelab-wiki/src/branch/main/guides/monitoring-foundation.md
|
||||
- alert: KubeJobFailed
|
||||
expr: max by(namespace,job_name) (kube_job_failed{job="kube-state-metrics",condition="true"}) == 1
|
||||
for: 5m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: Job {{ $labels.namespace }}/{{ $labels.job_name }} 已失败
|
||||
description: 检查失败 Job 的 Pod 和任务日志；成功重试中的失败 Pod 计数不作为失败 Job。
|
||||
runbook_url: https://git.ddupan.top/panxiao81/homelab-wiki/src/branch/main/guides/monitoring-foundation.md
|
||||
@@ -0,0 +1,58 @@
|
||||
apiVersion: operator.victoriametrics.com/v1beta1
|
||||
kind: VMRule
|
||||
metadata:
|
||||
name: monitoring-delivery
|
||||
namespace: monitoring
|
||||
labels:
|
||||
app.kubernetes.io/part-of: victoria-metrics
|
||||
spec:
|
||||
groups:
|
||||
- name: monitoring-delivery
|
||||
interval: 30s
|
||||
rules:
|
||||
- alert: KubeStateMetricsUnavailable
|
||||
expr: up{job="kube-state-metrics"} == 0 or absent(up{job="kube-state-metrics"})
|
||||
for: 5m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: kube-state-metrics 采集不可用
|
||||
description: 检查 HelmRelease、ServiceMonitor 转换、targets 与 exporter；目标消失也会触发。
|
||||
runbook_url: https://git.ddupan.top/panxiao81/homelab-wiki/src/branch/main/guides/monitoring-foundation.md
|
||||
- alert: NodeExporterUnavailable
|
||||
expr: up{job="node-exporter"} == 0 or absent(up{job="node-exporter"})
|
||||
for: 5m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: node-exporter 采集不可用
|
||||
description: 检查节点与 exporter，主机健康规则依赖此采集。
|
||||
runbook_url: https://git.ddupan.top/panxiao81/homelab-wiki/src/branch/main/guides/monitoring-foundation.md
|
||||
- alert: NatsMetricsUnavailable
|
||||
expr: up{job="nats/nats"} == 0 or absent(up{job="nats/nats"})
|
||||
for: 5m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: NATS 指标采集不可用
|
||||
description: 依次检查 PodMonitor、转换后的 VMPodScrape、:7777 exporter 及 targets。
|
||||
runbook_url: https://git.ddupan.top/panxiao81/homelab-wiki/src/branch/main/guides/monitoring-foundation.md
|
||||
- alert: AlertmanagerTelegramDeliveryFailed
|
||||
expr: sum by(job,instance,integration) (increase(alertmanager_notifications_failed_total{job="vmalertmanager-main",integration="telegram"}[5m]))
|
||||
> 0
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: Alertmanager 向 Telegram 发送失败
|
||||
description: 检查网络、bot token 和频道权限。若 Telegram 完全不可达，本告警也无法经同一渠道送达；需在 Alertmanager/Grafana 排查，集群外心跳另行建设。
|
||||
runbook_url: https://git.ddupan.top/panxiao81/homelab-wiki/src/branch/main/guides/monitoring-foundation.md
|
||||
- alert: AlertmanagerConfigurationReloadFailed
|
||||
expr: alertmanager_config_last_reload_successful{job="vmalertmanager-main"} == 0
|
||||
for: 5m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: Alertmanager 配置加载失败
|
||||
description: 检查 operator 和 Alertmanager 日志、配置格式及 Secret 引用。
|
||||
runbook_url: https://git.ddupan.top/panxiao81/homelab-wiki/src/branch/main/guides/monitoring-foundation.md
|
||||
@@ -0,0 +1,20 @@
|
||||
#!/usr/bin/env bash
|
||||
# 依赖 Python 3 + PyYAML，以及 PATH 中的 promtool（验证版本 3.5.0）。
|
||||
set -euo pipefail
|
||||
test_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"
|
||||
check_dir="$(mktemp -d)"
|
||||
trap 'rm -rf "$check_dir"' EXIT
|
||||
python3 - "$test_dir/../rules" "$check_dir/rules.yaml" <<'PY'
|
||||
import pathlib
|
||||
import sys
|
||||
import yaml
|
||||
|
||||
groups = []
|
||||
for name in ("kubernetes-health", "host-health", "monitoring-delivery"):
|
||||
path = pathlib.Path(sys.argv[1]) / f"{name}.yaml"
|
||||
groups.extend(yaml.safe_load(path.read_text())["spec"]["groups"])
|
||||
pathlib.Path(sys.argv[2]).write_text(yaml.safe_dump({"groups": groups}, allow_unicode=True))
|
||||
PY
|
||||
cp "$test_dir/foundation.test.yaml" "$check_dir/foundation.test.yaml"
|
||||
promtool check rules "$check_dir/rules.yaml"
|
||||
promtool test rules "$check_dir/foundation.test.yaml"
|
||||
@@ -0,0 +1,137 @@
|
||||
rule_files:
|
||||
- rules.yaml
|
||||
evaluation_interval: 1m
|
||||
tests:
|
||||
- name: 健康 ZFS 的零值故障状态不报警
|
||||
interval: 1m
|
||||
input_series:
|
||||
- series: node_zfs_zpool_state{job="node-exporter",instance="laptop",zpool="data",state="degraded"}
|
||||
values: '0x20'
|
||||
- series: node_zfs_zpool_state{job="node-exporter",instance="laptop",zpool="data",state="online"}
|
||||
values: 1x20
|
||||
alert_rule_test:
|
||||
- eval_time: 10m
|
||||
alertname: HostZpoolUnhealthy
|
||||
exp_alerts: &id001 []
|
||||
- name: ZFS 降级持续五分钟触发
|
||||
interval: 1m
|
||||
input_series:
|
||||
- series: node_zfs_zpool_state{job="node-exporter",instance="laptop",zpool="data",state="degraded"}
|
||||
values: 1x20
|
||||
alert_rule_test:
|
||||
- eval_time: 4m
|
||||
alertname: HostZpoolUnhealthy
|
||||
exp_alerts: *id001
|
||||
- eval_time: 6m
|
||||
alertname: HostZpoolUnhealthy
|
||||
exp_alerts:
|
||||
- exp_labels:
|
||||
job: node-exporter
|
||||
instance: laptop
|
||||
zpool: data
|
||||
state: degraded
|
||||
severity: critical
|
||||
exp_annotations:
|
||||
summary: 主机 laptop ZFS 池 data 状态 degraded
|
||||
description: 检查 zpool status、磁盘和冗余；零值状态不触发。
|
||||
runbook_url: https://git.ddupan.top/panxiao81/homelab-wiki/src/branch/main/guides/monitoring-foundation.md
|
||||
- name: 忽略旧 docker-hosts 重复主机指标
|
||||
interval: 1m
|
||||
input_series:
|
||||
- series: node_memory_MemAvailable_bytes{job="node-exporter",instance="laptop"}
|
||||
values: 50x20
|
||||
- series: node_memory_MemTotal_bytes{job="node-exporter",instance="laptop"}
|
||||
values: 100x20
|
||||
- series: node_memory_MemAvailable_bytes{job="docker-hosts",instance="laptop"}
|
||||
values: 1x20
|
||||
- series: node_memory_MemTotal_bytes{job="docker-hosts",instance="laptop"}
|
||||
values: 100x20
|
||||
alert_rule_test:
|
||||
- eval_time: 15m
|
||||
alertname: HostMemoryLow
|
||||
exp_alerts: *id001
|
||||
- name: 临时文件系统耗尽不触发持久磁盘告警
|
||||
interval: 1m
|
||||
input_series:
|
||||
- series: node_filesystem_avail_bytes{job="node-exporter",instance="laptop",device="tmpfs",mountpoint="/run",fstype="tmpfs"}
|
||||
values: 1x20
|
||||
- series: node_filesystem_size_bytes{job="node-exporter",instance="laptop",device="tmpfs",mountpoint="/run",fstype="tmpfs"}
|
||||
values: 100x20
|
||||
- series: node_filesystem_readonly{job="node-exporter",instance="laptop",device="tmpfs",mountpoint="/run",fstype="tmpfs"}
|
||||
values: '0x20'
|
||||
alert_rule_test:
|
||||
- eval_time: 20m
|
||||
alertname: HostFilesystemSpaceLow
|
||||
exp_alerts: *id001
|
||||
- name: Deployment 允许短暂滚动且缩零不报错
|
||||
interval: 1m
|
||||
input_series:
|
||||
- series: kube_deployment_spec_replicas{job="kube-state-metrics",namespace="app",deployment="web"}
|
||||
values: 2x20
|
||||
- series: kube_deployment_status_replicas_available{job="kube-state-metrics",namespace="app",deployment="web"}
|
||||
values: 1x20
|
||||
- series: kube_deployment_spec_replicas{job="kube-state-metrics",namespace="app",deployment="scaled-down"}
|
||||
values: '0x20'
|
||||
- series: kube_deployment_status_replicas_available{job="kube-state-metrics",namespace="app",deployment="scaled-down"}
|
||||
values: '0x20'
|
||||
alert_rule_test:
|
||||
- eval_time: 5m
|
||||
alertname: KubeDeploymentUnavailable
|
||||
exp_alerts: *id001
|
||||
- eval_time: 11m
|
||||
alertname: KubeDeploymentUnavailable
|
||||
exp_alerts:
|
||||
- exp_labels:
|
||||
namespace: app
|
||||
deployment: web
|
||||
severity: warning
|
||||
exp_annotations:
|
||||
summary: Deployment app/web 副本不足
|
||||
description: 可用副本持续 10 分钟低于声明值；缩容至零不会触发。
|
||||
runbook_url: https://git.ddupan.top/panxiao81/homelab-wiki/src/branch/main/guides/monitoring-foundation.md
|
||||
- name: NATS 发现集合完全消失也报警
|
||||
interval: 1m
|
||||
input_series: []
|
||||
alert_rule_test:
|
||||
- eval_time: 6m
|
||||
alertname: NatsMetricsUnavailable
|
||||
exp_alerts:
|
||||
- exp_labels:
|
||||
job: nats/nats
|
||||
severity: warning
|
||||
exp_annotations:
|
||||
summary: NATS 指标采集不可用
|
||||
description: 依次检查 PodMonitor、转换后的 VMPodScrape、:7777 exporter 及 targets。
|
||||
runbook_url: https://git.ddupan.top/panxiao81/homelab-wiki/src/branch/main/guides/monitoring-foundation.md
|
||||
- name: 历史 OOM 没有新重启不反复报警
|
||||
interval: 1m
|
||||
input_series:
|
||||
- series: kube_pod_container_status_last_terminated_reason{job="kube-state-metrics",namespace="app",pod="web",container="web",reason="OOMKilled"}
|
||||
values: 1x20
|
||||
- series: kube_pod_container_status_restarts_total{job="kube-state-metrics",namespace="app",pod="web",container="web"}
|
||||
values: 5x20
|
||||
alert_rule_test:
|
||||
- eval_time: 10m
|
||||
alertname: KubePodOOMKilled
|
||||
exp_alerts: *id001
|
||||
- name: Telegram 失败跨 reason 汇总为一条
|
||||
interval: 1m
|
||||
input_series:
|
||||
- series: alertmanager_notifications_failed_total{job="vmalertmanager-main",instance="am",integration="telegram",reason="clientError"}
|
||||
values: 0+1x20
|
||||
- series: alertmanager_notifications_failed_total{job="vmalertmanager-main",instance="am",integration="telegram",reason="serverError"}
|
||||
values: 0+1x20
|
||||
alert_rule_test:
|
||||
- eval_time: 6m
|
||||
alertname: AlertmanagerTelegramDeliveryFailed
|
||||
exp_alerts:
|
||||
- exp_labels:
|
||||
job: vmalertmanager-main
|
||||
instance: am
|
||||
integration: telegram
|
||||
severity: critical
|
||||
exp_annotations:
|
||||
summary: Alertmanager 向 Telegram 发送失败
|
||||
description: 检查网络、bot token 和频道权限。若 Telegram 完全不可达，本告警也无法经同一渠道送达；需在 Alertmanager/Grafana
|
||||
排查，集群外心跳另行建设。
|
||||
runbook_url: https://git.ddupan.top/panxiao81/homelab-wiki/src/branch/main/guides/monitoring-foundation.md
|
||||
@@ -24,6 +24,12 @@ spec:
|
||||
- receiver: telegram
|
||||
matchers:
|
||||
- severity="warning"
|
||||
inhibit_rules:
|
||||
- source_matchers:
|
||||
- alertname="KubePodCrashLooping"
|
||||
target_matchers:
|
||||
- alertname="KubePodFrequentRestarts"
|
||||
equal: [namespace, pod, container]
|
||||
receivers:
|
||||
- name: blackhole
|
||||
- name: telegram
|
||||
|
||||
@@ -4,6 +4,8 @@ metadata:
|
||||
name: vm-operator
|
||||
namespace: monitoring
|
||||
spec:
|
||||
dependsOn:
|
||||
- name: prometheus-operator-crds
|
||||
chart:
|
||||
spec:
|
||||
chart: victoria-metrics-operator
|
||||
|
||||
Reference in New Issue
Block a user