Establish clean homelab infrastructure baseline
Reorganize the brownfield repository, remove retired and generated artifacts, harden ignore rules, and record the GitOps/IaC redesign.
This commit is contained in:
@@ -0,0 +1,137 @@
|
||||
# Ported from ../../../../apps/victoriametrics/rules/alerts-vmagent.yml
|
||||
apiVersion: operator.victoriametrics.com/v1beta1
|
||||
kind: VMRule
|
||||
metadata:
|
||||
name: vmagent
|
||||
namespace: monitoring
|
||||
labels:
|
||||
app.kubernetes.io/part-of: victoria-metrics
|
||||
spec:
|
||||
groups:
|
||||
- name: vmagent
|
||||
interval: 30s
|
||||
concurrency: 2
|
||||
rules:
|
||||
- alert: PersistentQueueIsDroppingData
|
||||
expr: sum(increase(vm_persistentqueue_bytes_dropped_total[5m])) without (path) > 0
|
||||
for: 10m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "Instance {{ $labels.instance }} is dropping data from persistent queue"
|
||||
description: "Vmagent dropped {{ $value | humanize1024 }} from persistent queue
|
||||
on instance {{ $labels.instance }} for the last 10m."
|
||||
- alert: RejectedRemoteWriteDataBlocksAreDropped
|
||||
expr: sum(increase(vmagent_remotewrite_packets_dropped_total[5m])) without (url) > 0
|
||||
for: 15m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "Vmagent is dropping data blocks that are rejected by remote storage"
|
||||
description: "Job \"{{ $labels.job }}\" on instance {{ $labels.instance }} drops the rejected by
|
||||
remote-write server data blocks. Check the logs to find the reason for rejects."
|
||||
- alert: TooManyScrapeErrors
|
||||
expr: increase(vm_promscrape_scrapes_failed_total[5m]) > 0
|
||||
for: 15m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "Vmagent fails to scrape one or more targets"
|
||||
description: "Job \"{{ $labels.job }}\" on instance {{ $labels.instance }} fails to scrape targets for last 15m"
|
||||
- alert: ScrapePoolHasNoTargets
|
||||
expr: sum(vm_promscrape_scrape_pool_targets) without (status, instance, pod) == 0
|
||||
for: 30m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "Vmagent has scrape_pool with 0 configured/discovered targets"
|
||||
description: "Vmagent \"{{ $labels.job }}\" has scrape_pool \"{{ $labels.scrape_job }}\"
|
||||
with 0 discovered targets. It is likely a misconfiguration."
|
||||
- alert: TooManyWriteErrors
|
||||
expr: |
|
||||
(sum(increase(vm_ingestserver_request_errors_total[5m])) without (name,net,type)
|
||||
+
|
||||
sum(increase(vmagent_http_request_errors_total[5m])) without (path,protocol)) > 0
|
||||
for: 15m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "Vmagent responds with too many errors on data ingestion protocols"
|
||||
description: "Job \"{{ $labels.job }}\" on instance {{ $labels.instance }} responds with errors to write requests for last 15m."
|
||||
- alert: TooManyRemoteWriteErrors
|
||||
expr: rate(vmagent_remotewrite_retries_count_total[5m]) > 0
|
||||
for: 15m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "Job \"{{ $labels.job }}\" on instance {{ $labels.instance }} fails to push to remote storage"
|
||||
description: "Vmagent fails to push data via remote write protocol to destination \"{{ $labels.url }}\".
|
||||
Ensure that destination is up and reachable."
|
||||
- alert: RemoteWriteConnectionIsSaturated
|
||||
expr: |
|
||||
(
|
||||
rate(vmagent_remotewrite_send_duration_seconds_total[5m])
|
||||
/
|
||||
vmagent_remotewrite_queues
|
||||
) > 0.9
|
||||
for: 15m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "Remote write connection from \"{{ $labels.job }}\" (instance {{ $labels.instance }}) to {{ $labels.url }} is saturated"
|
||||
description: "The remote write connection between vmagent and destination \"{{ $labels.url }}\"
|
||||
is saturated by more than 90%. Increase -remoteWrite.queues or check the destination's capacity."
|
||||
- alert: PersistentQueueForWritesIsSaturated
|
||||
expr: rate(vm_persistentqueue_write_duration_seconds_total[5m]) > 0.9
|
||||
for: 15m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "Persistent queue writes for instance {{ $labels.instance }} are saturated"
|
||||
description: "Persistent queue writes are saturated by more than 90%. Reduce load or improve disk throughput."
|
||||
- alert: PersistentQueueForReadsIsSaturated
|
||||
expr: rate(vm_persistentqueue_read_duration_seconds_total[5m]) > 0.9
|
||||
for: 15m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "Persistent queue reads for instance {{ $labels.instance }} are saturated"
|
||||
description: "Persistent queue reads are saturated by more than 90%. Reduce load or improve disk throughput."
|
||||
- alert: SeriesLimitHourReached
|
||||
expr: (vmagent_hourly_series_limit_current_series / vmagent_hourly_series_limit_max_series) > 0.9
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "Instance {{ $labels.instance }} reached 90% of the hourly series limit"
|
||||
description: "Max series limit set via -remoteWrite.maxHourlySeries is close to the max value."
|
||||
- alert: SeriesLimitDayReached
|
||||
expr: (vmagent_daily_series_limit_current_series / vmagent_daily_series_limit_max_series) > 0.9
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "Instance {{ $labels.instance }} reached 90% of the daily series limit"
|
||||
description: "Max series limit set via -remoteWrite.maxDailySeries is close to the max value."
|
||||
- alert: ConfigurationReloadFailure
|
||||
expr: |
|
||||
vm_promscrape_config_last_reload_successful != 1
|
||||
or
|
||||
vmagent_relabel_config_last_reload_successful != 1
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "Configuration reload failed for vmagent instance {{ $labels.instance }}"
|
||||
description: "Configuration hot-reload failed for vmagent on instance {{ $labels.instance }}. Check the logs."
|
||||
- alert: StreamAggrFlushTimeout
|
||||
expr: increase(vm_streamaggr_flush_timeouts_total[5m]) > 0
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "Streaming aggregation at \"{{ $labels.job }}\" (instance {{ $labels.instance }}) can't keep up."
|
||||
description: "Stream aggregation can't finish within the configured interval and may produce incorrect results."
|
||||
- alert: StreamAggrDedupFlushTimeout
|
||||
expr: increase(vm_streamaggr_dedup_flush_timeouts_total[5m]) > 0
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "Deduplication \"{{ $labels.job }}\" (instance {{ $labels.instance }}) can't keep up."
|
||||
description: "Deduplication can't finish within the configured interval and may produce incorrect results."
|
||||
Reference in New Issue
Block a user