Establish clean homelab infrastructure baseline

Reorganize the brownfield repository, remove retired and generated artifacts, harden ignore rules, and record the GitOps/IaC redesign.
2026-09-09 16:47:20 +00:00
commit 88a02ababa
418 changed files with 50579 additions and 0 deletions
@@ -0,0 +1,127 @@
+# Ported from ../../../../apps/victoriametrics/rules/alerts-health.yml
+apiVersion: operator.victoriametrics.com/v1beta1
+kind: VMRule
+metadata:
+  name: vm-health
+  namespace: monitoring
+  labels:
+    app.kubernetes.io/part-of: victoria-metrics
+spec:
+  groups:
+    - name: vm-health
+      rules:
+        - alert: TooManyRestarts
+          expr: changes(process_start_time_seconds{job=~".*(victoriametrics|vmselect|vminsert|vmstorage|vmagent|vmalert|vmsingle|vmalertmanager|vmauth).*"}[15m]) > 2
+          labels:
+            severity: critical
+          annotations:
+            summary: "{{ $labels.job }} too many restarts (instance {{ $labels.instance }})"
+            description: >
+              Job {{ $labels.job }} (instance {{ $labels.instance }}) has restarted more than twice in the last 15 minutes.
+              It might be crashlooping.
+        - alert: ServiceDown
+          expr: up{job=~".*(victoriametrics|vmselect|vminsert|vmstorage|vmagent|vmalert|vmsingle|vmalertmanager|vmauth).*"} == 0
+          for: 2m
+          labels:
+            severity: critical
+          annotations:
+            summary: "Service {{ $labels.job }} is down on {{ $labels.instance }}"
+            description: "{{ $labels.instance }} of job {{ $labels.job }} has been down for more than 2 minutes."
+        - alert: ProcessNearFDLimits
+          expr: (process_max_fds - process_open_fds) < 100
+          for: 5m
+          labels:
+            severity: critical
+          annotations:
+            summary: "Number of free file descriptors is less than 100 for \"{{ $labels.job }}\"(\"{{ $labels.instance }}\") for the last 5m"
+            description: |
+              Exhausting OS file descriptors limit can cause severe degradation of the process.
+              Consider to increase the limit as fast as possible.
+        - alert: TooHighMemoryUsage
+          expr: (min_over_time(process_resident_memory_anon_bytes[10m]) / vm_available_memory_bytes) > 0.8
+          for: 5m
+          labels:
+            severity: critical
+          annotations:
+            summary: "It is more than 80% of memory used by \"{{ $labels.job }}\"(\"{{ $labels.instance }}\")"
+            description: |
+              Too high memory usage may result into multiple issues such as OOMs or degraded performance.
+              Consider to either increase available memory or decrease the load on the process.
+        - alert: TooHighCPUUsage
+          expr: rate(process_cpu_seconds_total[5m]) / process_cpu_cores_available > 0.9
+          for: 5m
+          labels:
+            severity: critical
+          annotations:
+            summary: "More than 90% of CPU is used by \"{{ $labels.job }}\"(\"{{ $labels.instance }}\") during the last 5m"
+            description: >
+              Too high CPU usage may be a sign of insufficient resources and make process unstable.
+              Consider to either increase available CPU resources or decrease the load on the process.
+        - alert: TooHighGoroutineSchedulingLatency
+          expr: histogram_quantile(0.99, sum(rate(go_sched_latencies_seconds_bucket{job=~".*(victoriametrics|vmselect|vminsert|vmstorage|vmagent|vmalert|vmsingle|vmalertmanager|vmauth).*"}[5m])) by (le, job, instance)) > 0.1
+          for: 15m
+          labels:
+            severity: critical
+          annotations:
+            summary: "\"{{ $labels.job }}\"(\"{{ $labels.instance }}\") has insufficient CPU resources for >15m"
+            description: >
+              Go runtime is unable to schedule goroutines execution in acceptable time. This is usually a sign of
+              insufficient CPU resources or CPU throttling.
+        - alert: TooManyLogs
+          expr: sum(increase(vm_log_messages_total{level="error"}[5m])) without (app_version, location) > 0
+          for: 15m
+          labels:
+            severity: warning
+          annotations:
+            summary: "Too many logs printed for job \"{{ $labels.job }}\" ({{ $labels.instance }})"
+            description: >
+              Logging rate for job \"{{ $labels.job }}\" ({{ $labels.instance }}) is {{ $value }} for last 15m.
+              Worth to check logs for specific error messages.
+        - alert: TooManyTSIDMisses
+          expr: increase(vm_missing_tsids_for_metric_id_total[5m]) > 0
+          for: 15m
+          labels:
+            severity: critical
+          annotations:
+            summary: "Unexpected TSID misses for job \"{{ $labels.job }}\" ({{ $labels.instance }}) for the last 15 minutes"
+            description: |
+              Unexpected TSID misses for \"{{ $labels.job }}\" ({{ $labels.instance }}) for the last 15 minutes.
+              If this happens after unclean shutdown of VictoriaMetrics process (via \"kill -9\", OOM or power off),
+              then this is OK - the alert must go away in a few minutes after the restart.
+        - alert: ConcurrentInsertsHitTheLimit
+          expr: avg_over_time(vm_concurrent_insert_current[1m]) >= vm_concurrent_insert_capacity
+          for: 15m
+          labels:
+            severity: warning
+          annotations:
+            summary: "{{ $labels.job }} on instance {{ $labels.instance }} is constantly hitting concurrent inserts limit"
+            description: |
+              The limit of concurrent inserts on instance {{ $labels.instance }} depends on the number of CPUs.
+              Usually, when component constantly hits the limit it is likely the component is overloaded and requires more CPU.
+        - alert: IndexDBRecordsDrop
+          expr: increase(vm_indexdb_items_dropped_total[5m]) > 0
+          labels:
+            severity: critical
+          annotations:
+            summary: "IndexDB skipped registering items during data ingestion with reason={{ $labels.reason }}."
+            description: |
+              VictoriaMetrics could skip registering new timeseries during ingestion if they fail the validation process.
+        - alert: RowsRejectedOnIngestion
+          expr: rate(vm_rows_ignored_total[5m]) > 0
+          for: 15m
+          labels:
+            severity: warning
+          annotations:
+            summary: "Some rows are rejected on \"{{ $labels.instance }}\" on ingestion attempt"
+            description: "Ingested rows on instance \"{{ $labels.instance }}\" are rejected due to the
+              following reason: \"{{ $labels.reason }}\""
+        - alert: TooHighQueryLoad
+          expr: increase(vm_concurrent_select_limit_timeout_total[5m]) > 0
+          for: 15m
+          labels:
+            severity: warning
+          annotations:
+            summary: "Read queries fail with timeout for {{ $labels.job }} on instance {{ $labels.instance }}"
+            description: |
+              Instance {{ $labels.instance }} ({{ $labels.job }}) is failing to serve read queries during last 15m.
+              Possible solutions: reduce the query load; increase compute resources; adjust search concurrency limits.
@@ -0,0 +1,137 @@
+# Ported from ../../../../apps/victoriametrics/rules/alerts-vmagent.yml
+apiVersion: operator.victoriametrics.com/v1beta1
+kind: VMRule
+metadata:
+  name: vmagent
+  namespace: monitoring
+  labels:
+    app.kubernetes.io/part-of: victoria-metrics
+spec:
+  groups:
+    - name: vmagent
+      interval: 30s
+      concurrency: 2
+      rules:
+        - alert: PersistentQueueIsDroppingData
+          expr: sum(increase(vm_persistentqueue_bytes_dropped_total[5m])) without (path) > 0
+          for: 10m
+          labels:
+            severity: critical
+          annotations:
+            summary: "Instance {{ $labels.instance }} is dropping data from persistent queue"
+            description: "Vmagent dropped {{ $value | humanize1024 }} from persistent queue
+                on instance {{ $labels.instance }} for the last 10m."
+        - alert: RejectedRemoteWriteDataBlocksAreDropped
+          expr: sum(increase(vmagent_remotewrite_packets_dropped_total[5m])) without (url) > 0
+          for: 15m
+          labels:
+            severity: warning
+          annotations:
+            summary: "Vmagent is dropping data blocks that are rejected by remote storage"
+            description: "Job \"{{ $labels.job }}\" on instance {{ $labels.instance }} drops the rejected by
+              remote-write server data blocks. Check the logs to find the reason for rejects."
+        - alert: TooManyScrapeErrors
+          expr: increase(vm_promscrape_scrapes_failed_total[5m]) > 0
+          for: 15m
+          labels:
+            severity: warning
+          annotations:
+            summary: "Vmagent fails to scrape one or more targets"
+            description: "Job \"{{ $labels.job }}\" on instance {{ $labels.instance }} fails to scrape targets for last 15m"
+        - alert: ScrapePoolHasNoTargets
+          expr: sum(vm_promscrape_scrape_pool_targets) without (status, instance, pod) == 0
+          for: 30m
+          labels:
+            severity: warning
+          annotations:
+            summary: "Vmagent has scrape_pool with 0 configured/discovered targets"
+            description: "Vmagent \"{{ $labels.job }}\" has scrape_pool \"{{ $labels.scrape_job }}\"
+              with 0 discovered targets. It is likely a misconfiguration."
+        - alert: TooManyWriteErrors
+          expr: |
+            (sum(increase(vm_ingestserver_request_errors_total[5m])) without (name,net,type)
+            +
+            sum(increase(vmagent_http_request_errors_total[5m])) without (path,protocol)) > 0
+          for: 15m
+          labels:
+            severity: warning
+          annotations:
+            summary: "Vmagent responds with too many errors on data ingestion protocols"
+            description: "Job \"{{ $labels.job }}\" on instance {{ $labels.instance }} responds with errors to write requests for last 15m."
+        - alert: TooManyRemoteWriteErrors
+          expr: rate(vmagent_remotewrite_retries_count_total[5m]) > 0
+          for: 15m
+          labels:
+            severity: warning
+          annotations:
+            summary: "Job \"{{ $labels.job }}\" on instance {{ $labels.instance }} fails to push to remote storage"
+            description: "Vmagent fails to push data via remote write protocol to destination \"{{ $labels.url }}\".
+              Ensure that destination is up and reachable."
+        - alert: RemoteWriteConnectionIsSaturated
+          expr: |
+            (
+             rate(vmagent_remotewrite_send_duration_seconds_total[5m])
+             /
+             vmagent_remotewrite_queues
+            ) > 0.9
+          for: 15m
+          labels:
+            severity: warning
+          annotations:
+            summary: "Remote write connection from \"{{ $labels.job }}\" (instance {{ $labels.instance }}) to {{ $labels.url }} is saturated"
+            description: "The remote write connection between vmagent and destination \"{{ $labels.url }}\"
+              is saturated by more than 90%. Increase -remoteWrite.queues or check the destination's capacity."
+        - alert: PersistentQueueForWritesIsSaturated
+          expr: rate(vm_persistentqueue_write_duration_seconds_total[5m]) > 0.9
+          for: 15m
+          labels:
+            severity: warning
+          annotations:
+            summary: "Persistent queue writes for instance {{ $labels.instance }} are saturated"
+            description: "Persistent queue writes are saturated by more than 90%. Reduce load or improve disk throughput."
+        - alert: PersistentQueueForReadsIsSaturated
+          expr: rate(vm_persistentqueue_read_duration_seconds_total[5m]) > 0.9
+          for: 15m
+          labels:
+            severity: warning
+          annotations:
+            summary: "Persistent queue reads for instance {{ $labels.instance }} are saturated"
+            description: "Persistent queue reads are saturated by more than 90%. Reduce load or improve disk throughput."
+        - alert: SeriesLimitHourReached
+          expr: (vmagent_hourly_series_limit_current_series / vmagent_hourly_series_limit_max_series) > 0.9
+          labels:
+            severity: critical
+          annotations:
+            summary: "Instance {{ $labels.instance }} reached 90% of the hourly series limit"
+            description: "Max series limit set via -remoteWrite.maxHourlySeries is close to the max value."
+        - alert: SeriesLimitDayReached
+          expr: (vmagent_daily_series_limit_current_series / vmagent_daily_series_limit_max_series) > 0.9
+          labels:
+            severity: critical
+          annotations:
+            summary: "Instance {{ $labels.instance }} reached 90% of the daily series limit"
+            description: "Max series limit set via -remoteWrite.maxDailySeries is close to the max value."
+        - alert: ConfigurationReloadFailure
+          expr: |
+            vm_promscrape_config_last_reload_successful != 1
+            or
+            vmagent_relabel_config_last_reload_successful != 1
+          labels:
+            severity: warning
+          annotations:
+            summary: "Configuration reload failed for vmagent instance {{ $labels.instance }}"
+            description: "Configuration hot-reload failed for vmagent on instance {{ $labels.instance }}. Check the logs."
+        - alert: StreamAggrFlushTimeout
+          expr: increase(vm_streamaggr_flush_timeouts_total[5m]) > 0
+          labels:
+            severity: warning
+          annotations:
+            summary: "Streaming aggregation at \"{{ $labels.job }}\" (instance {{ $labels.instance }}) can't keep up."
+            description: "Stream aggregation can't finish within the configured interval and may produce incorrect results."
+        - alert: StreamAggrDedupFlushTimeout
+          expr: increase(vm_streamaggr_dedup_flush_timeouts_total[5m]) > 0
+          labels:
+            severity: warning
+          annotations:
+            summary: "Deduplication \"{{ $labels.job }}\" (instance {{ $labels.instance }}) can't keep up."
+            description: "Deduplication can't finish within the configured interval and may produce incorrect results."
@@ -0,0 +1,77 @@
+# Ported from ../../../../apps/victoriametrics/rules/alerts-vmalert.yml
+apiVersion: operator.victoriametrics.com/v1beta1
+kind: VMRule
+metadata:
+  name: vmalert
+  namespace: monitoring
+  labels:
+    app.kubernetes.io/part-of: victoria-metrics
+spec:
+  groups:
+    - name: vmalert
+      interval: 30s
+      rules:
+        - alert: ConfigurationReloadFailure
+          expr: vmalert_config_last_reload_successful != 1
+          labels:
+            severity: warning
+          annotations:
+            summary: "Configuration reload failed for vmalert instance {{ $labels.instance }}"
+            description: "Configuration hot-reload failed for vmalert on instance {{ $labels.instance }}. Check the logs."
+        - alert: AlertingRulesError
+          expr: sum(increase(vmalert_alerting_rules_errors_total[5m])) without(id) > 0
+          for: 5m
+          labels:
+            severity: warning
+          annotations:
+            summary: "Alerting rules are failing for vmalert instance {{ $labels.instance }}"
+            description: "Alerting rules execution is failing for group \"{{ $labels.group }}\" in file \"{{ $labels.file }}\". Check the logs."
+        - alert: RecordingRulesError
+          expr: sum(increase(vmalert_recording_rules_errors_total[5m])) without(id) > 0
+          for: 5m
+          labels:
+            severity: warning
+          annotations:
+            summary: "Recording rules are failing for vmalert instance {{ $labels.instance }}"
+            description: "Recording rules execution is failing for group \"{{ $labels.group }}\" in file \"{{ $labels.file }}\". Check the logs."
+        - alert: RecordingRulesNoData
+          expr: sum(vmalert_recording_rules_last_evaluation_samples) without(id) < 1
+          for: 30m
+          labels:
+            severity: info
+          annotations:
+            summary: "Recording rule ({{ $labels.group }}) produces no data"
+            description: "Recording rule from group \"{{ $labels.group }}\" in file \"{{ $labels.file }}\"
+              produces 0 samples over the last 30min. Possible misconfiguration or incorrect query."
+        - alert: TooManyMissedIterations
+          expr: increase(vmalert_iteration_missed_total[5m]) > 0
+          for: 15m
+          labels:
+            severity: warning
+          annotations:
+            summary: "vmalert instance {{ $labels.instance }} is missing rules evaluations"
+            description: "Group evaluation takes longer than the configured interval. Increase interval or concurrency."
+        - alert: RemoteWriteErrors
+          expr: increase(vmalert_remotewrite_errors_total[5m]) > 0
+          for: 15m
+          labels:
+            severity: warning
+          annotations:
+            summary: "vmalert instance {{ $labels.instance }} is failing to push metrics to remote write URL"
+            description: "vmalert is failing to push alerting/recording rule metrics to remote write. Check the logs."
+        - alert: RemoteWriteDroppingData
+          expr: increase(vmalert_remotewrite_dropped_rows_total[5m]) > 0
+          for: 5m
+          labels:
+            severity: critical
+          annotations:
+            summary: "vmalert instance {{ $labels.instance }} is dropping data sent to remote write URL"
+            description: "vmalert is dropping alerting/recording rule results. This may cause gaps in state. Check the logs."
+        - alert: AlertmanagerErrors
+          expr: increase(vmalert_alerts_send_errors_total[5m]) > 0
+          for: 15m
+          labels:
+            severity: warning
+          annotations:
+            summary: "vmalert instance {{ $labels.instance }} is failing to send notifications to Alertmanager"
+            description: "vmalert is failing to send alert notifications to \"{{ $labels.addr }}\". Check the logs."
@@ -0,0 +1,127 @@
+# Ported from ../../../../apps/victoriametrics/rules/alerts.yml
+apiVersion: operator.victoriametrics.com/v1beta1
+kind: VMRule
+metadata:
+  name: vmsingle
+  namespace: monitoring
+  labels:
+    app.kubernetes.io/part-of: victoria-metrics
+spec:
+  groups:
+    - name: vmsingle
+      interval: 30s
+      concurrency: 2
+      rules:
+        - alert: DiskRunsOutOfSpaceIn3Days
+          expr: |
+            sum(vm_free_disk_space_bytes) without(path) /
+            (
+              (rate(vm_rows_added_to_storage_total[1d]) - sum(rate(vm_deduplicated_samples_total[1d])) without(type)) * (
+                sum(vm_data_size_bytes{type!~"indexdb.*"}) without(type) /
+                sum(vm_rows{type!~"indexdb.*"}) without(type)
+              )
+              +
+              rate(vm_new_timeseries_created_total[1d]) * (
+                sum(vm_data_size_bytes{type="indexdb/file"}) without(type)/
+                sum(vm_rows{type="indexdb/file"}) without(type)
+              )
+            ) < 3 * 24 * 3600 > 0
+          for: 30m
+          labels:
+            severity: critical
+          annotations:
+            summary: "Instance {{ $labels.instance }} will run out of disk space soon"
+            description: "Taking into account current ingestion rate, free disk space will be enough only
+              for {{ $value | humanizeDuration }} on instance {{ $labels.instance }}.\n
+              Consider to limit the ingestion rate, decrease retention or scale the disk space if possible."
+        - alert: NodeBecomesReadonlyIn3Days
+          expr: |
+            sum(vm_free_disk_space_bytes - vm_free_disk_space_limit_bytes) without(path) /
+            (
+                (rate(vm_rows_added_to_storage_total[1d]) - sum(rate(vm_deduplicated_samples_total[1d])) without(type)) * (
+                  sum(vm_data_size_bytes{type!~"indexdb.*"}) without(type) /
+                  sum(vm_rows{type!~"indexdb.*"}) without(type)
+                )
+                +
+                rate(vm_new_timeseries_created_total[1d]) * (
+                  sum(vm_data_size_bytes{type="indexdb/file"}) without(type) /
+                  sum(vm_rows{type="indexdb/file"}) without(type)
+                )
+            ) < 3 * 24 * 3600 > 0
+          for: 30m
+          labels:
+            severity: warning
+          annotations:
+            summary: "Instance {{ $labels.instance }} will become read-only in 3 days"
+            description: "Taking into account current ingestion rate and free disk space
+                instance {{ $labels.instance }} is writable for {{ $value | humanizeDuration }}.\n
+                Consider to limit the ingestion rate, decrease retention or scale the disk space up if possible."
+        - alert: DiskRunsOutOfSpace
+          expr: |
+            sum(vm_data_size_bytes) by(job, instance) /
+            (
+             sum(vm_free_disk_space_bytes) by(job, instance) +
+             sum(vm_data_size_bytes) by(job, instance)
+            ) > 0.8
+          for: 30m
+          labels:
+            severity: critical
+          annotations:
+            summary: "Instance {{ $labels.instance }} (job={{ $labels.job }}) will run out of disk space soon"
+            description: "Disk utilisation on instance {{ $labels.instance }} is more than 80%.\n
+              Having less than 20% of free disk space could cripple merge processes and overall performance.
+              Consider to limit the ingestion rate, decrease retention or scale the disk space if possible."
+        - alert: RequestErrorsToAPI
+          expr: increase(vm_http_request_errors_total[5m]) > 0
+          for: 15m
+          labels:
+            severity: warning
+          annotations:
+            summary: "Too many errors served for path {{ $labels.path }} (instance {{ $labels.instance }})"
+            description: "Requests to path {{ $labels.path }} are receiving errors.
+              Please verify if clients are sending correct requests."
+        - alert: TooHighChurnRate
+          expr: |
+            (
+               sum(rate(vm_new_timeseries_created_total[5m])) by(instance)
+               /
+               sum(rate(vm_rows_inserted_total[5m])) by(instance)
+             ) > 0.1
+          for: 15m
+          labels:
+            severity: warning
+          annotations:
+            summary: "Churn rate is more than 10% on \"{{ $labels.instance }}\" for the last 15m"
+            description: "VM constantly creates new time series on \"{{ $labels.instance }}\".\n
+              This effect is known as Churn Rate.\n
+              High Churn Rate is tightly connected with database performance and may
+              result in unexpected OOM's or slow queries."
+        - alert: TooHighChurnRate24h
+          expr: |
+            sum(increase(vm_new_timeseries_created_total[24h])) by(instance)
+            >
+            (sum(vm_cache_entries{type="storage/hour_metric_ids"}) by(instance) * 3)
+          for: 15m
+          labels:
+            severity: warning
+          annotations:
+            summary: "Too high number of new series on \"{{ $labels.instance }}\" created over last 24h"
+            description: "The number of created new time series over last 24h is 3x times higher than
+              current number of active series on \"{{ $labels.instance }}\".\n
+              This effect is known as Churn Rate.\n
+              High Churn Rate is tightly connected with database performance and may
+              result in unexpected OOM's or slow queries."
+        - alert: TooHighSlowInsertsRate
+          expr: |
+            (
+               sum(rate(vm_slow_row_inserts_total[5m])) by(instance)
+               /
+               sum(rate(vm_rows_inserted_total[5m])) by(instance)
+             ) > 0.05
+          for: 15m
+          labels:
+            severity: warning
+          annotations:
+            summary: "Percentage of slow inserts is more than 5% on \"{{ $labels.instance }}\" for the last 15m"
+            description: "High rate of slow inserts on \"{{ $labels.instance }}\" may be a sign of resource exhaustion
+              for the current load. It is likely more RAM is needed for optimal handling of the current number of active time series."