排除 OCI HelmRepository 正常缺失 Ready 条件的误报

Reviewed-on: #154

apps/backstage/deployment.yaml

@@ -5,6 +5,7 @@ metadata:
   namespace: backstage
   labels:
     app.kubernetes.io/name: backstage
+    backstage.io/kubernetes-id: homelab-backstage
 spec:
   replicas: 1
   strategy:
@@ -16,6 +17,7 @@ spec:
     metadata:
       labels:
         app.kubernetes.io/name: backstage
+        backstage.io/kubernetes-id: homelab-backstage
     spec:
       serviceAccountName: backstage
       securityContext:
@@ -25,7 +27,7 @@ spec:
           type: RuntimeDefault
       containers:
         - name: backstage
-          image: zot.ad.ddupan.top/panxiao81/backstage@sha256:6d6a11f8611a85237e3b0ffd30a2a6e119b1e5258b2ca6c193d60a8c3ec52553
+          image: zot.ad.ddupan.top/panxiao81/backstage@sha256:e5a12550726f19a680bc7c40e2cc07cc624318f9279ee121814d293a006ef210
           imagePullPolicy: IfNotPresent
           env:
             - name: BACKSTAGE_BASE_URL
@@ -40,8 +42,6 @@ spec:
               value: backstage
             - name: GITEA_HOST
               value: git.ddupan.top
-            - name: GITEA_ORG
-              value: panxiao81
           envFrom:
             - secretRef:
                 name: backstage
@@ -51,14 +51,14 @@ spec:
               protocol: TCP
           readinessProbe:
             httpGet:
-              path: /healthcheck
+              path: /.backstage/health/v1/readiness
               port: http
             initialDelaySeconds: 10
             periodSeconds: 10
             timeoutSeconds: 3
           livenessProbe:
             httpGet:
-              path: /healthcheck
+              path: /.backstage/health/v1/liveness
               port: http
             initialDelaySeconds: 30
             periodSeconds: 20

apps/backstage/kustomization.yaml

@@ -3,6 +3,7 @@ kind: Kustomization
 resources:
   - namespace.yaml
   - serviceaccount.yaml
+  - rbac.yaml
   - external-secret.yaml
   - deployment.yaml
   - service.yaml

apps/backstage/rbac.yaml

@@ -0,0 +1,51 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: backstage-read-only
+rules:
+  - apiGroups: [""]
+    resources:
+      - configmaps
+      - limitranges
+      - pods
+      - pods/log
+      - resourcequotas
+      - services
+    verbs: [get, list, watch]
+  - apiGroups: [apps]
+    resources:
+      - daemonsets
+      - deployments
+      - replicasets
+      - statefulsets
+    verbs: [get, list, watch]
+  - apiGroups: [autoscaling]
+    resources:
+      - horizontalpodautoscalers
+    verbs: [get, list, watch]
+  - apiGroups: [batch]
+    resources:
+      - cronjobs
+      - jobs
+    verbs: [get, list, watch]
+  - apiGroups: [networking.k8s.io]
+    resources:
+      - ingresses
+    verbs: [get, list, watch]
+  - apiGroups: [metrics.k8s.io]
+    resources:
+      - pods
+    verbs: [get, list]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: backstage-read-only
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: backstage-read-only
+subjects:
+  - kind: ServiceAccount
+    name: backstage
+    namespace: backstage

apps/backstage/service.yaml

@@ -3,6 +3,8 @@ kind: Service
 metadata:
   name: backstage
   namespace: backstage
+  labels:
+    backstage.io/kubernetes-id: homelab-backstage
 spec:
   selector:
     app.kubernetes.io/name: backstage

apps/backstage/serviceaccount.yaml

@@ -3,4 +3,4 @@ kind: ServiceAccount
 metadata:
   name: backstage
   namespace: backstage
-automountServiceAccountToken: false
+automountServiceAccountToken: true

platform/dynamic-runner/README.md

@@ -1,10 +1,10 @@
 # Gitea dynamic runner
 
 本目录部署单副本 Go controller，在同一进程运行 RunnerService scheduler、原生
-Kubernetes Pod worker、OpenSandbox VM worker 和 SPIFFE mTLS facade：
+Kubernetes worker、OpenSandbox worker 和 SPIFFE mTLS facade：
 
 ```text
-Gitea RunnerService -> scheduler -> JetStream ci.runner.pod
+Gitea RunnerService -> scheduler -> JetStream ci.runner.container.kubernetes
                                       |
                                       v
                               homelab Kubernetes Pod
@@ -15,7 +15,8 @@ Gitea RunnerService -> scheduler -> JetStream ci.runner.pod
                                      Gitea
 ```
 
-Pod backend 不经过 OpenSandbox。VM backend 后续启用时才访问 VyOS 暴露的
+`container+kubernetes` placement 不经过 OpenSandbox；`vm+opensandbox` placement
+访问 VyOS 暴露的
 OpenSandbox Lifecycle API；本目录不修改 sandbox 平台侧 ESO、Bao Terraform 或
 OpenSandbox chart 所有权边界。
 
@@ -82,9 +83,9 @@ kubectl -n dynamic-runner rollout status deploy/dynamic-runner-controller --time
 kubectl -n dynamic-runner logs deploy/dynamic-runner-controller -f
 ```
 
-确认 scheduler 只领取一条 `[self-hosted,pod]` task，然后验证：
+确认 scheduler 只领取一条 `[self-hosted,container,kubernetes]` task，然后验证：
 
-1. assignment message 进入并离开 durable `pod` consumer；
+1. assignment message 进入并离开 durable `container.kubernetes` consumer；
 2. `gitea-task-<task-id>` Pod 创建，取得实际 Pod UID；
 3. 同名 `ClusterStaticEntry` 的 parent ID 包含该 UID，SPIFFE ID 使用 repository/job key；
 4. 官方 runner v3.5.0 经 facade claim 精确 task，Gitea 实时收到日志和终态；
@@ -96,7 +97,8 @@ ClusterStaticEntry 全部消失；完整自动清理通过前不得提高 `POD_C
 `VM_CAPACITY`。
 
 VM backend 已通过 `vm-dev` canary 完成 Docker、kind、SPIFFE 和完整生命周期测试。
-生产调度标签为 `[self-hosted, vm]`，初始保持 `VM_CAPACITY=1`；扩容前先观察实际任务的
+生产规范标签为 `[self-hosted, vm, opensandbox]`，旧 `[self-hosted, vm]` 仍严格映射到
+同一 placement。初始保持 `VM_CAPACITY=1`；扩容前先观察实际任务的
 资源水位、等待时间以及 OpenSandbox 是否存在 terminal sandbox 残留。
 
 ## 回滚

platform/dynamic-runner/deployment.yaml

@@ -38,12 +38,12 @@ spec:
               mountPath: /trust
       containers:
         - name: controller
-          image: zot.ad.ddupan.top/panxiao81/gitea-dynamic-runner-controller@sha256:7374a08a238ca1c76a9aee33c57520ae0edaa9b5b443708487d5b3a69a61da4e
+          image: zot.ad.ddupan.top/panxiao81/gitea-dynamic-runner-controller@sha256:15735667e8990974a30d9a5e6a1030d63f791ed6d80fa2b0dc9e39959ffcf334
           imagePullPolicy: IfNotPresent
           args: [controller]
           env:
             - name: COMPONENTS
-              value: scheduler,pod-worker,vm-worker
+              value: scheduler,kubernetes-worker,opensandbox-worker
             - name: GITEA_INSTANCE_URL
               value: https://git.ddupan.top
             - name: GITEA_RUNNER_UUID_FILE
@@ -73,7 +73,7 @@ spec:
                 fieldRef:
                   fieldPath: metadata.namespace
             - name: POD_EXECUTOR_IMAGE
-              value: zot.ad.ddupan.top/panxiao81/gitea-dynamic-runner-runner@sha256:325d75a208b1a1c6f3b4d4705e47bbc58b34733edcd12f689c60769ab6772a59
+              value: zot.ad.ddupan.top/panxiao81/gitea-dynamic-runner-runner@sha256:83e96c3959af663a75a7229cb04cbbb97a642d4ef3704ac5cdceb1d502a72ed3
             - name: POD_SERVICE_ACCOUNT
               value: gitea-dynamic-runner
             - name: POD_EXECUTOR_UID

platform/nats/README.md

@@ -35,10 +35,11 @@ ci_worker_password
 
 ## CI stream 约定
 
-controller 首次启动时幂等创建 `CI_RUNNER` stream：`ci.runner.>`、
+controller 使用 `CI_RUNNER` stream：`ci.runner.>`、
 `WorkQueuePolicy`、file storage、24h/10000 条/256 MiB 上限。每类 runner 使用独立
-subject 和 durable pull consumer；`ci.runner.<backend>.binding` 传递 runner 实际
+placement subject 和 durable pull consumer；assignment v2 subject 为
-领取任务后的身份绑定。同类型的多个 worker 共享 durable consumer。ACK 后消息立即
+`ci.runner.<workload-class>.<driver>`，当前为 `ci.runner.container.kubernetes` 与
+`ci.runner.vm.opensandbox`。同 placement 的多个 worker 共享 durable consumer。ACK 后消息立即
 删除，不保存 CI 历史。
 
 ## 验证

platform/observability/grafana/values.yaml

@@ -32,6 +32,14 @@ datasources:
         isDefault: true
         jsonData:
           prometheusType: Prometheus
+      - name: Alertmanager
+        uid: alertmanager
+        type: alertmanager
+        access: proxy
+        url: http://vmalertmanager-main.monitoring.svc:9093
+        jsonData:
+          implementation: prometheus
+          handleGrafanaManagedAlerts: false
       - name: VictoriaLogs
         type: victoriametrics-logs-datasource
         access: proxy

platform/observability/metrics/alertmanager-external-secret.yaml

@@ -0,0 +1,18 @@
+apiVersion: external-secrets.io/v1
+kind: ExternalSecret
+metadata:
+  name: alertmanager-telegram
+  namespace: monitoring
+spec:
+  refreshInterval: 1h
+  secretStoreRef:
+    kind: ClusterSecretStore
+    name: openbao
+  target:
+    name: alertmanager-telegram
+    creationPolicy: Owner
+  data:
+    - secretKey: telegram_bot_token
+      remoteRef:
+        key: k8s/alertmanager
+        property: telegram_bot_token

platform/observability/metrics/exporters/kube-state-metrics-helmrelease.yaml

@@ -0,0 +1,257 @@
+apiVersion: helm.toolkit.fluxcd.io/v2
+kind: HelmRelease
+metadata:
+  name: kube-state-metrics
+  namespace: monitoring
+spec:
+  interval: 30m
+  timeout: 5m
+  dependsOn:
+  - name: vm-operator
+  - name: prometheus-operator-crds
+  chart:
+    spec:
+      chart: kube-state-metrics
+      version: 8.5.0
+      sourceRef:
+        kind: HelmRepository
+        name: prometheus-community
+        namespace: monitoring
+      interval: 1h
+  driftDetection:
+    mode: enabled
+  install:
+    remediation:
+      retries: 3
+  upgrade:
+    remediation:
+      retries: 3
+  values:
+    fullnameOverride: kube-state-metrics
+    collectors:
+    - cronjobs
+    - daemonsets
+    - deployments
+    - jobs
+    - namespaces
+    - nodes
+    - persistentvolumeclaims
+    - persistentvolumes
+    - pods
+    - statefulsets
+    resources:
+      requests:
+        cpu: 100m
+        memory: 128Mi
+      limits:
+        cpu: 500m
+        memory: 384Mi
+    prometheus:
+      monitor:
+        enabled: true
+        jobLabel: app.kubernetes.io/name
+        http:
+          interval: 30s
+          honorLabels: true
+        metrics:
+          interval: 30s
+          honorLabels: true
+    selfMonitor:
+      enabled: true
+    rbac:
+      extraRules:
+      - apiGroups:
+        - kustomize.toolkit.fluxcd.io
+        resources:
+        - kustomizations
+        verbs:
+        - list
+        - watch
+      - apiGroups:
+        - helm.toolkit.fluxcd.io
+        resources:
+        - helmreleases
+        verbs:
+        - list
+        - watch
+      - apiGroups:
+        - source.toolkit.fluxcd.io
+        resources:
+        - gitrepositories
+        - helmrepositories
+        - helmcharts
+        - ocirepositories
+        verbs:
+        - list
+        - watch
+    customResourceState:
+      enabled: true
+      config:
+        kind: CustomResourceStateMetrics
+        spec:
+          resources:
+          - groupVersionKind:
+              group: kustomize.toolkit.fluxcd.io
+              version: v1
+              kind: Kustomization
+            metricNamePrefix: gotk
+            metrics:
+            - name: resource_info
+              help: The current state of a GitOps Toolkit resource.
+              each:
+                type: Info
+                info:
+                  labelsFromPath:
+                    name:
+                    - metadata
+                    - name
+              labelsFromPath:
+                resource_namespace:
+                - metadata
+                - namespace
+                suspended:
+                - spec
+                - suspend
+                ready:
+                - status
+                - conditions
+                - '[type=Ready]'
+                - status
+          - groupVersionKind:
+              group: helm.toolkit.fluxcd.io
+              version: v2
+              kind: HelmRelease
+            metricNamePrefix: gotk
+            metrics:
+            - name: resource_info
+              help: The current state of a GitOps Toolkit resource.
+              each:
+                type: Info
+                info:
+                  labelsFromPath:
+                    name:
+                    - metadata
+                    - name
+              labelsFromPath:
+                resource_namespace:
+                - metadata
+                - namespace
+                suspended:
+                - spec
+                - suspend
+                ready:
+                - status
+                - conditions
+                - '[type=Ready]'
+                - status
+          - groupVersionKind:
+              group: source.toolkit.fluxcd.io
+              version: v1
+              kind: GitRepository
+            metricNamePrefix: gotk
+            metrics:
+            - name: resource_info
+              help: The current state of a GitOps Toolkit resource.
+              each:
+                type: Info
+                info:
+                  labelsFromPath:
+                    name:
+                    - metadata
+                    - name
+              labelsFromPath:
+                resource_namespace:
+                - metadata
+                - namespace
+                suspended:
+                - spec
+                - suspend
+                ready:
+                - status
+                - conditions
+                - '[type=Ready]'
+                - status
+          - groupVersionKind:
+              group: source.toolkit.fluxcd.io
+              version: v1
+              kind: HelmRepository
+            metricNamePrefix: gotk
+            metrics:
+            - name: resource_info
+              help: The current state of a GitOps Toolkit resource.
+              each:
+                type: Info
+                info:
+                  labelsFromPath:
+                    name:
+                    - metadata
+                    - name
+              labelsFromPath:
+                resource_namespace:
+                - metadata
+                - namespace
+                suspended:
+                - spec
+                - suspend
+                ready:
+                - status
+                - conditions
+                - '[type=Ready]'
+                - status
+                repository_type:
+                - spec
+                - type
+          - groupVersionKind:
+              group: source.toolkit.fluxcd.io
+              version: v1
+              kind: HelmChart
+            metricNamePrefix: gotk
+            metrics:
+            - name: resource_info
+              help: The current state of a GitOps Toolkit resource.
+              each:
+                type: Info
+                info:
+                  labelsFromPath:
+                    name:
+                    - metadata
+                    - name
+              labelsFromPath:
+                resource_namespace:
+                - metadata
+                - namespace
+                suspended:
+                - spec
+                - suspend
+                ready:
+                - status
+                - conditions
+                - '[type=Ready]'
+                - status
+          - groupVersionKind:
+              group: source.toolkit.fluxcd.io
+              version: v1
+              kind: OCIRepository
+            metricNamePrefix: gotk
+            metrics:
+            - name: resource_info
+              help: The current state of a GitOps Toolkit resource.
+              each:
+                type: Info
+                info:
+                  labelsFromPath:
+                    name:
+                    - metadata
+                    - name
+              labelsFromPath:
+                resource_namespace:
+                - metadata
+                - namespace
+                suspended:
+                - spec
+                - suspend
+                ready:
+                - status
+                - conditions
+                - '[type=Ready]'
+                - status

platform/observability/metrics/kustomization.yaml

@@ -6,6 +6,7 @@ resources:
   - vmagent.yaml
   - vmalert.yaml
   - vmalertmanager.yaml
+  - alertmanager-external-secret.yaml
   - reload-rbac.yaml
   - rules/vm-health.yaml
   - rules/vmagent.yaml
@@ -16,3 +17,12 @@ resources:
   - scrapes/kubelet.yaml
   - exporters/node-exporter.yaml
   - exporters/process-exporter.yaml
+  - exporters/kube-state-metrics-helmrelease.yaml
+  - rules/kubernetes-health.yaml
+  - rules/host-health.yaml
+  - rules/monitoring-delivery.yaml
+  - scrapes/platform-controllers.yaml
+  - rules/platform-controllers.yaml
+  - scrapes/cnpg.yaml
+  - scrapes/seaweedfs.yaml
+  - rules/data-services.yaml

platform/observability/metrics/rules/data-services.yaml

@@ -0,0 +1,148 @@
+apiVersion: monitoring.coreos.com/v1
+kind: PrometheusRule
+metadata:
+  name: data-services
+  namespace: monitoring
+spec:
+  groups:
+  - name: data-services
+    interval: 30s
+    rules:
+    - alert: DataServiceMetricsUnavailable
+      expr: up{job="cnpg"} == 0 or absent(up{job="cnpg"})
+      for: 5m
+      labels:
+        severity: critical
+        job: cnpg
+      annotations:
+        summary: 数据服务 {{ $labels.job }} 指标不可用
+        description: 检查监控对象、Pod 与 exporter；目标消失同样触发。
+        runbook_url: https://git.ddupan.top/panxiao81/homelab-wiki/src/branch/main/guides/monitoring-data-services.md
+    - alert: DataServiceMetricsUnavailable
+      expr: up{job="seaweedfs-master"} == 0 or absent(up{job="seaweedfs-master"})
+      for: 5m
+      labels:
+        severity: critical
+        job: seaweedfs-master
+      annotations:
+        summary: 数据服务 {{ $labels.job }} 指标不可用
+        description: 检查监控对象、Pod 与 exporter；目标消失同样触发。
+        runbook_url: https://git.ddupan.top/panxiao81/homelab-wiki/src/branch/main/guides/monitoring-data-services.md
+    - alert: DataServiceMetricsUnavailable
+      expr: up{job="seaweedfs-volume"} == 0 or absent(up{job="seaweedfs-volume"})
+      for: 5m
+      labels:
+        severity: critical
+        job: seaweedfs-volume
+      annotations:
+        summary: 数据服务 {{ $labels.job }} 指标不可用
+        description: 检查监控对象、Pod 与 exporter；目标消失同样触发。
+        runbook_url: https://git.ddupan.top/panxiao81/homelab-wiki/src/branch/main/guides/monitoring-data-services.md
+    - alert: DataServiceMetricsUnavailable
+      expr: up{job="seaweedfs-filer"} == 0 or absent(up{job="seaweedfs-filer"})
+      for: 5m
+      labels:
+        severity: critical
+        job: seaweedfs-filer
+      annotations:
+        summary: 数据服务 {{ $labels.job }} 指标不可用
+        description: 检查监控对象、Pod 与 exporter；目标消失同样触发。
+        runbook_url: https://git.ddupan.top/panxiao81/homelab-wiki/src/branch/main/guides/monitoring-data-services.md
+    - alert: CnpgPostgresDown
+      expr: cnpg_collector_up{job="cnpg"} == 0
+      for: 2m
+      labels:
+        severity: critical
+      annotations:
+        summary: CNPG {{ $labels.pod }} PostgreSQL 不可用
+        description: 检查数据库 Pod 状态、实例日志和存储；exporter up 不代表数据库可用。
+        runbook_url: https://git.ddupan.top/panxiao81/homelab-wiki/src/branch/main/guides/monitoring-data-services.md
+    - alert: CnpgMetricsCollectionFailed
+      expr: cnpg_collector_last_collection_error{job="cnpg"} == 1
+      for: 5m
+      labels:
+        severity: warning
+      annotations:
+        summary: CNPG {{ $labels.pod }} SQL 指标采集失败
+        description: 检查内置 exporter 日志、数据库连接与查询兼容性。
+        runbook_url: https://git.ddupan.top/panxiao81/homelab-wiki/src/branch/main/guides/monitoring-data-services.md
+    - alert: CnpgConnectionsHigh
+      expr: sum by (job,namespace,pod) (cnpg_backends_total{job="cnpg"}) / on(job,namespace,pod)
+        max by(job,namespace,pod) (cnpg_pg_settings_setting{job="cnpg",name="max_connections"})
+        > 0.8
+      for: 10m
+      labels:
+        severity: warning
+      annotations:
+        summary: CNPG {{ $labels.pod }} 连接使用率超过 80%
+        description: 检查连接池、空闲连接和连接泄漏，不直接提高 max_connections。
+        runbook_url: https://git.ddupan.top/panxiao81/homelab-wiki/src/branch/main/guides/monitoring-data-services.md
+    - alert: CnpgLongTransaction
+      expr: cnpg_backends_max_tx_duration_seconds{job="cnpg",state!="idle"} > 300
+      for: 5m
+      labels:
+        severity: warning
+      annotations:
+        summary: CNPG {{ $labels.pod }} 存在长事务
+        description: 事务持续超过五分钟并维持五分钟，检查阻塞与 idle in transaction；先确认业务再处理。
+        runbook_url: https://git.ddupan.top/panxiao81/homelab-wiki/src/branch/main/guides/monitoring-data-services.md
+    - alert: NatsJetStreamStorageHigh
+      expr: nats_varz_jetstream_stats_storage{job="nats/nats"} / nats_varz_jetstream_config_max_storage{job="nats/nats"}
+        > 0.8
+      for: 10m
+      labels:
+        severity: warning
+      annotations:
+        summary: NATS JetStream 文件容量超过 80%
+        description: 检查服务端配额与消息保留策略；该指标不代表单个 Account 或 stream 配额。
+        runbook_url: https://git.ddupan.top/panxiao81/homelab-wiki/src/branch/main/guides/monitoring-data-services.md
+    - alert: NatsJetStreamMemoryHigh
+      expr: nats_varz_jetstream_stats_memory{job="nats/nats"} / nats_varz_jetstream_config_max_memory{job="nats/nats"}
+        > 0.8
+      for: 10m
+      labels:
+        severity: warning
+      annotations:
+        summary: NATS JetStream 内存容量超过 80%
+        description: 检查 memory store 消息保留与服务端内存配额。
+        runbook_url: https://git.ddupan.top/panxiao81/homelab-wiki/src/branch/main/guides/monitoring-data-services.md
+    - alert: NatsSlowConsumers
+      expr: increase(nats_varz_slow_consumers{job="nats/nats"}[10m]) > 0
+      for: 5m
+      labels:
+        severity: warning
+      annotations:
+        summary: NATS 检测到慢消费者
+        description: 检查消费者处理速率与网络；此规则不能代替 JetStream consumer 积压监控。
+        runbook_url: https://git.ddupan.top/panxiao81/homelab-wiki/src/branch/main/guides/monitoring-data-services.md
+    - alert: SeaweedVolumeDiskLow
+      expr: SeaweedFS_volumeServer_resource{job="seaweedfs-volume",type="avail"} /
+        ignoring(type) SeaweedFS_volumeServer_resource{job="seaweedfs-volume",type="all"}
+        < 0.1
+      for: 10m
+      labels:
+        severity: warning
+      annotations:
+        summary: SeaweedFS {{ $labels.name }} 可用空间不足 10%
+        description: 检查 volume 文件系统容量、保留策略与底层 ZFS；不要直接删除卷文件。
+        runbook_url: https://git.ddupan.top/panxiao81/homelab-wiki/src/branch/main/guides/monitoring-data-services.md
+    - alert: SeaweedVolumeDiskReadOnly
+      expr: SeaweedFS_volumeServer_read_only_volumes{job="seaweedfs-volume",type="isDiskSpaceLow"}
+        > 0
+      for: 5m
+      labels:
+        severity: critical
+      annotations:
+        summary: SeaweedFS 因磁盘空间不足限制卷写入
+        description: 检查磁盘容量和 volume 日志；主动只读或满卷切换不按此故障处理。
+        runbook_url: https://git.ddupan.top/panxiao81/homelab-wiki/src/branch/main/guides/monitoring-data-services.md
+    - alert: SeaweedVolumeWriteFailures
+      expr: increase(SeaweedFS_volumeServer_file_write_failures{job="seaweedfs-volume"}[5m])
+        > 0
+      for: 2m
+      labels:
+        severity: warning
+      annotations:
+        summary: SeaweedFS 卷写入失败
+        description: 检查 volume 日志、磁盘状态及上游请求；不代表已经验证 S3 端到端可用。
+        runbook_url: https://git.ddupan.top/panxiao81/homelab-wiki/src/branch/main/guides/monitoring-data-services.md

platform/observability/metrics/rules/host-health.yaml

@@ -0,0 +1,66 @@
+apiVersion: monitoring.coreos.com/v1
+kind: PrometheusRule
+metadata:
+  name: host-health
+  namespace: monitoring
+  labels:
+    app.kubernetes.io/part-of: victoria-metrics
+spec:
+  groups:
+  - name: host-health
+    interval: 30s
+    rules:
+    - alert: HostMemoryLow
+      expr: (node_memory_MemAvailable_bytes{job="node-exporter"} / node_memory_MemTotal_bytes{job="node-exporter"})
+        < 0.10
+      for: 10m
+      labels:
+        severity: warning
+      annotations:
+        summary: 主机 {{ $labels.instance }} 可用内存不足 10%
+        description: 检查主机内存、Swap、进程 PSS；仅使用 node-exporter job，避免重复采集统计。
+        runbook_url: https://git.ddupan.top/panxiao81/homelab-wiki/src/branch/main/guides/monitoring-foundation.md
+    - alert: HostFilesystemSpaceLow
+      expr: (node_filesystem_avail_bytes{job="node-exporter",fstype!~"tmpfs|devtmpfs|overlay|squashfs|nsfs|fuse.*",mountpoint!~"/run(/.*)?|/var/lib/kubelet/.*|/var/lib/docker/.*"}
+        / node_filesystem_size_bytes{job="node-exporter",fstype!~"tmpfs|devtmpfs|overlay|squashfs|nsfs|fuse.*",mountpoint!~"/run(/.*)?|/var/lib/kubelet/.*|/var/lib/docker/.*"}
+        < 0.10) and on(instance,device,mountpoint) (node_filesystem_readonly{job="node-exporter",fstype!~"tmpfs|devtmpfs|overlay|squashfs|nsfs|fuse.*",mountpoint!~"/run(/.*)?|/var/lib/kubelet/.*|/var/lib/docker/.*"}
+        == 0)
+      for: 15m
+      labels:
+        severity: warning
+      annotations:
+        summary: 主机 {{ $labels.instance }} {{ $labels.mountpoint }} 可用空间不足 10%
+        description: 检查文件系统和 ZFS dataset 容量，清理前确认数据归属。
+        runbook_url: https://git.ddupan.top/panxiao81/homelab-wiki/src/branch/main/guides/monitoring-foundation.md
+    - alert: HostFilesystemInodesLow
+      expr: (node_filesystem_files_free{job="node-exporter",fstype!~"tmpfs|devtmpfs|overlay|squashfs|nsfs|fuse.*",mountpoint!~"/run(/.*)?|/var/lib/kubelet/.*|/var/lib/docker/.*"}
+        / node_filesystem_files{job="node-exporter",fstype!~"tmpfs|devtmpfs|overlay|squashfs|nsfs|fuse.*",mountpoint!~"/run(/.*)?|/var/lib/kubelet/.*|/var/lib/docker/.*"}
+        < 0.10) and on(instance,device,mountpoint) (node_filesystem_files{job="node-exporter",fstype!~"tmpfs|devtmpfs|overlay|squashfs|nsfs|fuse.*",mountpoint!~"/run(/.*)?|/var/lib/kubelet/.*|/var/lib/docker/.*"}
+        > 0) and on(instance,device,mountpoint) (node_filesystem_readonly{job="node-exporter",fstype!~"tmpfs|devtmpfs|overlay|squashfs|nsfs|fuse.*",mountpoint!~"/run(/.*)?|/var/lib/kubelet/.*|/var/lib/docker/.*"}
+        == 0)
+      for: 15m
+      labels:
+        severity: warning
+      annotations:
+        summary: 主机 {{ $labels.instance }} {{ $labels.mountpoint }} inode 不足 10%
+        description: 检查小文件数量，确认 filesystem 是否耗尽 inode。
+        runbook_url: https://git.ddupan.top/panxiao81/homelab-wiki/src/branch/main/guides/monitoring-foundation.md
+    - alert: HostFilesystemReadOnly
+      expr: node_filesystem_readonly{job="node-exporter",fstype!~"tmpfs|devtmpfs|overlay|squashfs|nsfs|fuse.*",mountpoint!~"/run(/.*)?|/var/lib/kubelet/.*|/var/lib/docker/.*"}
+        == 1
+      for: 5m
+      labels:
+        severity: critical
+      annotations:
+        summary: 主机 {{ $labels.instance }} {{ $labels.mountpoint }} 变为只读
+        description: 排查磁盘和文件系统错误；不要直接强制重新挂载。
+        runbook_url: https://git.ddupan.top/panxiao81/homelab-wiki/src/branch/main/guides/monitoring-foundation.md
+    - alert: HostZpoolUnhealthy
+      expr: node_zfs_zpool_state{job="node-exporter",state!="online"} == 1
+      for: 5m
+      labels:
+        severity: critical
+      annotations:
+        summary: 主机 {{ $labels.instance }} ZFS 池 {{ $labels.zpool }} 状态 {{ $labels.state }}
+        description: 检查 zpool status、磁盘和冗余；零值状态不触发。
+        runbook_url: https://git.ddupan.top/panxiao81/homelab-wiki/src/branch/main/guides/monitoring-foundation.md

platform/observability/metrics/rules/kubernetes-health.yaml

@@ -0,0 +1,121 @@
+apiVersion: monitoring.coreos.com/v1
+kind: PrometheusRule
+metadata:
+  name: kubernetes-health
+  namespace: monitoring
+  labels:
+    app.kubernetes.io/part-of: victoria-metrics
+spec:
+  groups:
+  - name: kubernetes-health
+    interval: 30s
+    rules:
+    - alert: KubeNodeNotReady
+      expr: max by (node) (kube_node_status_condition{job="kube-state-metrics",condition="Ready",status="true"})
+        == 0
+      for: 5m
+      labels:
+        severity: critical
+      annotations:
+        summary: 节点 {{ $labels.node }} 未就绪
+        description: 检查节点、kubelet 和网络；维护时按 node 精确静默。
+        runbook_url: https://git.ddupan.top/panxiao81/homelab-wiki/src/branch/main/guides/monitoring-foundation.md
+    - alert: KubeNodePressure
+      expr: max by (node, condition) (kube_node_status_condition{job="kube-state-metrics",condition=~"MemoryPressure|DiskPressure|PIDPressure",status="true"})
+        == 1
+      for: 5m
+      labels:
+        severity: warning
+      annotations:
+        summary: 节点 {{ $labels.node }} 出现 {{ $labels.condition }}
+        description: 检查内存、磁盘/inode 或 PID 资源压力。
+        runbook_url: https://git.ddupan.top/panxiao81/homelab-wiki/src/branch/main/guides/monitoring-foundation.md
+    - alert: KubePodCrashLooping
+      expr: max by (namespace,pod,container) (kube_pod_container_status_waiting_reason{job="kube-state-metrics",reason="CrashLoopBackOff"})
+        == 1
+      for: 10m
+      labels:
+        severity: warning
+      annotations:
+        summary: 容器 {{ $labels.namespace }}/{{ $labels.pod }}/{{ $labels.container }} 持续崩溃
+        description: 检查容器退出原因和日志；持续 10 分钟后通知，避免短暂重启噪声。
+        runbook_url: https://git.ddupan.top/panxiao81/homelab-wiki/src/branch/main/guides/monitoring-foundation.md
+    - alert: KubePodFrequentRestarts
+      expr: sum by (namespace,pod,container) (increase(kube_pod_container_status_restarts_total{job="kube-state-metrics"}[15m]))
+        > 3
+      for: 5m
+      labels:
+        severity: warning
+      annotations:
+        summary: 容器 {{ $labels.namespace }}/{{ $labels.pod }}/{{ $labels.container }} 频繁重启
+        description: 15 分钟内重启超过 3 次，检查退出原因和资源限制。
+        runbook_url: https://git.ddupan.top/panxiao81/homelab-wiki/src/branch/main/guides/monitoring-foundation.md
+    - alert: KubePodOOMKilled
+      expr: (max by (namespace,pod,container) (kube_pod_container_status_last_terminated_reason{job="kube-state-metrics",reason="OOMKilled"})
+        == 1) and on(namespace,pod,container) (sum by (namespace,pod,container) (increase(kube_pod_container_status_restarts_total{job="kube-state-metrics"}[10m]))
+        > 0)
+      for: 1m
+      labels:
+        severity: warning
+      annotations:
+        summary: 容器 {{ $labels.namespace }}/{{ $labels.pod }}/{{ $labels.container }} 发生 OOM 重启
+        description: 仅在近期有重启且最近终止原因为 OOMKilled 时触发，检查内存峰值和限制。
+        runbook_url: https://git.ddupan.top/panxiao81/homelab-wiki/src/branch/main/guides/monitoring-foundation.md
+    - alert: KubePodPending
+      expr: max by (namespace,pod) (kube_pod_status_phase{job="kube-state-metrics",phase="Pending"}) == 1
+      for: 15m
+      labels:
+        severity: warning
+      annotations:
+        summary: Pod {{ $labels.namespace }}/{{ $labels.pod }} 长时间 Pending
+        description: 检查调度事件、资源、卷和镜像拉取。
+        runbook_url: https://git.ddupan.top/panxiao81/homelab-wiki/src/branch/main/guides/monitoring-foundation.md
+    - alert: KubeDeploymentUnavailable
+      expr: (max by(namespace,deployment) (kube_deployment_spec_replicas{job="kube-state-metrics"}) - max by(namespace,deployment)
+        (kube_deployment_status_replicas_available{job="kube-state-metrics"})) > 0
+      for: 10m
+      labels:
+        severity: warning
+      annotations:
+        summary: Deployment {{ $labels.namespace }}/{{ $labels.deployment }} 副本不足
+        description: 可用副本持续 10 分钟低于声明值；缩容至零不会触发。
+        runbook_url: https://git.ddupan.top/panxiao81/homelab-wiki/src/branch/main/guides/monitoring-foundation.md
+    - alert: KubeStatefulSetUnavailable
+      expr: (max by(namespace,statefulset) (kube_statefulset_replicas{job="kube-state-metrics"}) - max by(namespace,statefulset)
+        (kube_statefulset_status_replicas_ready{job="kube-state-metrics"})) > 0
+      for: 10m
+      labels:
+        severity: warning
+      annotations:
+        summary: StatefulSet {{ $labels.namespace }}/{{ $labels.statefulset }} 副本不足
+        description: 检查 Pod 就绪、存储和依赖；已知维护按工作负载精确静默。
+        runbook_url: https://git.ddupan.top/panxiao81/homelab-wiki/src/branch/main/guides/monitoring-foundation.md
+    - alert: KubeDaemonSetUnavailable
+      expr: (max by(namespace,daemonset) (kube_daemonset_status_desired_number_scheduled{job="kube-state-metrics"})
+        - max by(namespace,daemonset) (kube_daemonset_status_number_ready{job="kube-state-metrics"})) > 0
+      for: 10m
+      labels:
+        severity: warning
+      annotations:
+        summary: DaemonSet {{ $labels.namespace }}/{{ $labels.daemonset }} 副本不足
+        description: 检查节点和 Pod 就绪；不按整个 namespace 屏蔽。
+        runbook_url: https://git.ddupan.top/panxiao81/homelab-wiki/src/branch/main/guides/monitoring-foundation.md
+    - alert: KubePVCPending
+      expr: max by(namespace,persistentvolumeclaim) (kube_persistentvolumeclaim_status_phase{job="kube-state-metrics",phase="Pending"})
+        == 1
+      for: 15m
+      labels:
+        severity: warning
+      annotations:
+        summary: PVC {{ $labels.namespace }}/{{ $labels.persistentvolumeclaim }} 长时间 Pending
+        description: 检查 StorageClass、调度拓扑、容量与 provisioner。
+        runbook_url: https://git.ddupan.top/panxiao81/homelab-wiki/src/branch/main/guides/monitoring-foundation.md
+    - alert: KubeJobFailed
+      expr: max by(namespace,job_name) (kube_job_failed{job="kube-state-metrics",condition="true"}) == 1
+      for: 5m
+      labels:
+        severity: warning
+      annotations:
+        summary: Job {{ $labels.namespace }}/{{ $labels.job_name }} 已失败
+        description: 检查失败 Job 的 Pod 和任务日志；成功重试中的失败 Pod 计数不作为失败 Job。
+        runbook_url: https://git.ddupan.top/panxiao81/homelab-wiki/src/branch/main/guides/monitoring-foundation.md

platform/observability/metrics/rules/monitoring-delivery.yaml

@@ -0,0 +1,58 @@
+apiVersion: monitoring.coreos.com/v1
+kind: PrometheusRule
+metadata:
+  name: monitoring-delivery
+  namespace: monitoring
+  labels:
+    app.kubernetes.io/part-of: victoria-metrics
+spec:
+  groups:
+  - name: monitoring-delivery
+    interval: 30s
+    rules:
+    - alert: KubeStateMetricsUnavailable
+      expr: up{job="kube-state-metrics"} == 0 or absent(up{job="kube-state-metrics"})
+      for: 5m
+      labels:
+        severity: critical
+      annotations:
+        summary: kube-state-metrics 采集不可用
+        description: 检查 HelmRelease、ServiceMonitor 转换、targets 与 exporter；目标消失也会触发。
+        runbook_url: https://git.ddupan.top/panxiao81/homelab-wiki/src/branch/main/guides/monitoring-foundation.md
+    - alert: NodeExporterUnavailable
+      expr: up{job="node-exporter"} == 0 or absent(up{job="node-exporter"})
+      for: 5m
+      labels:
+        severity: critical
+      annotations:
+        summary: node-exporter 采集不可用
+        description: 检查节点与 exporter，主机健康规则依赖此采集。
+        runbook_url: https://git.ddupan.top/panxiao81/homelab-wiki/src/branch/main/guides/monitoring-foundation.md
+    - alert: NatsMetricsUnavailable
+      expr: up{job="nats/nats"} == 0 or absent(up{job="nats/nats"})
+      for: 5m
+      labels:
+        severity: warning
+      annotations:
+        summary: NATS 指标采集不可用
+        description: 依次检查 PodMonitor、转换后的 VMPodScrape、:7777 exporter 及 targets。
+        runbook_url: https://git.ddupan.top/panxiao81/homelab-wiki/src/branch/main/guides/monitoring-foundation.md
+    - alert: AlertmanagerTelegramDeliveryFailed
+      expr: sum by(job,instance,integration) (increase(alertmanager_notifications_failed_total{job="vmalertmanager-main",integration="telegram"}[5m]))
+        > 0
+      for: 1m
+      labels:
+        severity: critical
+      annotations:
+        summary: Alertmanager 向 Telegram 发送失败
+        description: 检查网络、bot token 和频道权限。若 Telegram 完全不可达，本告警也无法经同一渠道送达；需在 Alertmanager/Grafana 排查，集群外心跳另行建设。
+        runbook_url: https://git.ddupan.top/panxiao81/homelab-wiki/src/branch/main/guides/monitoring-foundation.md
+    - alert: AlertmanagerConfigurationReloadFailed
+      expr: alertmanager_config_last_reload_successful{job="vmalertmanager-main"} == 0
+      for: 5m
+      labels:
+        severity: critical
+      annotations:
+        summary: Alertmanager 配置加载失败
+        description: 检查 operator 和 Alertmanager 日志、配置格式及 Secret 引用。
+        runbook_url: https://git.ddupan.top/panxiao81/homelab-wiki/src/branch/main/guides/monitoring-foundation.md

platform/observability/metrics/rules/platform-controllers.yaml

@@ -0,0 +1,161 @@
+apiVersion: monitoring.coreos.com/v1
+kind: PrometheusRule
+metadata:
+  name: platform-controllers
+  namespace: monitoring
+spec:
+  groups:
+  - name: platform-controllers
+    interval: 30s
+    rules:
+    - alert: PlatformControllerMetricsUnavailable
+      expr: up{job="cert-manager"} == 0 or absent(up{job="cert-manager"})
+      for: 5m
+      labels:
+        severity: warning
+        job: cert-manager
+      annotations:
+        summary: 控制器 {{ $labels.job }} 指标不可用
+        description: 检查对应 Pod、监控 CR 转换与 targets；目标消失同样触发。
+        runbook_url: https://git.ddupan.top/panxiao81/homelab-wiki/src/branch/main/guides/monitoring-controllers.md
+    - alert: PlatformControllerMetricsUnavailable
+      expr: up{job="external-secrets"} == 0 or absent(up{job="external-secrets"})
+      for: 5m
+      labels:
+        severity: warning
+        job: external-secrets
+      annotations:
+        summary: 控制器 {{ $labels.job }} 指标不可用
+        description: 检查对应 Pod、监控 CR 转换与 targets；目标消失同样触发。
+        runbook_url: https://git.ddupan.top/panxiao81/homelab-wiki/src/branch/main/guides/monitoring-controllers.md
+    - alert: PlatformControllerMetricsUnavailable
+      expr: up{job="external-secrets-cert-controller"} == 0 or absent(up{job="external-secrets-cert-controller"})
+      for: 5m
+      labels:
+        severity: warning
+        job: external-secrets-cert-controller
+      annotations:
+        summary: 控制器 {{ $labels.job }} 指标不可用
+        description: 检查对应 Pod、监控 CR 转换与 targets；目标消失同样触发。
+        runbook_url: https://git.ddupan.top/panxiao81/homelab-wiki/src/branch/main/guides/monitoring-controllers.md
+    - alert: PlatformControllerMetricsUnavailable
+      expr: up{job="external-secrets-webhook"} == 0 or absent(up{job="external-secrets-webhook"})
+      for: 5m
+      labels:
+        severity: warning
+        job: external-secrets-webhook
+      annotations:
+        summary: 控制器 {{ $labels.job }} 指标不可用
+        description: 检查对应 Pod、监控 CR 转换与 targets；目标消失同样触发。
+        runbook_url: https://git.ddupan.top/panxiao81/homelab-wiki/src/branch/main/guides/monitoring-controllers.md
+    - alert: PlatformControllerMetricsUnavailable
+      expr: up{job="helm-controller"} == 0 or absent(up{job="helm-controller"})
+      for: 5m
+      labels:
+        severity: warning
+        job: helm-controller
+      annotations:
+        summary: 控制器 {{ $labels.job }} 指标不可用
+        description: 检查对应 Pod、监控 CR 转换与 targets；目标消失同样触发。
+        runbook_url: https://git.ddupan.top/panxiao81/homelab-wiki/src/branch/main/guides/monitoring-controllers.md
+    - alert: PlatformControllerMetricsUnavailable
+      expr: up{job="kustomize-controller"} == 0 or absent(up{job="kustomize-controller"})
+      for: 5m
+      labels:
+        severity: warning
+        job: kustomize-controller
+      annotations:
+        summary: 控制器 {{ $labels.job }} 指标不可用
+        description: 检查对应 Pod、监控 CR 转换与 targets；目标消失同样触发。
+        runbook_url: https://git.ddupan.top/panxiao81/homelab-wiki/src/branch/main/guides/monitoring-controllers.md
+    - alert: PlatformControllerMetricsUnavailable
+      expr: up{job="source-controller"} == 0 or absent(up{job="source-controller"})
+      for: 5m
+      labels:
+        severity: warning
+        job: source-controller
+      annotations:
+        summary: 控制器 {{ $labels.job }} 指标不可用
+        description: 检查对应 Pod、监控 CR 转换与 targets；目标消失同样触发。
+        runbook_url: https://git.ddupan.top/panxiao81/homelab-wiki/src/branch/main/guides/monitoring-controllers.md
+    - alert: PlatformControllerMetricsUnavailable
+      expr: up{job="notification-controller"} == 0 or absent(up{job="notification-controller"})
+      for: 5m
+      labels:
+        severity: warning
+        job: notification-controller
+      annotations:
+        summary: 控制器 {{ $labels.job }} 指标不可用
+        description: 检查对应 Pod、监控 CR 转换与 targets；目标消失同样触发。
+        runbook_url: https://git.ddupan.top/panxiao81/homelab-wiki/src/branch/main/guides/monitoring-controllers.md
+    - alert: CertificateNotReady
+      expr: certmanager_certificate_ready_status{job="cert-manager",condition="True"}
+        == 0
+      for: 15m
+      labels:
+        severity: warning
+      annotations:
+        summary: 证书 {{ $labels.namespace }}/{{ $labels.name }} 未就绪
+        description: 检查 Certificate、CertificateRequest、Issuer、Order 与 Challenge；允许短暂签发过程。
+        runbook_url: https://git.ddupan.top/panxiao81/homelab-wiki/src/branch/main/guides/monitoring-controllers.md
+    - alert: CertificateExpiringSoon
+      expr: (certmanager_certificate_expiration_timestamp_seconds{job="cert-manager"}
+        - time() < 7 * 86400) and (certmanager_certificate_expiration_timestamp_seconds{job="cert-manager"}
+        - time() >= 86400)
+      for: 15m
+      labels:
+        severity: warning
+      annotations:
+        summary: 证书 {{ $labels.namespace }}/{{ $labels.name }} 将在七天内到期
+        description: 检查自动续期链路及实际入口证书；一天内到期由 critical 规则接替。
+        runbook_url: https://git.ddupan.top/panxiao81/homelab-wiki/src/branch/main/guides/monitoring-controllers.md
+    - alert: CertificateExpiryCritical
+      expr: certmanager_certificate_expiration_timestamp_seconds{job="cert-manager"}
+        - time() < 86400
+      for: 5m
+      labels:
+        severity: critical
+      annotations:
+        summary: 证书 {{ $labels.namespace }}/{{ $labels.name }} 即将到期或已过期
+        description: 不足一天或已经过期；立即检查续期与入口证书。
+        runbook_url: https://git.ddupan.top/panxiao81/homelab-wiki/src/branch/main/guides/monitoring-controllers.md
+    - alert: ExternalSecretNotReady
+      expr: externalsecret_status_condition{job="external-secrets",condition="Ready",status="True"}
+        == 0
+      for: 10m
+      labels:
+        severity: warning
+      annotations:
+        summary: Secret 同步 {{ $labels.namespace }}/{{ $labels.name }} 未就绪
+        description: 检查 ExternalSecret 状态及 provider 权限、网络；不要输出 Secret 内容。
+        runbook_url: https://git.ddupan.top/panxiao81/homelab-wiki/src/branch/main/guides/monitoring-controllers.md
+    - alert: ClusterSecretStoreNotReady
+      expr: clustersecretstore_status_condition{job="external-secrets",condition="Ready",status="True"}
+        == 0
+      for: 5m
+      labels:
+        severity: critical
+      annotations:
+        summary: ClusterSecretStore {{ $labels.name }} 未就绪
+        description: 检查 OpenBao 可达性与 ESO 身份鉴权；可能影响多个命名空间。
+        runbook_url: https://git.ddupan.top/panxiao81/homelab-wiki/src/branch/main/guides/monitoring-controllers.md
+    - alert: FluxResourceNotReady
+      expr: gotk_resource_info{job="kube-state-metrics",ready!="True",suspended!="true",repository_type!="oci"}
+        == 1
+      for: 15m
+      labels:
+        severity: warning
+      annotations:
+        summary: Flux {{ $labels.customresource_kind }} {{ $labels.resource_namespace
+          }}/{{ $labels.name }} 未就绪
+        description: 检查对象 conditions、依赖和 controller 日志；主动 suspend 不触发，不自动解除暂停。
+        runbook_url: https://git.ddupan.top/panxiao81/homelab-wiki/src/branch/main/guides/monitoring-controllers.md
+    - alert: FluxStateMetricsMissing
+      expr: absent(gotk_resource_info{job="kube-state-metrics",customresource_kind="Kustomization",resource_namespace="flux-system",name="flux-system"})
+      for: 5m
+      labels:
+        severity: warning
+      annotations:
+        summary: Flux 对象状态指标缺失
+        description: 检查 kube-state-metrics 自定义资源配置、RBAC 与采集；不能把空结果当作健康。
+        runbook_url: https://git.ddupan.top/panxiao81/homelab-wiki/src/branch/main/guides/monitoring-controllers.md

platform/observability/metrics/scrapes/cnpg.yaml

@@ -0,0 +1,22 @@
+apiVersion: monitoring.coreos.com/v1
+kind: PodMonitor
+metadata:
+  name: cnpg
+  namespace: monitoring
+spec:
+  namespaceSelector:
+    matchNames:
+    - shared-db
+  selector:
+    matchLabels:
+      cnpg.io/cluster: shared-postgresql
+      cnpg.io/podRole: instance
+  podTargetLabels:
+  - cnpg.io/cluster
+  podMetricsEndpoints:
+  - port: metrics
+    interval: 30s
+    honorLabels: true
+    relabelings:
+    - targetLabel: job
+      replacement: cnpg

platform/observability/metrics/scrapes/docker-hosts.yaml

@@ -1,6 +1,6 @@
-# Pull metrics from Docker hosts running ../../docker-hosts/compose.yaml.
+# 仅抓取已存在的宿主机 node-exporter。旧 :8080 目标实际返回 nginx 404，
-# vmagent scrapes each host's node-exporter (:9100) and cAdvisor (:8080) over the LAN.
+# 独立 Docker cAdvisor 尚未部署；部署并确认端口后再添加其目标。
-# Add one target block per host; keep the `host` label in sync with its HOST_LABEL.
+# Kubernetes 容器指标由 kubelet.yaml 的 /metrics/cadvisor 独立采集。
 apiVersion: operator.victoriametrics.com/v1beta1
 kind: VMStaticScrape
 metadata:
@@ -17,13 +17,6 @@ spec:
       labels:
         job: node-exporter
         host: docker-01
-    - targets:
+    # --- 新增主机前确认 exporter 已部署且 /metrics 返回成功 ---
-        - "192.168.10.127:8080"
-      labels:
-        job: cadvisor
-        host: docker-01
-    # --- add more hosts below, mirroring the two blocks above ---
     # - targets: ["192.168.10.x:9100"]
     #   labels: { job: node-exporter, host: docker-02 }
-    # - targets: ["192.168.10.x:8080"]
-    #   labels: { job: cadvisor, host: docker-02 }

platform/observability/metrics/scrapes/kubelet.yaml

@@ -13,6 +13,9 @@ spec:
   honorLabels: true
   honorTimestamps: false
   interval: 30s
+  # k3s 此端点包含 apiserver/etcd 指标，响应已超过默认 16 MiB。
+  # 仅放宽 kubelet 任务，其他采集目标保留默认限制。
+  max_scrape_size: "32MiB"
   tlsConfig:
     insecureSkipVerify: true
     caFile: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt

platform/observability/metrics/scrapes/platform-controllers.yaml

@@ -0,0 +1,57 @@
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  name: cert-manager
+  namespace: monitoring
+spec:
+  namespaceSelector:
+    matchNames:
+    - cert-manager
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: cert-manager
+      app.kubernetes.io/component: controller
+  jobLabel: app.kubernetes.io/name
+  endpoints:
+  - port: http-metrics
+    interval: 30s
+    scrapeTimeout: 10s
+    honorLabels: true
+---
+apiVersion: monitoring.coreos.com/v1
+kind: PodMonitor
+metadata:
+  name: external-secrets
+  namespace: monitoring
+spec:
+  namespaceSelector:
+    matchNames:
+    - external-secrets
+  selector:
+    matchLabels:
+      app.kubernetes.io/instance: external-secrets
+  jobLabel: app.kubernetes.io/name
+  podMetricsEndpoints:
+  - port: metrics
+    interval: 30s
+    scrapeTimeout: 10s
+    honorLabels: true
+---
+apiVersion: monitoring.coreos.com/v1
+kind: PodMonitor
+metadata:
+  name: flux-controllers
+  namespace: monitoring
+spec:
+  namespaceSelector:
+    matchNames:
+    - flux-system
+  selector:
+    matchLabels:
+      app.kubernetes.io/part-of: flux
+  jobLabel: app
+  podMetricsEndpoints:
+  - port: http-prom
+    interval: 30s
+    scrapeTimeout: 10s
+    honorLabels: true

platform/observability/metrics/scrapes/seaweedfs.yaml

@@ -0,0 +1,25 @@
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  name: seaweedfs
+  namespace: monitoring
+spec:
+  namespaceSelector:
+    matchNames:
+    - seaweedfs
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: seaweedfs
+      app.kubernetes.io/instance: seaweedfs
+  endpoints:
+  - port: metrics
+    interval: 30s
+    honorLabels: true
+    relabelings:
+    - sourceLabels:
+      - __meta_kubernetes_service_name
+      regex: seaweedfs-(master|volume|filer)
+      action: keep
+    - sourceLabels:
+      - __meta_kubernetes_service_name
+      targetLabel: job

platform/observability/metrics/tests/check-rules.sh

@@ -0,0 +1,20 @@
+#!/usr/bin/env bash
+# 依赖 Python 3 + PyYAML，以及 PATH 中的 promtool（验证版本 3.5.0）。
+set -euo pipefail
+test_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"
+check_dir="$(mktemp -d)"
+trap 'rm -rf "$check_dir"' EXIT
+python3 - "$test_dir/../rules" "$check_dir/rules.yaml" <<'PY'
+import pathlib
+import sys
+import yaml
+
+groups = []
+for name in ("kubernetes-health", "host-health", "monitoring-delivery", "platform-controllers", "data-services"):
+    path = pathlib.Path(sys.argv[1]) / f"{name}.yaml"
+    groups.extend(yaml.safe_load(path.read_text())["spec"]["groups"])
+pathlib.Path(sys.argv[2]).write_text(yaml.safe_dump({"groups": groups}, allow_unicode=True))
+PY
+cp "$test_dir/"*.test.yaml "$check_dir/"
+promtool check rules "$check_dir/rules.yaml"
+promtool test rules "$check_dir/"*.test.yaml

platform/observability/metrics/tests/foundation.test.yaml

@@ -0,0 +1,137 @@
+rule_files:
+- rules.yaml
+evaluation_interval: 1m
+tests:
+- name: 健康 ZFS 的零值故障状态不报警
+  interval: 1m
+  input_series:
+  - series: node_zfs_zpool_state{job="node-exporter",instance="laptop",zpool="data",state="degraded"}
+    values: '0x20'
+  - series: node_zfs_zpool_state{job="node-exporter",instance="laptop",zpool="data",state="online"}
+    values: 1x20
+  alert_rule_test:
+  - eval_time: 10m
+    alertname: HostZpoolUnhealthy
+    exp_alerts: &id001 []
+- name: ZFS 降级持续五分钟触发
+  interval: 1m
+  input_series:
+  - series: node_zfs_zpool_state{job="node-exporter",instance="laptop",zpool="data",state="degraded"}
+    values: 1x20
+  alert_rule_test:
+  - eval_time: 4m
+    alertname: HostZpoolUnhealthy
+    exp_alerts: *id001
+  - eval_time: 6m
+    alertname: HostZpoolUnhealthy
+    exp_alerts:
+    - exp_labels:
+        job: node-exporter
+        instance: laptop
+        zpool: data
+        state: degraded
+        severity: critical
+      exp_annotations:
+        summary: 主机 laptop ZFS 池 data 状态 degraded
+        description: 检查 zpool status、磁盘和冗余；零值状态不触发。
+        runbook_url: https://git.ddupan.top/panxiao81/homelab-wiki/src/branch/main/guides/monitoring-foundation.md
+- name: 忽略旧 docker-hosts 重复主机指标
+  interval: 1m
+  input_series:
+  - series: node_memory_MemAvailable_bytes{job="node-exporter",instance="laptop"}
+    values: 50x20
+  - series: node_memory_MemTotal_bytes{job="node-exporter",instance="laptop"}
+    values: 100x20
+  - series: node_memory_MemAvailable_bytes{job="docker-hosts",instance="laptop"}
+    values: 1x20
+  - series: node_memory_MemTotal_bytes{job="docker-hosts",instance="laptop"}
+    values: 100x20
+  alert_rule_test:
+  - eval_time: 15m
+    alertname: HostMemoryLow
+    exp_alerts: *id001
+- name: 临时文件系统耗尽不触发持久磁盘告警
+  interval: 1m
+  input_series:
+  - series: node_filesystem_avail_bytes{job="node-exporter",instance="laptop",device="tmpfs",mountpoint="/run",fstype="tmpfs"}
+    values: 1x20
+  - series: node_filesystem_size_bytes{job="node-exporter",instance="laptop",device="tmpfs",mountpoint="/run",fstype="tmpfs"}
+    values: 100x20
+  - series: node_filesystem_readonly{job="node-exporter",instance="laptop",device="tmpfs",mountpoint="/run",fstype="tmpfs"}
+    values: '0x20'
+  alert_rule_test:
+  - eval_time: 20m
+    alertname: HostFilesystemSpaceLow
+    exp_alerts: *id001
+- name: Deployment 允许短暂滚动且缩零不报错
+  interval: 1m
+  input_series:
+  - series: kube_deployment_spec_replicas{job="kube-state-metrics",namespace="app",deployment="web"}
+    values: 2x20
+  - series: kube_deployment_status_replicas_available{job="kube-state-metrics",namespace="app",deployment="web"}
+    values: 1x20
+  - series: kube_deployment_spec_replicas{job="kube-state-metrics",namespace="app",deployment="scaled-down"}
+    values: '0x20'
+  - series: kube_deployment_status_replicas_available{job="kube-state-metrics",namespace="app",deployment="scaled-down"}
+    values: '0x20'
+  alert_rule_test:
+  - eval_time: 5m
+    alertname: KubeDeploymentUnavailable
+    exp_alerts: *id001
+  - eval_time: 11m
+    alertname: KubeDeploymentUnavailable
+    exp_alerts:
+    - exp_labels:
+        namespace: app
+        deployment: web
+        severity: warning
+      exp_annotations:
+        summary: Deployment app/web 副本不足
+        description: 可用副本持续 10 分钟低于声明值；缩容至零不会触发。
+        runbook_url: https://git.ddupan.top/panxiao81/homelab-wiki/src/branch/main/guides/monitoring-foundation.md
+- name: NATS 发现集合完全消失也报警
+  interval: 1m
+  input_series: []
+  alert_rule_test:
+  - eval_time: 6m
+    alertname: NatsMetricsUnavailable
+    exp_alerts:
+    - exp_labels:
+        job: nats/nats
+        severity: warning
+      exp_annotations:
+        summary: NATS 指标采集不可用
+        description: 依次检查 PodMonitor、转换后的 VMPodScrape、:7777 exporter 及 targets。
+        runbook_url: https://git.ddupan.top/panxiao81/homelab-wiki/src/branch/main/guides/monitoring-foundation.md
+- name: 历史 OOM 没有新重启不反复报警
+  interval: 1m
+  input_series:
+  - series: kube_pod_container_status_last_terminated_reason{job="kube-state-metrics",namespace="app",pod="web",container="web",reason="OOMKilled"}
+    values: 1x20
+  - series: kube_pod_container_status_restarts_total{job="kube-state-metrics",namespace="app",pod="web",container="web"}
+    values: 5x20
+  alert_rule_test:
+  - eval_time: 10m
+    alertname: KubePodOOMKilled
+    exp_alerts: *id001
+- name: Telegram 失败跨 reason 汇总为一条
+  interval: 1m
+  input_series:
+  - series: alertmanager_notifications_failed_total{job="vmalertmanager-main",instance="am",integration="telegram",reason="clientError"}
+    values: 0+1x20
+  - series: alertmanager_notifications_failed_total{job="vmalertmanager-main",instance="am",integration="telegram",reason="serverError"}
+    values: 0+1x20
+  alert_rule_test:
+  - eval_time: 6m
+    alertname: AlertmanagerTelegramDeliveryFailed
+    exp_alerts:
+    - exp_labels:
+        job: vmalertmanager-main
+        instance: am
+        integration: telegram
+        severity: critical
+      exp_annotations:
+        summary: Alertmanager 向 Telegram 发送失败
+        description: 检查网络、bot token 和频道权限。若 Telegram 完全不可达，本告警也无法经同一渠道送达；需在 Alertmanager/Grafana
+          排查，集群外心跳另行建设。
+        runbook_url: https://git.ddupan.top/panxiao81/homelab-wiki/src/branch/main/guides/monitoring-foundation.md

platform/observability/metrics/tests/platform.test.yaml

@@ -0,0 +1,185 @@
+rule_files:
+- rules.yaml
+evaluation_interval: 1m
+tests:
+- name: Flux 暂停的失败对象不报警
+  interval: 1m
+  input_series:
+  - series: gotk_resource_info{job="kube-state-metrics",ready="False",suspended="true",customresource_kind="HelmRelease",resource_namespace="app",name="paused"}
+    values: 1x30
+  alert_rule_test:
+  - eval_time: 20m
+    alertname: FluxResourceNotReady
+    exp_alerts: []
+- name: Flux 缺省 suspend 仍检测失败
+  interval: 1m
+  input_series:
+  - series: gotk_resource_info{job="kube-state-metrics",ready="False",customresource_kind="HelmRelease",resource_namespace="app",name="broken"}
+    values: 1x30
+  alert_rule_test:
+  - eval_time: 10m
+    alertname: FluxResourceNotReady
+    exp_alerts: []
+  - eval_time: 16m
+    alertname: FluxResourceNotReady
+    exp_alerts:
+    - exp_labels:
+        job: kube-state-metrics
+        ready: 'False'
+        customresource_kind: HelmRelease
+        resource_namespace: app
+        name: broken
+        severity: warning
+      exp_annotations:
+        summary: Flux HelmRelease app/broken 未就绪
+        description: 检查对象 conditions、依赖和 controller 日志；主动 suspend 不触发，不自动解除暂停。
+        runbook_url: https://git.ddupan.top/panxiao81/homelab-wiki/src/branch/main/guides/monitoring-controllers.md
+- name: 证书一天内到期只由 critical 接替
+  interval: 1m
+  input_series:
+  - series: certmanager_certificate_expiration_timestamp_seconds{job="cert-manager",namespace="app",name="tls"}
+    values: 3600x30
+  alert_rule_test:
+  - eval_time: 20m
+    alertname: CertificateExpiringSoon
+    exp_alerts: []
+- name: 过期证书仍持续 critical
+  interval: 1m
+  input_series:
+  - series: certmanager_certificate_expiration_timestamp_seconds{job="cert-manager",namespace="app",name="tls"}
+    values: '0x30'
+  alert_rule_test:
+  - eval_time: 4m
+    alertname: CertificateExpiryCritical
+    exp_alerts: []
+  - eval_time: 6m
+    alertname: CertificateExpiryCritical
+    exp_alerts:
+    - exp_labels:
+        job: cert-manager
+        namespace: app
+        name: tls
+        severity: critical
+      exp_annotations:
+        summary: 证书 app/tls 即将到期或已过期
+        description: 不足一天或已经过期；立即检查续期与入口证书。
+        runbook_url: https://git.ddupan.top/panxiao81/homelab-wiki/src/branch/main/guides/monitoring-controllers.md
+- name: ESO False 零值不能导致健康对象误报
+  interval: 1m
+  input_series:
+  - series: externalsecret_status_condition{job="external-secrets",condition="Ready",status="False",namespace="app",name="credentials"}
+    values: '0x30'
+  - series: externalsecret_status_condition{job="external-secrets",condition="Ready",status="True",namespace="app",name="credentials"}
+    values: 1x30
+  alert_rule_test:
+  - eval_time: 20m
+    alertname: ExternalSecretNotReady
+    exp_alerts: []
+- name: ESO 失败持续十分钟报警
+  interval: 1m
+  input_series:
+  - series: externalsecret_status_condition{job="external-secrets",condition="Ready",status="True",namespace="app",name="credentials"}
+    values: '0x30'
+  alert_rule_test:
+  - eval_time: 9m
+    alertname: ExternalSecretNotReady
+    exp_alerts: []
+  - eval_time: 11m
+    alertname: ExternalSecretNotReady
+    exp_alerts:
+    - exp_labels:
+        job: external-secrets
+        condition: Ready
+        status: 'True'
+        namespace: app
+        name: credentials
+        severity: warning
+      exp_annotations:
+        summary: Secret 同步 app/credentials 未就绪
+        description: 检查 ExternalSecret 状态及 provider 权限、网络；不要输出 Secret 内容。
+        runbook_url: https://git.ddupan.top/panxiao81/homelab-wiki/src/branch/main/guides/monitoring-controllers.md
+- name: CNPG 聚合数据库连接并对齐实例限额
+  interval: 1m
+  input_series:
+  - series: cnpg_backends_total{job="cnpg",namespace="db",pod="pg-1",datname="a"}
+    values: 50x30
+  - series: cnpg_backends_total{job="cnpg",namespace="db",pod="pg-1",datname="b"}
+    values: 40x30
+  - series: cnpg_pg_settings_setting{job="cnpg",namespace="db",pod="pg-1",name="max_connections"}
+    values: 100x30
+  alert_rule_test:
+  - eval_time: 9m
+    alertname: CnpgConnectionsHigh
+    exp_alerts: []
+  - eval_time: 11m
+    alertname: CnpgConnectionsHigh
+    exp_alerts:
+    - exp_labels:
+        job: cnpg
+        namespace: db
+        pod: pg-1
+        severity: warning
+      exp_annotations:
+        summary: CNPG pg-1 连接使用率超过 80%
+        description: 检查连接池、空闲连接和连接泄漏，不直接提高 max_connections。
+        runbook_url: https://git.ddupan.top/panxiao81/homelab-wiki/src/branch/main/guides/monitoring-data-services.md
+- name: SeaweedFS 磁盘分母忽略 type 标签
+  interval: 1m
+  input_series:
+  - series: SeaweedFS_volumeServer_resource{job="seaweedfs-volume",name="/data",type="avail"}
+    values: 5x30
+  - series: SeaweedFS_volumeServer_resource{job="seaweedfs-volume",name="/data",type="all"}
+    values: 100x30
+  alert_rule_test:
+  - eval_time: 9m
+    alertname: SeaweedVolumeDiskLow
+    exp_alerts: []
+  - eval_time: 11m
+    alertname: SeaweedVolumeDiskLow
+    exp_alerts:
+    - exp_labels:
+        job: seaweedfs-volume
+        name: /data
+        severity: warning
+      exp_annotations:
+        summary: SeaweedFS /data 可用空间不足 10%
+        description: 检查 volume 文件系统容量、保留策略与底层 ZFS；不要直接删除卷文件。
+        runbook_url: https://git.ddupan.top/panxiao81/homelab-wiki/src/branch/main/guides/monitoring-data-services.md
+- name: NATS 历史慢消费者计数不持续报警
+  interval: 1m
+  input_series:
+  - series: nats_varz_slow_consumers{job="nats/nats"}
+    values: 10x30
+  alert_rule_test:
+  - eval_time: 20m
+    alertname: NatsSlowConsumers
+    exp_alerts: []
+- name: Flux 根对象状态指标缺失报警
+  interval: 1m
+  input_series: []
+  alert_rule_test:
+  - eval_time: 4m
+    alertname: FluxStateMetricsMissing
+    exp_alerts: []
+  - eval_time: 6m
+    alertname: FluxStateMetricsMissing
+    exp_alerts:
+    - exp_labels:
+        job: kube-state-metrics
+        customresource_kind: Kustomization
+        resource_namespace: flux-system
+        name: flux-system
+        severity: warning
+      exp_annotations:
+        summary: Flux 对象状态指标缺失
+        description: 检查 kube-state-metrics 自定义资源配置、RBAC 与采集；不能把空结果当作健康。
+        runbook_url: https://git.ddupan.top/panxiao81/homelab-wiki/src/branch/main/guides/monitoring-controllers.md
+- name: OCI HelmRepository 没有 Ready 条件仍不误报
+  interval: 1m
+  input_series:
+  - series: gotk_resource_info{job="kube-state-metrics",customresource_kind="HelmRepository",repository_type="oci",name="envoy-gateway"}
+    values: 1x30
+  alert_rule_test:
+  - eval_time: 20m
+    alertname: FluxResourceNotReady
+    exp_alerts: []

platform/observability/metrics/vmalert.yaml

@@ -10,6 +10,11 @@ spec:
   replicaCount: 1
   selectAllByDefault: true
   evaluationInterval: 30s
+  # 告警 Source 复用已认证的 Grafana 入口，携带表达式和触发时间。
+  # 整个 Explore JSON 一次性 URL 编码，避免表达式中的引号、&、+ 损坏链接。
+  extraArgs:
+    external.url: https://grafana.ad.ddupan.top
+    external.alert.source: 'explore?left={{ printf "{\"datasource\":\"VictoriaMetrics\",\"queries\":[{\"expr\":%s,\"refId\":\"A\"}],\"range\":{\"from\":\"%d\",\"to\":\"now\"}}" (.Expr | jsonEscape) .ActiveAt.UnixMilli | queryEscape }}'
   datasource:
     url: http://vmsingle-main.monitoring.svc:8428
   remoteWrite:

platform/observability/metrics/vmalertmanager.yaml

@@ -1,6 +1,4 @@
-# Alert router/notifier. Migrated from the retired Compose stack (see Git history),
+# Telegram token 由 ExternalSecret 从 OpenBao 投射，配置中仅引用文件。
-# which currently blackholes everything. Wire real receivers here (email via the
-# in-cluster smtp-relay, or a webhook) when you want notifications.
 apiVersion: operator.victoriametrics.com/v1beta1
 kind: VMAlertmanager
 metadata:
@@ -8,19 +6,37 @@ metadata:
   namespace: monitoring
 spec:
   replicaCount: 1
+  secrets:
+    - alertmanager-telegram
   configRawYaml: |
     route:
       receiver: blackhole
+      group_by: [alertname, cluster, job, severity]
+      group_wait: 1m
+      group_interval: 5m
+      repeat_interval: 4h
+      routes:
+        - receiver: telegram
+          matchers:
+            - severity="critical"
+          group_wait: 10s
+          repeat_interval: 1h
+        - receiver: telegram
+          matchers:
+            - severity="warning"
+    inhibit_rules:
+      - source_matchers:
+          - alertname="KubePodCrashLooping"
+        target_matchers:
+          - alertname="KubePodFrequentRestarts"
+        equal: [namespace, pod, container]
     receivers:
       - name: blackhole
-    # Example email receiver via the in-cluster Postfix relay (smtp-relay/):
+      - name: telegram
-    # receivers:
+        telegram_configs:
-    #   - name: email
+          - bot_token_file: /etc/vm/secrets/alertmanager-telegram/telegram_bot_token
-    #     email_configs:
+            chat_id: -1003956377923
-    #       - to: '[email protected]'
+            send_resolved: true
-    #         from: 'Alertmanager <[email protected]>'
-    #         smarthost: 'smtp-relay.smtp-relay.svc.cluster.local:25'
-    #         require_tls: false
   resources:
     requests:
       cpu: 25m

platform/observability/operator/helmrelease.yaml

@@ -4,6 +4,8 @@ metadata:
   name: vm-operator
   namespace: monitoring
 spec:
+  dependsOn:
+    - name: prometheus-operator-crds
   chart:
     spec:
       chart: victoria-metrics-operator