feat: 接入 sandbox Kubernetes 监控

clusters/sandbox/README.md

@@ -10,7 +10,7 @@ Ansible 将 homelab CA 注入 `GitRepository/flux-system` 引用的同名 Secret
 长期 Git 凭据。root Kustomization 从 `./clusters/sandbox` 开始 reconciliation，
 初始保持 `prune: false`。
 
-当前 root 为空，作为 bootstrap canary。后续按依赖顺序分别引入：
+Root bootstrap 已完成。后续按依赖顺序分别引入：
 
 1. 监控 CRD、kube-state-metrics 以及 kubelet/cAdvisor 抓取配置；
 2. SPIRE Agent、SPIFFE CSI Driver 与 workload registration；
@@ -18,6 +18,8 @@ Ansible 将 homelab CA 注入 `GitRepository/flux-system` 引用的同名 Secret
 4. OpenSandbox operator/server 及 `ci-pod`、`ci-vm` Pools。
 
 每一阶段单独合并并等待对应 Flux Kustomization Ready，不在 bootstrap 时一次性部署。
+第一阶段监控拆为 `monitoring-operator` 与依赖它的 `monitoring`，防止 VM CR 在
+VictoriaMetrics Operator CRD Ready 前进入 reconciliation。
 
 ## 监控边界
 

clusters/sandbox/apps/monitoring-operator.yaml

@@ -0,0 +1,15 @@
+---
+apiVersion: kustomize.toolkit.fluxcd.io/v1
+kind: Kustomization
+metadata:
+  name: monitoring-operator
+  namespace: flux-system
+spec:
+  interval: 10m
+  path: ./platform/sandbox-monitoring/operator
+  prune: true
+  sourceRef:
+    kind: GitRepository
+    name: flux-system
+  timeout: 10m
+  wait: true

clusters/sandbox/apps/monitoring.yaml

@@ -0,0 +1,17 @@
+---
+apiVersion: kustomize.toolkit.fluxcd.io/v1
+kind: Kustomization
+metadata:
+  name: monitoring
+  namespace: flux-system
+spec:
+  dependsOn:
+    - name: monitoring-operator
+  interval: 10m
+  path: ./platform/sandbox-monitoring/workloads
+  prune: true
+  sourceRef:
+    kind: GitRepository
+    name: flux-system
+  timeout: 10m
+  wait: true

clusters/sandbox/kustomization.yaml

@@ -1,4 +1,6 @@
 ---
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
-resources: []
+resources:
+  - apps/monitoring-operator.yaml
+  - apps/monitoring.yaml

infrastructure/dns/records.yml

@@ -15,6 +15,7 @@ homelab_dns:
       - { zone: ad.ddupan.top, name: sandbox-k8s, type: A, values: [10.60.0.13] }
       - { zone: ad.ddupan.top, name: retrolab, type: A, values: [10.60.0.10] }
       - { zone: ad.ddupan.top, name: grafana, type: A, values: [192.168.10.127] }
+      - { zone: ad.ddupan.top, name: metrics-write, type: A, values: [192.168.10.127] }
       - { zone: ad.ddupan.top, name: netbox, type: A, values: [192.168.10.127] }
       - { zone: ad.ddupan.top, name: nats, type: A, values: [192.168.10.127] }
       - { zone: ad.ddupan.top, name: s3, type: A, values: [192.168.10.127] }

platform/observability/metrics/kustomization.yaml

@@ -2,6 +2,7 @@ apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 resources:
   - vmsingle.yaml
+  - vmsingle-write-route.yaml
   - vmagent.yaml
   - vmalert.yaml
   - vmalertmanager.yaml

platform/observability/metrics/vmsingle-write-route.yaml

@@ -0,0 +1,24 @@
+---
+# Internal-only remote_write ingress for vmagent instances in other homelab
+# clusters. Expose only the write endpoint, not VictoriaMetrics query/admin APIs.
+apiVersion: gateway.networking.k8s.io/v1
+kind: HTTPRoute
+metadata:
+  name: vmsingle-remote-write
+  namespace: monitoring
+spec:
+  parentRefs:
+    - name: eg
+      namespace: envoy-gateway-system
+      sectionName: https
+  hostnames:
+    - metrics-write.ad.ddupan.top
+  rules:
+    - matches:
+        - method: POST
+          path:
+            type: Exact
+            value: /api/v1/write
+      backendRefs:
+        - name: vmsingle-main
+          port: 8428

platform/sandbox-monitoring/README.md

@@ -0,0 +1,15 @@
+# Sandbox 监控
+
+该目录只采集 sandbox Kubernetes 与 workload 指标：
+
+- kubelet 与 cAdvisor；
+- kube-state-metrics；
+- vmagent 自身运行状态。
+
+不得在 sandbox LXC 内部署 node_exporter。LXC 的 `/proc/stat` 暴露 PVE 宿主 CPU
+视图，会产生重复且语义混合的指标。PVE 宿主监控不属于本目录。
+
+vmagent 为所有远端样本增加 `cluster=sandbox`，并通过仅允许
+`POST /api/v1/write` 的 `metrics-write.ad.ddupan.top` 路由写入 homelab VMSingle。
+Operator 与 workload 拆成两个 Flux Kustomization，确保 VM CRD Ready 后再创建
+VMAgent、VMNodeScrape 和 VMServiceScrape。

platform/sandbox-monitoring/operator/helmrelease.yaml

@@ -0,0 +1,32 @@
+---
+apiVersion: helm.toolkit.fluxcd.io/v2
+kind: HelmRelease
+metadata:
+  name: vm-operator
+  namespace: monitoring
+spec:
+  chart:
+    spec:
+      chart: victoria-metrics-operator
+      interval: 1h
+      sourceRef:
+        kind: HelmRepository
+        name: vm
+      version: 0.66.2
+  install:
+    crds: CreateReplace
+    strategy:
+      name: RetryOnFailure
+      retryInterval: 5m
+  interval: 30m
+  releaseName: vm-operator
+  targetNamespace: monitoring
+  timeout: 10m
+  upgrade:
+    crds: CreateReplace
+    strategy:
+      name: RetryOnFailure
+      retryInterval: 5m
+  valuesFrom:
+    - kind: ConfigMap
+      name: vm-operator-values

platform/sandbox-monitoring/operator/kustomization.yaml

@@ -0,0 +1,8 @@
+---
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - namespace.yaml
+  - repositories.yaml
+  - values.yaml
+  - helmrelease.yaml

platform/sandbox-monitoring/operator/namespace.yaml

@@ -0,0 +1,5 @@
+---
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: monitoring

platform/sandbox-monitoring/operator/repositories.yaml

@@ -0,0 +1,18 @@
+---
+apiVersion: source.toolkit.fluxcd.io/v1
+kind: HelmRepository
+metadata:
+  name: vm
+  namespace: monitoring
+spec:
+  interval: 1h
+  url: https://victoriametrics.github.io/helm-charts/
+---
+apiVersion: source.toolkit.fluxcd.io/v1
+kind: HelmRepository
+metadata:
+  name: prometheus-community
+  namespace: monitoring
+spec:
+  interval: 1h
+  url: https://prometheus-community.github.io/helm-charts

platform/sandbox-monitoring/operator/values.yaml

@@ -0,0 +1,23 @@
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: vm-operator-values
+  namespace: monitoring
+data:
+  values.yaml: |
+    crds:
+      plain: true
+      cleanup:
+        enabled: false
+    operator:
+      disable_prometheus_converter: true
+    serviceMonitor:
+      enabled: false
+    resources:
+      requests:
+        cpu: 25m
+        memory: 96Mi
+      limits:
+        cpu: 250m
+        memory: 256Mi

platform/sandbox-monitoring/workloads/kube-state-metrics-scrape.yaml

@@ -0,0 +1,14 @@
+---
+apiVersion: operator.victoriametrics.com/v1beta1
+kind: VMServiceScrape
+metadata:
+  name: kube-state-metrics
+  namespace: monitoring
+spec:
+  endpoints:
+    - interval: 30s
+      port: http
+  selector:
+    matchLabels:
+      app.kubernetes.io/instance: kube-state-metrics
+      app.kubernetes.io/name: kube-state-metrics

platform/sandbox-monitoring/workloads/kube-state-metrics.yaml

@@ -0,0 +1,39 @@
+---
+apiVersion: helm.toolkit.fluxcd.io/v2
+kind: HelmRelease
+metadata:
+  name: kube-state-metrics
+  namespace: monitoring
+spec:
+  chart:
+    spec:
+      chart: kube-state-metrics
+      interval: 1h
+      sourceRef:
+        kind: HelmRepository
+        name: prometheus-community
+        namespace: monitoring
+      version: 8.3.0
+  install:
+    strategy:
+      name: RetryOnFailure
+      retryInterval: 5m
+  interval: 30m
+  releaseName: kube-state-metrics
+  targetNamespace: monitoring
+  timeout: 10m
+  upgrade:
+    strategy:
+      name: RetryOnFailure
+      retryInterval: 5m
+  values:
+    prometheus:
+      monitor:
+        enabled: false
+    resources:
+      requests:
+        cpu: 25m
+        memory: 64Mi
+      limits:
+        cpu: 250m
+        memory: 256Mi

platform/sandbox-monitoring/workloads/kubelet-scrapes.yaml

@@ -0,0 +1,41 @@
+---
+apiVersion: operator.victoriametrics.com/v1beta1
+kind: VMNodeScrape
+metadata:
+  name: kubelet
+  namespace: monitoring
+spec:
+  bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token
+  honorLabels: true
+  honorTimestamps: false
+  interval: 30s
+  scheme: https
+  tlsConfig:
+    caFile: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
+    insecureSkipVerify: true
+  relabelConfigs:
+    - action: labelmap
+      regex: __meta_kubernetes_node_label_(.+)
+    - targetLabel: job
+      replacement: kubelet
+---
+apiVersion: operator.victoriametrics.com/v1beta1
+kind: VMNodeScrape
+metadata:
+  name: cadvisor
+  namespace: monitoring
+spec:
+  bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token
+  honorLabels: true
+  honorTimestamps: false
+  interval: 30s
+  path: /metrics/cadvisor
+  scheme: https
+  tlsConfig:
+    caFile: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
+    insecureSkipVerify: true
+  relabelConfigs:
+    - action: labelmap
+      regex: __meta_kubernetes_node_label_(.+)
+    - targetLabel: job
+      replacement: cadvisor

platform/sandbox-monitoring/workloads/kustomization.yaml

@@ -0,0 +1,8 @@
+---
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - vmagent.yaml
+  - kubelet-scrapes.yaml
+  - kube-state-metrics.yaml
+  - kube-state-metrics-scrape.yaml

platform/sandbox-monitoring/workloads/vmagent.yaml

@@ -0,0 +1,21 @@
+---
+apiVersion: operator.victoriametrics.com/v1beta1
+kind: VMAgent
+metadata:
+  name: sandbox
+  namespace: monitoring
+spec:
+  externalLabels:
+    cluster: sandbox
+  remoteWrite:
+    - url: https://metrics-write.ad.ddupan.top/api/v1/write
+  replicaCount: 1
+  resources:
+    requests:
+      cpu: 50m
+      memory: 128Mi
+    limits:
+      cpu: 500m
+      memory: 512Mi
+  scrapeInterval: 30s
+  selectAllByDefault: true