From 60836c33602f15121e47ed844b5e63022acde79d Mon Sep 17 00:00:00 2001 From: panxiao81 Date: Thu, 17 Sep 2026 16:02:48 +0000 Subject: [PATCH] =?UTF-8?q?feat:=20=E6=8E=A5=E5=85=A5=20sandbox=20Kubernet?= =?UTF-8?q?es=20=E7=9B=91=E6=8E=A7?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- clusters/sandbox/README.md | 4 +- .../sandbox/apps/monitoring-operator.yaml | 15 +++++++ clusters/sandbox/apps/monitoring.yaml | 17 ++++++++ clusters/sandbox/kustomization.yaml | 4 +- infrastructure/dns/records.yml | 1 + .../observability/metrics/kustomization.yaml | 1 + .../metrics/vmsingle-write-route.yaml | 24 +++++++++++ platform/sandbox-monitoring/README.md | 15 +++++++ .../operator/helmrelease.yaml | 32 +++++++++++++++ .../operator/kustomization.yaml | 8 ++++ .../operator/namespace.yaml | 5 +++ .../operator/repositories.yaml | 18 ++++++++ .../sandbox-monitoring/operator/values.yaml | 23 +++++++++++ .../workloads/kube-state-metrics-scrape.yaml | 14 +++++++ .../workloads/kube-state-metrics.yaml | 39 ++++++++++++++++++ .../workloads/kubelet-scrapes.yaml | 41 +++++++++++++++++++ .../workloads/kustomization.yaml | 8 ++++ .../sandbox-monitoring/workloads/vmagent.yaml | 21 ++++++++++ 18 files changed, 288 insertions(+), 2 deletions(-) create mode 100644 clusters/sandbox/apps/monitoring-operator.yaml create mode 100644 clusters/sandbox/apps/monitoring.yaml create mode 100644 platform/observability/metrics/vmsingle-write-route.yaml create mode 100644 platform/sandbox-monitoring/README.md create mode 100644 platform/sandbox-monitoring/operator/helmrelease.yaml create mode 100644 platform/sandbox-monitoring/operator/kustomization.yaml create mode 100644 platform/sandbox-monitoring/operator/namespace.yaml create mode 100644 platform/sandbox-monitoring/operator/repositories.yaml create mode 100644 platform/sandbox-monitoring/operator/values.yaml create mode 100644 platform/sandbox-monitoring/workloads/kube-state-metrics-scrape.yaml create mode 100644 platform/sandbox-monitoring/workloads/kube-state-metrics.yaml create mode 100644 platform/sandbox-monitoring/workloads/kubelet-scrapes.yaml create mode 100644 platform/sandbox-monitoring/workloads/kustomization.yaml create mode 100644 platform/sandbox-monitoring/workloads/vmagent.yaml diff --git a/clusters/sandbox/README.md b/clusters/sandbox/README.md index 395474c..c5c4a1f 100644 --- a/clusters/sandbox/README.md +++ b/clusters/sandbox/README.md @@ -10,7 +10,7 @@ Ansible 将 homelab CA 注入 `GitRepository/flux-system` 引用的同名 Secret 长期 Git 凭据。root Kustomization 从 `./clusters/sandbox` 开始 reconciliation， 初始保持 `prune: false`。 -当前 root 为空，作为 bootstrap canary。后续按依赖顺序分别引入： +Root bootstrap 已完成。后续按依赖顺序分别引入： 1. 监控 CRD、kube-state-metrics 以及 kubelet/cAdvisor 抓取配置； 2. SPIRE Agent、SPIFFE CSI Driver 与 workload registration； @@ -18,6 +18,8 @@ Ansible 将 homelab CA 注入 `GitRepository/flux-system` 引用的同名 Secret 4. OpenSandbox operator/server 及 `ci-pod`、`ci-vm` Pools。 每一阶段单独合并并等待对应 Flux Kustomization Ready，不在 bootstrap 时一次性部署。 +第一阶段监控拆为 `monitoring-operator` 与依赖它的 `monitoring`，防止 VM CR 在 +VictoriaMetrics Operator CRD Ready 前进入 reconciliation。 ## 监控边界 diff --git a/clusters/sandbox/apps/monitoring-operator.yaml b/clusters/sandbox/apps/monitoring-operator.yaml new file mode 100644 index 0000000..b48c307 --- /dev/null +++ b/clusters/sandbox/apps/monitoring-operator.yaml @@ -0,0 +1,15 @@ +--- +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: monitoring-operator + namespace: flux-system +spec: + interval: 10m + path: ./platform/sandbox-monitoring/operator + prune: true + sourceRef: + kind: GitRepository + name: flux-system + timeout: 10m + wait: true diff --git a/clusters/sandbox/apps/monitoring.yaml b/clusters/sandbox/apps/monitoring.yaml new file mode 100644 index 0000000..ae704f0 --- /dev/null +++ b/clusters/sandbox/apps/monitoring.yaml @@ -0,0 +1,17 @@ +--- +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: monitoring + namespace: flux-system +spec: + dependsOn: + - name: monitoring-operator + interval: 10m + path: ./platform/sandbox-monitoring/workloads + prune: true + sourceRef: + kind: GitRepository + name: flux-system + timeout: 10m + wait: true diff --git a/clusters/sandbox/kustomization.yaml b/clusters/sandbox/kustomization.yaml index fe0f332..30a0613 100644 --- a/clusters/sandbox/kustomization.yaml +++ b/clusters/sandbox/kustomization.yaml @@ -1,4 +1,6 @@ --- apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization -resources: [] +resources: + - apps/monitoring-operator.yaml + - apps/monitoring.yaml diff --git a/infrastructure/dns/records.yml b/infrastructure/dns/records.yml index 2562783..6b5607c 100644 --- a/infrastructure/dns/records.yml +++ b/infrastructure/dns/records.yml @@ -15,6 +15,7 @@ homelab_dns: - { zone: ad.ddupan.top, name: sandbox-k8s, type: A, values: [10.60.0.13] } - { zone: ad.ddupan.top, name: retrolab, type: A, values: [10.60.0.10] } - { zone: ad.ddupan.top, name: grafana, type: A, values: [192.168.10.127] } + - { zone: ad.ddupan.top, name: metrics-write, type: A, values: [192.168.10.127] } - { zone: ad.ddupan.top, name: netbox, type: A, values: [192.168.10.127] } - { zone: ad.ddupan.top, name: nats, type: A, values: [192.168.10.127] } - { zone: ad.ddupan.top, name: s3, type: A, values: [192.168.10.127] } diff --git a/platform/observability/metrics/kustomization.yaml b/platform/observability/metrics/kustomization.yaml index 888fffc..3c8f70f 100644 --- a/platform/observability/metrics/kustomization.yaml +++ b/platform/observability/metrics/kustomization.yaml @@ -2,6 +2,7 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: - vmsingle.yaml + - vmsingle-write-route.yaml - vmagent.yaml - vmalert.yaml - vmalertmanager.yaml diff --git a/platform/observability/metrics/vmsingle-write-route.yaml b/platform/observability/metrics/vmsingle-write-route.yaml new file mode 100644 index 0000000..396b5a6 --- /dev/null +++ b/platform/observability/metrics/vmsingle-write-route.yaml @@ -0,0 +1,24 @@ +--- +# Internal-only remote_write ingress for vmagent instances in other homelab +# clusters. Expose only the write endpoint, not VictoriaMetrics query/admin APIs. +apiVersion: gateway.networking.k8s.io/v1 +kind: HTTPRoute +metadata: + name: vmsingle-remote-write + namespace: monitoring +spec: + parentRefs: + - name: eg + namespace: envoy-gateway-system + sectionName: https + hostnames: + - metrics-write.ad.ddupan.top + rules: + - matches: + - method: POST + path: + type: Exact + value: /api/v1/write + backendRefs: + - name: vmsingle-main + port: 8428 diff --git a/platform/sandbox-monitoring/README.md b/platform/sandbox-monitoring/README.md new file mode 100644 index 0000000..7dda179 --- /dev/null +++ b/platform/sandbox-monitoring/README.md @@ -0,0 +1,15 @@ +# Sandbox 监控 + +该目录只采集 sandbox Kubernetes 与 workload 指标： + +- kubelet 与 cAdvisor； +- kube-state-metrics； +- vmagent 自身运行状态。 + +不得在 sandbox LXC 内部署 node_exporter。LXC 的 `/proc/stat` 暴露 PVE 宿主 CPU +视图，会产生重复且语义混合的指标。PVE 宿主监控不属于本目录。 + +vmagent 为所有远端样本增加 `cluster=sandbox`，并通过仅允许 +`POST /api/v1/write` 的 `metrics-write.ad.ddupan.top` 路由写入 homelab VMSingle。 +Operator 与 workload 拆成两个 Flux Kustomization，确保 VM CRD Ready 后再创建 +VMAgent、VMNodeScrape 和 VMServiceScrape。 diff --git a/platform/sandbox-monitoring/operator/helmrelease.yaml b/platform/sandbox-monitoring/operator/helmrelease.yaml new file mode 100644 index 0000000..4017443 --- /dev/null +++ b/platform/sandbox-monitoring/operator/helmrelease.yaml @@ -0,0 +1,32 @@ +--- +apiVersion: helm.toolkit.fluxcd.io/v2 +kind: HelmRelease +metadata: + name: vm-operator + namespace: monitoring +spec: + chart: + spec: + chart: victoria-metrics-operator + interval: 1h + sourceRef: + kind: HelmRepository + name: vm + version: 0.66.2 + install: + crds: CreateReplace + strategy: + name: RetryOnFailure + retryInterval: 5m + interval: 30m + releaseName: vm-operator + targetNamespace: monitoring + timeout: 10m + upgrade: + crds: CreateReplace + strategy: + name: RetryOnFailure + retryInterval: 5m + valuesFrom: + - kind: ConfigMap + name: vm-operator-values diff --git a/platform/sandbox-monitoring/operator/kustomization.yaml b/platform/sandbox-monitoring/operator/kustomization.yaml new file mode 100644 index 0000000..2d7cdae --- /dev/null +++ b/platform/sandbox-monitoring/operator/kustomization.yaml @@ -0,0 +1,8 @@ +--- +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - namespace.yaml + - repositories.yaml + - values.yaml + - helmrelease.yaml diff --git a/platform/sandbox-monitoring/operator/namespace.yaml b/platform/sandbox-monitoring/operator/namespace.yaml new file mode 100644 index 0000000..ff7ae1b --- /dev/null +++ b/platform/sandbox-monitoring/operator/namespace.yaml @@ -0,0 +1,5 @@ +--- +apiVersion: v1 +kind: Namespace +metadata: + name: monitoring diff --git a/platform/sandbox-monitoring/operator/repositories.yaml b/platform/sandbox-monitoring/operator/repositories.yaml new file mode 100644 index 0000000..8587b95 --- /dev/null +++ b/platform/sandbox-monitoring/operator/repositories.yaml @@ -0,0 +1,18 @@ +--- +apiVersion: source.toolkit.fluxcd.io/v1 +kind: HelmRepository +metadata: + name: vm + namespace: monitoring +spec: + interval: 1h + url: https://victoriametrics.github.io/helm-charts/ +--- +apiVersion: source.toolkit.fluxcd.io/v1 +kind: HelmRepository +metadata: + name: prometheus-community + namespace: monitoring +spec: + interval: 1h + url: https://prometheus-community.github.io/helm-charts diff --git a/platform/sandbox-monitoring/operator/values.yaml b/platform/sandbox-monitoring/operator/values.yaml new file mode 100644 index 0000000..2b55ec4 --- /dev/null +++ b/platform/sandbox-monitoring/operator/values.yaml @@ -0,0 +1,23 @@ +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: vm-operator-values + namespace: monitoring +data: + values.yaml: | + crds: + plain: true + cleanup: + enabled: false + operator: + disable_prometheus_converter: true + serviceMonitor: + enabled: false + resources: + requests: + cpu: 25m + memory: 96Mi + limits: + cpu: 250m + memory: 256Mi diff --git a/platform/sandbox-monitoring/workloads/kube-state-metrics-scrape.yaml b/platform/sandbox-monitoring/workloads/kube-state-metrics-scrape.yaml new file mode 100644 index 0000000..0703b8e --- /dev/null +++ b/platform/sandbox-monitoring/workloads/kube-state-metrics-scrape.yaml @@ -0,0 +1,14 @@ +--- +apiVersion: operator.victoriametrics.com/v1beta1 +kind: VMServiceScrape +metadata: + name: kube-state-metrics + namespace: monitoring +spec: + endpoints: + - interval: 30s + port: http + selector: + matchLabels: + app.kubernetes.io/instance: kube-state-metrics + app.kubernetes.io/name: kube-state-metrics diff --git a/platform/sandbox-monitoring/workloads/kube-state-metrics.yaml b/platform/sandbox-monitoring/workloads/kube-state-metrics.yaml new file mode 100644 index 0000000..68c43bc --- /dev/null +++ b/platform/sandbox-monitoring/workloads/kube-state-metrics.yaml @@ -0,0 +1,39 @@ +--- +apiVersion: helm.toolkit.fluxcd.io/v2 +kind: HelmRelease +metadata: + name: kube-state-metrics + namespace: monitoring +spec: + chart: + spec: + chart: kube-state-metrics + interval: 1h + sourceRef: + kind: HelmRepository + name: prometheus-community + namespace: monitoring + version: 8.3.0 + install: + strategy: + name: RetryOnFailure + retryInterval: 5m + interval: 30m + releaseName: kube-state-metrics + targetNamespace: monitoring + timeout: 10m + upgrade: + strategy: + name: RetryOnFailure + retryInterval: 5m + values: + prometheus: + monitor: + enabled: false + resources: + requests: + cpu: 25m + memory: 64Mi + limits: + cpu: 250m + memory: 256Mi diff --git a/platform/sandbox-monitoring/workloads/kubelet-scrapes.yaml b/platform/sandbox-monitoring/workloads/kubelet-scrapes.yaml new file mode 100644 index 0000000..35984e1 --- /dev/null +++ b/platform/sandbox-monitoring/workloads/kubelet-scrapes.yaml @@ -0,0 +1,41 @@ +--- +apiVersion: operator.victoriametrics.com/v1beta1 +kind: VMNodeScrape +metadata: + name: kubelet + namespace: monitoring +spec: + bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token + honorLabels: true + honorTimestamps: false + interval: 30s + scheme: https + tlsConfig: + caFile: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + insecureSkipVerify: true + relabelConfigs: + - action: labelmap + regex: __meta_kubernetes_node_label_(.+) + - targetLabel: job + replacement: kubelet +--- +apiVersion: operator.victoriametrics.com/v1beta1 +kind: VMNodeScrape +metadata: + name: cadvisor + namespace: monitoring +spec: + bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token + honorLabels: true + honorTimestamps: false + interval: 30s + path: /metrics/cadvisor + scheme: https + tlsConfig: + caFile: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + insecureSkipVerify: true + relabelConfigs: + - action: labelmap + regex: __meta_kubernetes_node_label_(.+) + - targetLabel: job + replacement: cadvisor diff --git a/platform/sandbox-monitoring/workloads/kustomization.yaml b/platform/sandbox-monitoring/workloads/kustomization.yaml new file mode 100644 index 0000000..499d777 --- /dev/null +++ b/platform/sandbox-monitoring/workloads/kustomization.yaml @@ -0,0 +1,8 @@ +--- +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - vmagent.yaml + - kubelet-scrapes.yaml + - kube-state-metrics.yaml + - kube-state-metrics-scrape.yaml diff --git a/platform/sandbox-monitoring/workloads/vmagent.yaml b/platform/sandbox-monitoring/workloads/vmagent.yaml new file mode 100644 index 0000000..f2820a0 --- /dev/null +++ b/platform/sandbox-monitoring/workloads/vmagent.yaml @@ -0,0 +1,21 @@ +--- +apiVersion: operator.victoriametrics.com/v1beta1 +kind: VMAgent +metadata: + name: sandbox + namespace: monitoring +spec: + externalLabels: + cluster: sandbox + remoteWrite: + - url: https://metrics-write.ad.ddupan.top/api/v1/write + replicaCount: 1 + resources: + requests: + cpu: 50m + memory: 128Mi + limits: + cpu: 500m + memory: 512Mi + scrapeInterval: 30s + selectAllByDefault: true