完善内存监控并迁移 Grafana 内网入口

Co-authored-by: panxiao81 <[email protected]>
2026-09-16 12:47:52 +00:00
parent fd6bd62a4f
commit 47042d4df4
11 changed files with 745 additions and 14 deletions
@@ -0,0 +1,502 @@
+{
+  "uid": "homelab-memory",
+  "title": "Homelab 内存与 Swap",
+  "schemaVersion": 39,
+  "version": 1,
+  "tags": [
+    "homelab",
+    "memory"
+  ],
+  "timezone": "utc",
+  "refresh": "1m",
+  "time": {
+    "from": "now-6h",
+    "to": "now"
+  },
+  "templating": {
+    "list": [
+      {
+        "name": "DS_VM",
+        "label": "数据源",
+        "type": "datasource",
+        "query": "prometheus",
+        "current": {
+          "text": "VictoriaMetrics",
+          "value": "VictoriaMetrics"
+        }
+      }
+    ]
+  },
+  "panels": [
+    {
+      "id": 1,
+      "type": "timeseries",
+      "title": "主机物理内存",
+      "description": "ZFS ARC 是内存的一部分；不能与此面板或 Pod/PSS 再叠加。",
+      "datasource": {
+        "type": "prometheus",
+        "uid": "${DS_VM}"
+      },
+      "gridPos": {
+        "x": 0,
+        "y": 0,
+        "w": 12,
+        "h": 8
+      },
+      "targets": [
+        {
+          "refId": "A",
+          "expr": "node_memory_MemTotal_bytes{job=\"node-exporter\"} - node_memory_MemAvailable_bytes{job=\"node-exporter\"}",
+          "legendFormat": "已用（total - available）"
+        },
+        {
+          "refId": "B",
+          "expr": "node_memory_MemAvailable_bytes{job=\"node-exporter\"}",
+          "legendFormat": "可用"
+        },
+        {
+          "refId": "C",
+          "expr": "node_memory_MemTotal_bytes{job=\"node-exporter\"}",
+          "legendFormat": "总量"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "bytes",
+          "min": 0
+        },
+        "overrides": []
+      },
+      "options": {
+        "legend": {
+          "displayMode": "table",
+          "placement": "bottom",
+          "calcs": [
+            "lastNotNull"
+          ]
+        },
+        "tooltip": {
+          "mode": "multi"
+        }
+      }
+    },
+    {
+      "id": 2,
+      "type": "timeseries",
+      "title": "ZFS ARC",
+      "description": "缓存可回收性取决于实际压力，ARC 不等同于 free 的 buff/cache。",
+      "datasource": {
+        "type": "prometheus",
+        "uid": "${DS_VM}"
+      },
+      "gridPos": {
+        "x": 12,
+        "y": 0,
+        "w": 12,
+        "h": 8
+      },
+      "targets": [
+        {
+          "refId": "A",
+          "expr": "node_zfs_arc_size{job=\"node-exporter\"}",
+          "legendFormat": "当前 ARC"
+        },
+        {
+          "refId": "B",
+          "expr": "node_zfs_arc_c{job=\"node-exporter\"}",
+          "legendFormat": "目标"
+        },
+        {
+          "refId": "C",
+          "expr": "node_zfs_arc_c_max{job=\"node-exporter\"}",
+          "legendFormat": "上限"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "bytes",
+          "min": 0
+        },
+        "overrides": []
+      },
+      "options": {
+        "legend": {
+          "displayMode": "table",
+          "placement": "bottom",
+          "calcs": [
+            "lastNotNull"
+          ]
+        },
+        "tooltip": {
+          "mode": "multi"
+        }
+      }
+    },
+    {
+      "id": 3,
+      "type": "timeseries",
+      "title": "Swap 使用量",
+      "description": "",
+      "datasource": {
+        "type": "prometheus",
+        "uid": "${DS_VM}"
+      },
+      "gridPos": {
+        "x": 0,
+        "y": 8,
+        "w": 12,
+        "h": 8
+      },
+      "targets": [
+        {
+          "refId": "A",
+          "expr": "node_memory_SwapTotal_bytes{job=\"node-exporter\"} - node_memory_SwapFree_bytes{job=\"node-exporter\"}",
+          "legendFormat": "已用"
+        },
+        {
+          "refId": "B",
+          "expr": "node_memory_SwapTotal_bytes{job=\"node-exporter\"}",
+          "legendFormat": "总量"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "bytes",
+          "min": 0
+        },
+        "overrides": []
+      },
+      "options": {
+        "legend": {
+          "displayMode": "table",
+          "placement": "bottom",
+          "calcs": [
+            "lastNotNull"
+          ]
+        },
+        "tooltip": {
+          "mode": "multi"
+        }
+      }
+    },
+    {
+      "id": 4,
+      "type": "timeseries",
+      "title": "Swap 换页速率",
+      "description": "单位为页/秒，不假定页大小。",
+      "datasource": {
+        "type": "prometheus",
+        "uid": "${DS_VM}"
+      },
+      "gridPos": {
+        "x": 12,
+        "y": 8,
+        "w": 12,
+        "h": 8
+      },
+      "targets": [
+        {
+          "refId": "A",
+          "expr": "rate(node_vmstat_pswpin{job=\"node-exporter\"}[5m])",
+          "legendFormat": "读入 pages/s"
+        },
+        {
+          "refId": "B",
+          "expr": "rate(node_vmstat_pswpout{job=\"node-exporter\"}[5m])",
+          "legendFormat": "写出 pages/s"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "ops",
+          "min": 0
+        },
+        "overrides": []
+      },
+      "options": {
+        "legend": {
+          "displayMode": "table",
+          "placement": "bottom",
+          "calcs": [
+            "lastNotNull"
+          ]
+        },
+        "tooltip": {
+          "mode": "multi"
+        }
+      }
+    },
+    {
+      "id": 5,
+      "type": "timeseries",
+      "title": "程序与虚拟机 PSS：前 15",
+      "description": "PSS 按共享页比例分摊。vm: 表示 QEMU 在宿主机的占用，不是来宾内部应用占用。与 Pod working set、ARC 不能相加。",
+      "datasource": {
+        "type": "prometheus",
+        "uid": "${DS_VM}"
+      },
+      "gridPos": {
+        "x": 0,
+        "y": 16,
+        "w": 12,
+        "h": 8
+      },
+      "targets": [
+        {
+          "refId": "A",
+          "expr": "topk(15, namedprocess_namegroup_memory_bytes{job=\"process-exporter\",memtype=\"proportionalResident\"})",
+          "legendFormat": "{{groupname}}"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "bytes",
+          "min": 0
+        },
+        "overrides": []
+      },
+      "options": {
+        "legend": {
+          "displayMode": "table",
+          "placement": "bottom",
+          "calcs": [
+            "lastNotNull"
+          ]
+        },
+        "tooltip": {
+          "mode": "multi"
+        }
+      }
+    },
+    {
+      "id": 6,
+      "type": "timeseries",
+      "title": "程序与虚拟机 SwapPss：前 15",
+      "description": "通过 smaps 对共享换出页按比例分摊。",
+      "datasource": {
+        "type": "prometheus",
+        "uid": "${DS_VM}"
+      },
+      "gridPos": {
+        "x": 12,
+        "y": 16,
+        "w": 12,
+        "h": 8
+      },
+      "targets": [
+        {
+          "refId": "A",
+          "expr": "topk(15, namedprocess_namegroup_memory_bytes{job=\"process-exporter\",memtype=\"proportionalSwapped\"})",
+          "legendFormat": "{{groupname}}"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "bytes",
+          "min": 0
+        },
+        "overrides": []
+      },
+      "options": {
+        "legend": {
+          "displayMode": "table",
+          "placement": "bottom",
+          "calcs": [
+            "lastNotNull"
+          ]
+        },
+        "tooltip": {
+          "mode": "multi"
+        }
+      }
+    },
+    {
+      "id": 7,
+      "type": "timeseries",
+      "title": "Pod working set：前 15",
+      "description": "cgroup working set 与 PSS 口径不同，不相加。",
+      "datasource": {
+        "type": "prometheus",
+        "uid": "${DS_VM}"
+      },
+      "gridPos": {
+        "x": 0,
+        "y": 24,
+        "w": 12,
+        "h": 8
+      },
+      "targets": [
+        {
+          "refId": "A",
+          "expr": "topk(15, sum by (namespace,pod) (container_memory_working_set_bytes{job=\"cadvisor\",container!=\"\",container!=\"POD\",pod!=\"\"}))",
+          "legendFormat": "{{namespace}}/{{pod}}"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "bytes",
+          "min": 0
+        },
+        "overrides": []
+      },
+      "options": {
+        "legend": {
+          "displayMode": "table",
+          "placement": "bottom",
+          "calcs": [
+            "lastNotNull"
+          ]
+        },
+        "tooltip": {
+          "mode": "multi"
+        }
+      }
+    },
+    {
+      "id": 8,
+      "type": "timeseries",
+      "title": "内存压力 PSI",
+      "description": "",
+      "datasource": {
+        "type": "prometheus",
+        "uid": "${DS_VM}"
+      },
+      "gridPos": {
+        "x": 12,
+        "y": 24,
+        "w": 12,
+        "h": 8
+      },
+      "targets": [
+        {
+          "refId": "A",
+          "expr": "rate(node_pressure_memory_waiting_seconds_total{job=\"node-exporter\"}[5m])",
+          "legendFormat": "some"
+        },
+        {
+          "refId": "B",
+          "expr": "rate(node_pressure_memory_stalled_seconds_total{job=\"node-exporter\"}[5m])",
+          "legendFormat": "full"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "percentunit",
+          "min": 0
+        },
+        "overrides": []
+      },
+      "options": {
+        "legend": {
+          "displayMode": "table",
+          "placement": "bottom",
+          "calcs": [
+            "lastNotNull"
+          ]
+        },
+        "tooltip": {
+          "mode": "multi"
+        }
+      }
+    },
+    {
+      "id": 9,
+      "type": "timeseries",
+      "title": "Samba RPC worker 数量",
+      "description": "仅在 exporter 健康时将无 worker 解释为 0；见采集健康面板。",
+      "datasource": {
+        "type": "prometheus",
+        "uid": "${DS_VM}"
+      },
+      "gridPos": {
+        "x": 0,
+        "y": 32,
+        "w": 12,
+        "h": 8
+      },
+      "targets": [
+        {
+          "refId": "A",
+          "expr": "namedprocess_namegroup_num_procs{job=\"process-exporter\",groupname=\"rpcd_lsad\"} or on() (0 * max(up{job=\"process-exporter\"} == 1))",
+          "legendFormat": "rpcd_lsad"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "short",
+          "min": 0
+        },
+        "overrides": []
+      },
+      "options": {
+        "legend": {
+          "displayMode": "table",
+          "placement": "bottom",
+          "calcs": [
+            "lastNotNull"
+          ]
+        },
+        "tooltip": {
+          "mode": "multi"
+        }
+      }
+    },
+    {
+      "id": 10,
+      "type": "timeseries",
+      "title": "进程采集健康",
+      "description": "up 仅表示抓取成功；还需检查读取错误。进程退出等竞态可能造成偶发 partial errors，持续增长时检查 AppArmor 审计。",
+      "datasource": {
+        "type": "prometheus",
+        "uid": "${DS_VM}"
+      },
+      "gridPos": {
+        "x": 12,
+        "y": 32,
+        "w": 12,
+        "h": 8
+      },
+      "targets": [
+        {
+          "refId": "A",
+          "expr": "up{job=\"process-exporter\"}",
+          "legendFormat": "up"
+        },
+        {
+          "refId": "B",
+          "expr": "scrape_duration_seconds{job=\"process-exporter\"}",
+          "legendFormat": "scrape 秒"
+        },
+        {
+          "refId": "C",
+          "expr": "namedprocess_scrape_errors{job=\"process-exporter\"}",
+          "legendFormat": "采集错误"
+        },
+        {
+          "refId": "D",
+          "expr": "rate(namedprocess_scrape_partial_errors{job=\"process-exporter\"}[5m])",
+          "legendFormat": "部分字段读取失败/s"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "short",
+          "min": 0
+        },
+        "overrides": []
+      },
+      "options": {
+        "legend": {
+          "displayMode": "table",
+          "placement": "bottom",
+          "calcs": [
+            "lastNotNull"
+          ]
+        },
+        "tooltip": {
+          "mode": "multi"
+        }
+      }
+    }
+  ]
+}
@@ -0,0 +1,17 @@
+# Grafana 自身使用 OIDC，不增加第二层 forward-auth。
+apiVersion: gateway.networking.k8s.io/v1
+kind: HTTPRoute
+metadata:
+  name: grafana-lan
+  namespace: monitoring
+spec:
+  parentRefs:
+    - name: eg
+      namespace: envoy-gateway-system
+      sectionName: https
+  hostnames:
+    - grafana.ad.ddupan.top
+  rules:
+    - backendRefs:
+        - name: grafana
+          port: 80
@@ -17,5 +17,13 @@ configMapGenerator:
    options:
      labels:
        grafana_dashboard: "1"
+  - name: grafana-dashboard-homelab-memory
+    namespace: monitoring
+    files:
+      - homelab-memory.json=dashboards/homelab-memory.json
+    options:
+      labels:
+        grafana_dashboard: "1"
 resources:
  - helmrelease.yaml
+  - httproute.yaml
@@ -1,6 +1,6 @@
 # Grafana — single pane over metrics (VictoriaMetrics), logs (VictoriaLogs) and
-# traces (VictoriaTraces via Jaeger API). Exposed privately on the tailnet
-# (grafana.tail7e769.ts.net) and authenticated via Authelia OIDC. Local admin is
+# traces (VictoriaTraces via Jaeger API). LAN: grafana.ad.ddupan.top,
+# authenticated via Authelia OIDC. Local admin is
 # break-glass only.
 #
 # Chart: grafana/grafana   (repo: https://grafana.github.io/helm-charts)
@@ -54,16 +54,10 @@ persistence:
 deploymentStrategy:
  type: Recreate

-# --- Private exposure via the Tailscale ingress (like seaweedfs-admin) ---
-# The tailscale operator provisions grafana.<tailnet>.ts.net and a TLS cert.
+# 内网入口由 httproute.yaml 接入 Envoy，使用现有通配 TLS 证书。
+# 客户端通过既有内网 DNS 转发解析，无需独立 Tailscale Ingress。
 ingress:
-  enabled: true
-  ingressClassName: tailscale
-  hosts:
-    - grafana
-  tls:
-    - hosts:
-        - grafana
+  enabled: false

 # --- OIDC via Authelia (AD groups -> Grafana roles) ---
 # client_secret is injected from the grafana-oidc Secret (see oidc-secret.yaml),
@@ -76,7 +70,7 @@ envValueFrom:

 grafana.ini:
  server:
-    root_url: "https://grafana.tail7e769.ts.net"   # must match the tailnet FQDN + Authelia redirect_uri
+    root_url: "https://grafana.ad.ddupan.top"   # 与 Authelia redirect_uri 一致
  auth:
    # Keep the local admin login available as break-glass; don't force OIDC-only.
    disable_login_form: false