完善内存监控并迁移 Grafana 内网入口
yaml / yaml (push) Successful in 20s
ansible / collection-test (push) Successful in 1m10s
ansible / lint (push) Successful in 13m9s

Co-authored-by: panxiao81 <[email protected]>
This commit was merged in pull request #63.
This commit is contained in:
2026-09-16 12:47:52 +00:00
committed by panxiao81
parent fd6bd62a4f
commit 47042d4df4
11 changed files with 745 additions and 14 deletions
+1 -1
View File
@@ -229,7 +229,7 @@ configMap:
require_pkce: false
token_endpoint_auth_method: 'client_secret_basic'
redirect_uris:
- 'https://grafana.tail7e769.ts.net/login/generic_oauth'
- 'https://grafana.ad.ddupan.top/login/generic_oauth'
scopes:
- 'openid'
- 'profile'
+1
View File
@@ -11,6 +11,7 @@ homelab_dns:
- { zone: ad.ddupan.top, name: pve2, type: A, values: [192.168.10.7] }
- { zone: ad.ddupan.top, name: pve3, type: A, values: [192.168.10.9] }
- { zone: ad.ddupan.top, name: retrolab, type: A, values: [10.60.0.10] }
- { zone: ad.ddupan.top, name: grafana, type: A, values: [192.168.10.127] }
- { zone: ad.ddupan.top, name: netbox, type: A, values: [192.168.10.127] }
- { zone: ad.ddupan.top, name: nats, type: A, values: [192.168.10.127] }
- { zone: ad.ddupan.top, name: s3, type: A, values: [192.168.10.127] }
+16 -1
View File
@@ -4,7 +4,22 @@ Metrics, logs, and traces for the cluster — one VictoriaMetrics-ecosystem stac
operator-driven, running in the `monitoring` namespace. Replaces the standalone
Docker Compose stack in `../../apps/victoriametrics/`.
## Status — DEPLOYED (2026-07-10)
## Grafana 当前入口(2026-09-16)
统一使用 <https://grafana.ad.ddupan.top>,A 记录指向 `192.168.10.127`,由共享
Envoy Gateway 的 `https` listener 与 `*.ad.ddupan.top` 证书提供 TLS。
Grafana root_url 和 Authelia 回调均使用此域名;旧专用 Tailscale Ingress 已停用。
远程客户端仍需有到 LAN 的路由及已配置的内网 DNS 转发。
内存看板:`/d/homelab-memory`;采集配置及 AppArmor 规则见
[主机与进程内存采集](metrics/exporters/README.md)。
Grafana 由 Flux 管理;修改 values 后通过 Git 合并触发 HelmRelease,避免现场 Helm
修改被漂移检测回滚。Authelia 当前不由 Flux 管理,需单独 Helm upgrade 并先结构比较
live values。迁移时先添加 DNS 和回调,再切换 Grafana。Recreate 策略会短暂中断访问,
但保留原 PVC、用户、看板和数据源。回滚需同时恢复 root_url、回调和 Ingress 配置。
## 初始部署记录(2026-07-10)
Live and verified in the `monitoring` namespace (operator chart 0.66.2):
metrics (data queryable), logs (pods ingesting), traces (VTSingle CRD, verified via
@@ -0,0 +1,502 @@
{
"uid": "homelab-memory",
"title": "Homelab 内存与 Swap",
"schemaVersion": 39,
"version": 1,
"tags": [
"homelab",
"memory"
],
"timezone": "utc",
"refresh": "1m",
"time": {
"from": "now-6h",
"to": "now"
},
"templating": {
"list": [
{
"name": "DS_VM",
"label": "数据源",
"type": "datasource",
"query": "prometheus",
"current": {
"text": "VictoriaMetrics",
"value": "VictoriaMetrics"
}
}
]
},
"panels": [
{
"id": 1,
"type": "timeseries",
"title": "主机物理内存",
"description": "ZFS ARC 是内存的一部分;不能与此面板或 Pod/PSS 再叠加。",
"datasource": {
"type": "prometheus",
"uid": "${DS_VM}"
},
"gridPos": {
"x": 0,
"y": 0,
"w": 12,
"h": 8
},
"targets": [
{
"refId": "A",
"expr": "node_memory_MemTotal_bytes{job=\"node-exporter\"} - node_memory_MemAvailable_bytes{job=\"node-exporter\"}",
"legendFormat": "已用(total - available)"
},
{
"refId": "B",
"expr": "node_memory_MemAvailable_bytes{job=\"node-exporter\"}",
"legendFormat": "可用"
},
{
"refId": "C",
"expr": "node_memory_MemTotal_bytes{job=\"node-exporter\"}",
"legendFormat": "总量"
}
],
"fieldConfig": {
"defaults": {
"unit": "bytes",
"min": 0
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "table",
"placement": "bottom",
"calcs": [
"lastNotNull"
]
},
"tooltip": {
"mode": "multi"
}
}
},
{
"id": 2,
"type": "timeseries",
"title": "ZFS ARC",
"description": "缓存可回收性取决于实际压力,ARC 不等同于 free 的 buff/cache。",
"datasource": {
"type": "prometheus",
"uid": "${DS_VM}"
},
"gridPos": {
"x": 12,
"y": 0,
"w": 12,
"h": 8
},
"targets": [
{
"refId": "A",
"expr": "node_zfs_arc_size{job=\"node-exporter\"}",
"legendFormat": "当前 ARC"
},
{
"refId": "B",
"expr": "node_zfs_arc_c{job=\"node-exporter\"}",
"legendFormat": "目标"
},
{
"refId": "C",
"expr": "node_zfs_arc_c_max{job=\"node-exporter\"}",
"legendFormat": "上限"
}
],
"fieldConfig": {
"defaults": {
"unit": "bytes",
"min": 0
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "table",
"placement": "bottom",
"calcs": [
"lastNotNull"
]
},
"tooltip": {
"mode": "multi"
}
}
},
{
"id": 3,
"type": "timeseries",
"title": "Swap 使用量",
"description": "",
"datasource": {
"type": "prometheus",
"uid": "${DS_VM}"
},
"gridPos": {
"x": 0,
"y": 8,
"w": 12,
"h": 8
},
"targets": [
{
"refId": "A",
"expr": "node_memory_SwapTotal_bytes{job=\"node-exporter\"} - node_memory_SwapFree_bytes{job=\"node-exporter\"}",
"legendFormat": "已用"
},
{
"refId": "B",
"expr": "node_memory_SwapTotal_bytes{job=\"node-exporter\"}",
"legendFormat": "总量"
}
],
"fieldConfig": {
"defaults": {
"unit": "bytes",
"min": 0
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "table",
"placement": "bottom",
"calcs": [
"lastNotNull"
]
},
"tooltip": {
"mode": "multi"
}
}
},
{
"id": 4,
"type": "timeseries",
"title": "Swap 换页速率",
"description": "单位为页/秒,不假定页大小。",
"datasource": {
"type": "prometheus",
"uid": "${DS_VM}"
},
"gridPos": {
"x": 12,
"y": 8,
"w": 12,
"h": 8
},
"targets": [
{
"refId": "A",
"expr": "rate(node_vmstat_pswpin{job=\"node-exporter\"}[5m])",
"legendFormat": "读入 pages/s"
},
{
"refId": "B",
"expr": "rate(node_vmstat_pswpout{job=\"node-exporter\"}[5m])",
"legendFormat": "写出 pages/s"
}
],
"fieldConfig": {
"defaults": {
"unit": "ops",
"min": 0
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "table",
"placement": "bottom",
"calcs": [
"lastNotNull"
]
},
"tooltip": {
"mode": "multi"
}
}
},
{
"id": 5,
"type": "timeseries",
"title": "程序与虚拟机 PSS:前 15",
"description": "PSS 按共享页比例分摊。vm: 表示 QEMU 在宿主机的占用,不是来宾内部应用占用。与 Pod working set、ARC 不能相加。",
"datasource": {
"type": "prometheus",
"uid": "${DS_VM}"
},
"gridPos": {
"x": 0,
"y": 16,
"w": 12,
"h": 8
},
"targets": [
{
"refId": "A",
"expr": "topk(15, namedprocess_namegroup_memory_bytes{job=\"process-exporter\",memtype=\"proportionalResident\"})",
"legendFormat": "{{groupname}}"
}
],
"fieldConfig": {
"defaults": {
"unit": "bytes",
"min": 0
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "table",
"placement": "bottom",
"calcs": [
"lastNotNull"
]
},
"tooltip": {
"mode": "multi"
}
}
},
{
"id": 6,
"type": "timeseries",
"title": "程序与虚拟机 SwapPss:前 15",
"description": "通过 smaps 对共享换出页按比例分摊。",
"datasource": {
"type": "prometheus",
"uid": "${DS_VM}"
},
"gridPos": {
"x": 12,
"y": 16,
"w": 12,
"h": 8
},
"targets": [
{
"refId": "A",
"expr": "topk(15, namedprocess_namegroup_memory_bytes{job=\"process-exporter\",memtype=\"proportionalSwapped\"})",
"legendFormat": "{{groupname}}"
}
],
"fieldConfig": {
"defaults": {
"unit": "bytes",
"min": 0
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "table",
"placement": "bottom",
"calcs": [
"lastNotNull"
]
},
"tooltip": {
"mode": "multi"
}
}
},
{
"id": 7,
"type": "timeseries",
"title": "Pod working set:前 15",
"description": "cgroup working set 与 PSS 口径不同,不相加。",
"datasource": {
"type": "prometheus",
"uid": "${DS_VM}"
},
"gridPos": {
"x": 0,
"y": 24,
"w": 12,
"h": 8
},
"targets": [
{
"refId": "A",
"expr": "topk(15, sum by (namespace,pod) (container_memory_working_set_bytes{job=\"cadvisor\",container!=\"\",container!=\"POD\",pod!=\"\"}))",
"legendFormat": "{{namespace}}/{{pod}}"
}
],
"fieldConfig": {
"defaults": {
"unit": "bytes",
"min": 0
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "table",
"placement": "bottom",
"calcs": [
"lastNotNull"
]
},
"tooltip": {
"mode": "multi"
}
}
},
{
"id": 8,
"type": "timeseries",
"title": "内存压力 PSI",
"description": "",
"datasource": {
"type": "prometheus",
"uid": "${DS_VM}"
},
"gridPos": {
"x": 12,
"y": 24,
"w": 12,
"h": 8
},
"targets": [
{
"refId": "A",
"expr": "rate(node_pressure_memory_waiting_seconds_total{job=\"node-exporter\"}[5m])",
"legendFormat": "some"
},
{
"refId": "B",
"expr": "rate(node_pressure_memory_stalled_seconds_total{job=\"node-exporter\"}[5m])",
"legendFormat": "full"
}
],
"fieldConfig": {
"defaults": {
"unit": "percentunit",
"min": 0
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "table",
"placement": "bottom",
"calcs": [
"lastNotNull"
]
},
"tooltip": {
"mode": "multi"
}
}
},
{
"id": 9,
"type": "timeseries",
"title": "Samba RPC worker 数量",
"description": "仅在 exporter 健康时将无 worker 解释为 0;见采集健康面板。",
"datasource": {
"type": "prometheus",
"uid": "${DS_VM}"
},
"gridPos": {
"x": 0,
"y": 32,
"w": 12,
"h": 8
},
"targets": [
{
"refId": "A",
"expr": "namedprocess_namegroup_num_procs{job=\"process-exporter\",groupname=\"rpcd_lsad\"} or on() (0 * max(up{job=\"process-exporter\"} == 1))",
"legendFormat": "rpcd_lsad"
}
],
"fieldConfig": {
"defaults": {
"unit": "short",
"min": 0
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "table",
"placement": "bottom",
"calcs": [
"lastNotNull"
]
},
"tooltip": {
"mode": "multi"
}
}
},
{
"id": 10,
"type": "timeseries",
"title": "进程采集健康",
"description": "up 仅表示抓取成功;还需检查读取错误。进程退出等竞态可能造成偶发 partial errors,持续增长时检查 AppArmor 审计。",
"datasource": {
"type": "prometheus",
"uid": "${DS_VM}"
},
"gridPos": {
"x": 12,
"y": 32,
"w": 12,
"h": 8
},
"targets": [
{
"refId": "A",
"expr": "up{job=\"process-exporter\"}",
"legendFormat": "up"
},
{
"refId": "B",
"expr": "scrape_duration_seconds{job=\"process-exporter\"}",
"legendFormat": "scrape 秒"
},
{
"refId": "C",
"expr": "namedprocess_scrape_errors{job=\"process-exporter\"}",
"legendFormat": "采集错误"
},
{
"refId": "D",
"expr": "rate(namedprocess_scrape_partial_errors{job=\"process-exporter\"}[5m])",
"legendFormat": "部分字段读取失败/s"
}
],
"fieldConfig": {
"defaults": {
"unit": "short",
"min": 0
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "table",
"placement": "bottom",
"calcs": [
"lastNotNull"
]
},
"tooltip": {
"mode": "multi"
}
}
}
]
}
@@ -0,0 +1,17 @@
# Grafana 自身使用 OIDC,不增加第二层 forward-auth。
apiVersion: gateway.networking.k8s.io/v1
kind: HTTPRoute
metadata:
name: grafana-lan
namespace: monitoring
spec:
parentRefs:
- name: eg
namespace: envoy-gateway-system
sectionName: https
hostnames:
- grafana.ad.ddupan.top
rules:
- backendRefs:
- name: grafana
port: 80
@@ -17,5 +17,13 @@ configMapGenerator:
options:
labels:
grafana_dashboard: "1"
- name: grafana-dashboard-homelab-memory
namespace: monitoring
files:
- homelab-memory.json=dashboards/homelab-memory.json
options:
labels:
grafana_dashboard: "1"
resources:
- helmrelease.yaml
- httproute.yaml
+6 -12
View File
@@ -1,6 +1,6 @@
# Grafana — single pane over metrics (VictoriaMetrics), logs (VictoriaLogs) and
# traces (VictoriaTraces via Jaeger API). Exposed privately on the tailnet
# (grafana.tail7e769.ts.net) and authenticated via Authelia OIDC. Local admin is
# traces (VictoriaTraces via Jaeger API). LAN: grafana.ad.ddupan.top,
# authenticated via Authelia OIDC. Local admin is
# break-glass only.
#
# Chart: grafana/grafana (repo: https://grafana.github.io/helm-charts)
@@ -54,16 +54,10 @@ persistence:
deploymentStrategy:
type: Recreate
# --- Private exposure via the Tailscale ingress (like seaweedfs-admin) ---
# The tailscale operator provisions grafana.<tailnet>.ts.net and a TLS cert.
# 内网入口由 httproute.yaml 接入 Envoy,使用现有通配 TLS 证书。
# 客户端通过既有内网 DNS 转发解析,无需独立 Tailscale Ingress。
ingress:
enabled: true
ingressClassName: tailscale
hosts:
- grafana
tls:
- hosts:
- grafana
enabled: false
# --- OIDC via Authelia (AD groups -> Grafana roles) ---
# client_secret is injected from the grafana-oidc Secret (see oidc-secret.yaml),
@@ -76,7 +70,7 @@ envValueFrom:
grafana.ini:
server:
root_url: "https://grafana.tail7e769.ts.net" # must match the tailnet FQDN + Authelia redirect_uri
root_url: "https://grafana.ad.ddupan.top" # 与 Authelia redirect_uri 一致
auth:
# Keep the local admin login available as break-glass; don't force OIDC-only.
disable_login_form: false
@@ -0,0 +1,42 @@
# 主机与进程内存采集
2026-09-16 已增加 `process-exporter.yaml`,固定上游 0.8.7,以 DaemonSet 运行,
只读挂载宿主机 `/proc`,每 60 秒由 VMPodScrape 抓取,不开宿主机端口。
NetworkPolicy 仅放行同 namespace 的 vmagent。进程按名称分组,QEMU 按 guest 名,
NetBox 和 VS Code 按路径归组;不把完整命令行或 PID 放进指标标签。
当前 live 使用 `gather-smaps=true`,导出 RSS、VmSwap、PSS、SwapPss 与进程数。
`apparmor/homelab-process-exporter` 已安装到 `/etc/apparmor.d/homelab-process-exporter`
并以 enforce 加载。用户明确批准了跨进程读取:SYS_PTRACE、DAC_READ_SEARCH 与
`ptrace (read) peer=**`;文件权限限于指标需要的 proc 文件,不允许 ptrace trace,
不开放 `/proc/<pid>/mem`、`environ`,不关闭 AppArmor,也未修改虚拟机或容器默认 profile。
当前只部署到 laptop。其他节点必须先安装同名 profile,再扩展 nodeSelector:
```bash
sudo install -m 0644 apparmor/homelab-process-exporter /etc/apparmor.d/homelab-process-exporter
sudo apparmor_parser -r /etc/apparmor.d/homelab-process-exporter
```
修改 profile 可原位重载,无需重启业务。禁用时先删除 exporter DaemonSet,再卸载
专用 profile;不要让引用 profile 的 Pod 在 profile 缺失时启动。
验证已读到三台虚拟机、NetBox、Codex 的非零 PSS/SwapPss。`namedprocess_scrape_errors`
和 `namedprocess_scrape_procread_errors` 为零。上游会先读取再匹配进程,内核线程没有
可读的 smaps_rollup 会计入 partial errors;现场有约 430 个这种线程。
partial errors 不保证为零,应结合业务组 PSS 和 AppArmor 审计判断。
RSS、PSS、cAdvisor working set、ARC 是不同口径,不能直接相加。
已有采集无需重复部署:
- node-exporter:主机内存、Swap、PSI、ZFS ARC。
- ARC 实际指标名为 `node_zfs_arc_size`、`node_zfs_arc_c`、`node_zfs_arc_c_max`。
- kubelet/cAdvisor:Pod/container working set、Swap 等。
Grafana 新增 `Homelab 内存与 Swap`(UID `homelab-memory`),由 sidecar ConfigMap 加载。
仅使用 `job="node-exporter"` 查询主机指标,避免目前 `docker-hosts` 对同一 9100 端口
的重复抓取。另有 `192.168.10.127:8080` Docker cAdvisor target 失效,本次未修改该旧配置。
所有新增 manifest 已纳入对应 Kustomization,并已单独应用到 live;尚未提交到远端,
Flux source 尚未包含这些新增资源。修改 exporter ConfigMap 后需滚动重启 DaemonSet,
程序不会自动重载进程匹配配置。
@@ -0,0 +1,27 @@
#include <tunables/global>
# 限定为指标所需文件;跨 profile 读取由用户明确批准;不允许 trace 或读取进程 mem/environ。
profile homelab-process-exporter flags=(attach_disconnected,mediate_deleted) {
#include <abstractions/base>
network inet stream,
network inet6 stream,
capability sys_ptrace,
capability dac_read_search,
ptrace (read) peer=**,
signal (receive) peer=unconfined,
signal (receive) peer=cri-containerd.apparmor.d,
signal (receive) peer=runc,
/bin/process-exporter mr,
/process-exporter mr,
/config/** r,
/etc/{passwd,group,nsswitch.conf} r,
/{host/,}proc/ r,
/{host/,}proc/{stat,meminfo,cpuinfo,uptime,version,sys/kernel/random/boot_id} r,
/{host/,}proc/[0-9]*/ r,
/{host/,}proc/[0-9]*/{stat,status,cmdline,smaps,smaps_rollup,io,limits,cgroup,wchan} r,
/{host/,}proc/[0-9]*/fd/ r,
/{host/,}proc/[0-9]*/task/ r,
/{host/,}proc/[0-9]*/task/[0-9]*/{stat,status,io,cmdline,wchan,cgroup,limits,smaps,smaps_rollup} r,
/proc/sys/net/core/somaxconn r,
/proc/self/{stat,status,smaps,smaps_rollup,limits,cgroup,mountinfo} r,
}
@@ -0,0 +1,124 @@
apiVersion: v1
kind: ConfigMap
metadata:
name: process-exporter-config
namespace: monitoring
data:
config.yml: |
process_names:
# 只暴露虚拟机名,不把完整命令行、PID 或凭据写入标签。
- name: 'vm:{{.Matches.VM}}'
comm: [qemu-system-x86]
cmdline: ['-name\s+guest=(?P<VM>[^,\s]+)']
- name: netbox
cmdline: ['/opt/netbox/']
- name: vscode
cmdline: ['\.vscode-server/']
- name: '{{.Comm}}'
cmdline: ['.+']
---
apiVersion: apps/v1
kind: DaemonSet
metadata:
name: process-exporter
namespace: monitoring
spec:
selector:
matchLabels:
app.kubernetes.io/name: process-exporter
template:
metadata:
labels:
app.kubernetes.io/name: process-exporter
spec:
nodeSelector:
kubernetes.io/hostname: laptop
hostPID: true
automountServiceAccountToken: false
tolerations:
- operator: Exists
containers:
- name: process-exporter
image: ncabatoff/process-exporter:0.8.7
args:
- -procfs=/host/proc
- -config.path=/config/config.yml
- -gather-smaps=true
- -threads=false
- -children=false
securityContext:
runAsUser: 0
readOnlyRootFilesystem: true
allowPrivilegeEscalation: false
appArmorProfile:
type: Localhost
localhostProfile: homelab-process-exporter
capabilities:
drop: [ALL]
add: [SYS_PTRACE, DAC_READ_SEARCH]
ports:
- name: metrics
containerPort: 9256
readinessProbe:
tcpSocket:
port: metrics
resources:
requests:
cpu: 25m
memory: 32Mi
limits:
cpu: 500m
memory: 128Mi
volumeMounts:
- name: proc
mountPath: /host/proc
readOnly: true
- name: config
mountPath: /config
readOnly: true
volumes:
- name: proc
hostPath:
path: /proc
type: Directory
- name: config
configMap:
name: process-exporter-config
---
apiVersion: operator.victoriametrics.com/v1beta1
kind: VMPodScrape
metadata:
name: process-exporter
namespace: monitoring
spec:
selector:
matchLabels:
app.kubernetes.io/name: process-exporter
podMetricsEndpoints:
- port: metrics
interval: 60s
scrapeTimeout: 30s
relabelConfigs:
- targetLabel: job
replacement: process-exporter
- sourceLabels: [__meta_kubernetes_pod_node_name]
targetLabel: node
---
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
name: process-exporter
namespace: monitoring
spec:
podSelector:
matchLabels:
app.kubernetes.io/name: process-exporter
policyTypes: [Ingress]
ingress:
- from:
- podSelector:
matchLabels:
app.kubernetes.io/name: vmagent
ports:
- protocol: TCP
port: 9256
@@ -14,3 +14,4 @@ resources:
- scrapes/docker-hosts.yaml
- scrapes/kubelet.yaml
- exporters/node-exporter.yaml
- exporters/process-exporter.yaml