From aeb8c49d0a44ed0848f4efca1f61a199b9a5c4f2 Mon Sep 17 00:00:00 2001 From: panxiao81 Date: Thu, 17 Sep 2026 17:23:25 +0000 Subject: [PATCH] =?UTF-8?q?feat:=20=E9=83=A8=E7=BD=B2=20sandbox=20SPIRE=20?= =?UTF-8?q?Agent=20=E4=B8=8E=20CSI?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- clusters/sandbox/README.md | 10 +++ clusters/sandbox/apps/spire-agents.yaml | 18 ++++ clusters/sandbox/kustomization.yaml | 1 + infrastructure/dns/records.yml | 1 + platform/sandbox-monitoring/README.md | 3 +- .../workloads/kustomization.yaml | 1 + .../workloads/spire-agent-scrape.yaml | 17 ++++ .../sandbox-spire/agents/kustomization.yaml | 7 ++ platform/sandbox-spire/agents/release.yaml | 35 ++++++++ .../sandbox-spire/agents/smoke-identity.yaml | 28 ++++++ platform/sandbox-spire/agents/values.yaml | 86 +++++++++++++++++++ .../bootstrap/controller-manager.yaml | 33 +++++++ platform/spire/README.md | 5 ++ platform/spire/values.yaml | 36 ++++++++ 14 files changed, 280 insertions(+), 1 deletion(-) create mode 100644 clusters/sandbox/apps/spire-agents.yaml create mode 100644 platform/sandbox-monitoring/workloads/spire-agent-scrape.yaml create mode 100644 platform/sandbox-spire/agents/kustomization.yaml create mode 100644 platform/sandbox-spire/agents/release.yaml create mode 100644 platform/sandbox-spire/agents/smoke-identity.yaml create mode 100644 platform/sandbox-spire/agents/values.yaml diff --git a/clusters/sandbox/README.md b/clusters/sandbox/README.md index 03f2825..1f55762 100644 --- a/clusters/sandbox/README.md +++ b/clusters/sandbox/README.md @@ -36,6 +36,16 @@ SPIFFE CR status/finalizer 和 leader election。它不复用只允许 TokenRevi reviewer。Ansible 将两份 kubeconfig 写入同一个 central Secret 的不同 key，便于 central chart 分别绑定 `sandbox` 与 `sandbox-controller`。 +Central SPIRE Server 通过内网 `spire-server.ad.ddupan.top:8081` 接收 sandbox Agent +attestation。Server 使用 external bundle publisher 持续维护 sandbox +`spire-system/spire-bundle`，Agent 不固定或复制 trust bundle。Sandbox HelmRelease +显式关闭 Server 与 OIDC Provider，只部署 Agent DaemonSet 和 SPIFFE CSI Driver；因此 +不会产生第二个 trust root。 + +`spire-smoke` namespace、ServiceAccount 和 `sandbox-spire-smoke` ClusterSPIFFEID 是 +普通 Pod 与后续 Kata guest 的回归夹具，稳定身份为 +`spiffe://ddupan.top/sandbox/smoke`。测试 Pod 临时创建并在验收后删除，身份声明保留。 + ## 监控边界 这里只管理 sandbox LXC 内的 Kubernetes 监控，不负责 PVE 宿主监控。LXC 与宿主共享 diff --git a/clusters/sandbox/apps/spire-agents.yaml b/clusters/sandbox/apps/spire-agents.yaml new file mode 100644 index 0000000..1ea8f4a --- /dev/null +++ b/clusters/sandbox/apps/spire-agents.yaml @@ -0,0 +1,18 @@ +--- +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: spire-agents + namespace: flux-system +spec: + dependsOn: + - name: spire-bootstrap + - name: monitoring-operator + interval: 10m + path: ./platform/sandbox-spire/agents + prune: true + sourceRef: + kind: GitRepository + name: flux-system + timeout: 15m + wait: true diff --git a/clusters/sandbox/kustomization.yaml b/clusters/sandbox/kustomization.yaml index 9c4837e..5a47e5a 100644 --- a/clusters/sandbox/kustomization.yaml +++ b/clusters/sandbox/kustomization.yaml @@ -5,3 +5,4 @@ resources: - apps/monitoring-operator.yaml - apps/monitoring.yaml - apps/spire-bootstrap.yaml + - apps/spire-agents.yaml diff --git a/infrastructure/dns/records.yml b/infrastructure/dns/records.yml index 6b5607c..f467def 100644 --- a/infrastructure/dns/records.yml +++ b/infrastructure/dns/records.yml @@ -20,6 +20,7 @@ homelab_dns: - { zone: ad.ddupan.top, name: nats, type: A, values: [192.168.10.127] } - { zone: ad.ddupan.top, name: s3, type: A, values: [192.168.10.127] } - { zone: ad.ddupan.top, name: spire-oidc, type: A, values: [192.168.10.127] } + - { zone: ad.ddupan.top, name: spire-server, type: A, values: [192.168.10.127] } - { zone: ad.ddupan.top, name: zot, type: A, values: [192.168.10.127] } - { zone: ad.ddupan.top, name: zot-push, type: A, values: [192.168.10.127] } diff --git a/platform/sandbox-monitoring/README.md b/platform/sandbox-monitoring/README.md index 7dda179..02575ad 100644 --- a/platform/sandbox-monitoring/README.md +++ b/platform/sandbox-monitoring/README.md @@ -4,7 +4,8 @@ - kubelet 与 cAdvisor； - kube-state-metrics； -- vmagent 自身运行状态。 +- vmagent 自身运行状态； +- SPIRE Agent attestation、SVID 与连接状态。 不得在 sandbox LXC 内部署 node_exporter。LXC 的 `/proc/stat` 暴露 PVE 宿主 CPU 视图，会产生重复且语义混合的指标。PVE 宿主监控不属于本目录。 diff --git a/platform/sandbox-monitoring/workloads/kustomization.yaml b/platform/sandbox-monitoring/workloads/kustomization.yaml index 499d777..a34a5fc 100644 --- a/platform/sandbox-monitoring/workloads/kustomization.yaml +++ b/platform/sandbox-monitoring/workloads/kustomization.yaml @@ -6,3 +6,4 @@ resources: - kubelet-scrapes.yaml - kube-state-metrics.yaml - kube-state-metrics-scrape.yaml + - spire-agent-scrape.yaml diff --git a/platform/sandbox-monitoring/workloads/spire-agent-scrape.yaml b/platform/sandbox-monitoring/workloads/spire-agent-scrape.yaml new file mode 100644 index 0000000..16239dc --- /dev/null +++ b/platform/sandbox-monitoring/workloads/spire-agent-scrape.yaml @@ -0,0 +1,17 @@ +--- +apiVersion: operator.victoriametrics.com/v1beta1 +kind: VMPodScrape +metadata: + name: spire-agent + namespace: monitoring +spec: + namespaceSelector: + matchNames: + - spire-system + podMetricsEndpoints: + - interval: 30s + port: prom + selector: + matchLabels: + app.kubernetes.io/instance: sandbox-spire + app.kubernetes.io/name: agent diff --git a/platform/sandbox-spire/agents/kustomization.yaml b/platform/sandbox-spire/agents/kustomization.yaml new file mode 100644 index 0000000..54ec34e --- /dev/null +++ b/platform/sandbox-spire/agents/kustomization.yaml @@ -0,0 +1,7 @@ +--- +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - values.yaml + - release.yaml + - smoke-identity.yaml diff --git a/platform/sandbox-spire/agents/release.yaml b/platform/sandbox-spire/agents/release.yaml new file mode 100644 index 0000000..4d4a1a9 --- /dev/null +++ b/platform/sandbox-spire/agents/release.yaml @@ -0,0 +1,35 @@ +--- +apiVersion: helm.toolkit.fluxcd.io/v2 +kind: HelmRelease +metadata: + name: sandbox-spire + namespace: spire-mgmt +spec: + chart: + spec: + chart: spire + interval: 1h + sourceRef: + kind: HelmRepository + name: spiffe-hardened + version: 0.30.2 + dependsOn: + - name: spire-crds + namespace: spire-mgmt + driftDetection: + mode: enabled + install: + strategy: + name: RetryOnFailure + retryInterval: 5m + interval: 30m + releaseName: sandbox-spire + targetNamespace: spire-mgmt + timeout: 15m + upgrade: + strategy: + name: RetryOnFailure + retryInterval: 5m + valuesFrom: + - kind: ConfigMap + name: sandbox-spire-values diff --git a/platform/sandbox-spire/agents/smoke-identity.yaml b/platform/sandbox-spire/agents/smoke-identity.yaml new file mode 100644 index 0000000..ea8affc --- /dev/null +++ b/platform/sandbox-spire/agents/smoke-identity.yaml @@ -0,0 +1,28 @@ +--- +apiVersion: v1 +kind: Namespace +metadata: + name: spire-smoke +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: spire-smoke + namespace: spire-smoke +--- +apiVersion: spire.spiffe.io/v1alpha1 +kind: ClusterSPIFFEID +metadata: + name: sandbox-spire-smoke +spec: + className: spire-mgmt-spire + namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: spire-smoke + podSelector: + matchLabels: + app.kubernetes.io/name: spire-smoke + spiffeIDTemplate: spiffe://{{ .TrustDomain }}/sandbox/smoke + workloadSelectorTemplates: + - k8s:ns:spire-smoke + - k8s:sa:spire-smoke diff --git a/platform/sandbox-spire/agents/values.yaml b/platform/sandbox-spire/agents/values.yaml new file mode 100644 index 0000000..eb229bf --- /dev/null +++ b/platform/sandbox-spire/agents/values.yaml @@ -0,0 +1,86 @@ +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: sandbox-spire-values + namespace: spire-mgmt +data: + values.yaml: | + global: + k8s: + clusterDomain: cluster.local + spire: + bundleConfigMap: spire-bundle + clusterName: sandbox + trustDomain: ddupan.top + namespaces: + create: false + system: + name: spire-system + server: + name: spire-server + recommendations: + enabled: true + namespaceLayout: true + namespacePSS: true + priorityClassName: true + strictMode: true + securityContexts: true + prometheus: false + + spire-server: + enabled: false + + spire-agent: + enabled: true + serviceAccount: + name: spire-agent + server: + address: spire-server.ad.ddupan.top + port: 8081 + nodeAttestor: + k8sPSAT: + enabled: true + workloadAttestors: + k8s: + enabled: true + unix: + enabled: false + telemetry: + prometheus: + enabled: true + podMonitor: + enabled: false + resources: + requests: + cpu: 25m + memory: 64Mi + limits: + cpu: 250m + memory: 192Mi + + spiffe-csi-driver: + enabled: true + resources: + requests: + cpu: 10m + memory: 32Mi + limits: + cpu: 100m + memory: 96Mi + + spiffe-oidc-discovery-provider: + enabled: false + + upstream: + enabled: false + tornjak-frontend: + enabled: false + spire-identity-exchange: + enabled: false + spike-keeper: + enabled: false + spike-nexus: + enabled: false + spike-pilot: + enabled: false diff --git a/platform/sandbox-spire/bootstrap/controller-manager.yaml b/platform/sandbox-spire/bootstrap/controller-manager.yaml index b473c2f..18dc8f2 100644 --- a/platform/sandbox-spire/bootstrap/controller-manager.yaml +++ b/platform/sandbox-spire/bootstrap/controller-manager.yaml @@ -78,6 +78,39 @@ subjects: --- apiVersion: rbac.authorization.k8s.io/v1 kind: Role +metadata: + name: spire-bundle-publisher + namespace: spire-system +rules: + - apiGroups: + - "" + resources: + - configmaps + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: spire-bundle-publisher + namespace: spire-system +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: spire-bundle-publisher +subjects: + - kind: ServiceAccount + name: spire-controller-manager + namespace: spire-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role metadata: name: spire-controller-manager-leader-election namespace: spire-server diff --git a/platform/spire/README.md b/platform/spire/README.md index 021ae5d..2bf4486 100644 --- a/platform/spire/README.md +++ b/platform/spire/README.md @@ -19,6 +19,11 @@ Flux 安装 SPIFFE hardened charts： API 或 Broker API。Trust domain 是 `ddupan.top`，Kubernetes cluster name 是 `homelab`。 +同一 Server 也接受 cluster name 为 `sandbox` 的 external PSAT attestation。SPIRE gRPC +只通过内网 `spire-server.ad.ddupan.top:8081` 暴露；external PSAT、external +controller-manager 与 bundle publisher 使用由 sandbox Ansible bootstrap 的独立、受限 +kubeconfig。Sandbox 不运行第二套 Server 或 OIDC Provider。 + ## PostgreSQL bootstrap SPIRE registration datastore 使用共享 CloudNativePG： diff --git a/platform/spire/values.yaml b/platform/spire/values.yaml index 550a878..c13ddfe 100644 --- a/platform/spire/values.yaml +++ b/platform/spire/values.yaml @@ -30,6 +30,42 @@ spire-server: kind: statefulset replicaCount: 1 auditLogEnabled: true + service: + type: LoadBalancer + port: 8081 + loadBalancerIP: 192.168.10.127 + kubeConfigs: + sandbox: + externalSecret: + name: spire-external-kubeconfigs + key: sandbox + sandbox-controller: + externalSecret: + name: spire-external-kubeconfigs + key: sandbox-controller + nodeAttestor: + externalK8sPSAT: + enabled: true + clusters: + sandbox: + kubeConfigName: sandbox + serviceAccountAllowList: + - spire-system:spire-agent + externalControllerManagers: + enabled: true + clusters: + sandbox: + kubeConfigName: sandbox-controller + bundlePublisher: + externalK8sConfigMap: + enabled: true + clusters: + sandbox: + kubeConfigName: sandbox-controller + namespace: spire-system + configMapName: spire-bundle + configMapKey: bundle.spiffe + format: spiffe persistence: # PostgreSQL stores registrations, but the disk KeyManager still needs durable # storage for the trust-domain signing keys.