diff --git a/clusters/sandbox/README.md b/clusters/sandbox/README.md index c5c4a1f..97db2ee 100644 --- a/clusters/sandbox/README.md +++ b/clusters/sandbox/README.md @@ -21,6 +21,15 @@ Root bootstrap 已完成。后续按依赖顺序分别引入： 第一阶段监控拆为 `monitoring-operator` 与依赖它的 `monitoring`，防止 VM CR 在 VictoriaMetrics Operator CRD Ready 前进入 reconciliation。 +SPIRE 阶段先由 `spire-bootstrap` 安装 CRD，并声明只允许 `tokenreviews.create` 的 +central Server reviewer。Agent ServiceAccount 留给后续 HelmRelease 创建，避免两个 +声明方争夺同一资源。随后运行 +`infrastructure/sandbox-cluster/ansible/spire-bootstrap.yml`：playbook 从 sandbox +读取 reviewer token，在内存中组成受限 kubeconfig，再通过 stdin reconcile 到 central +集群的 `spire-server/spire-external-kubeconfigs` Secret。凭据不写入仓库、日志或控制机 +文件；该 Secret 准备完成后，才能启用 central external PSAT/controller-manager 和 +sandbox Agent/CSI。 + ## 监控边界 这里只管理 sandbox LXC 内的 Kubernetes 监控，不负责 PVE 宿主监控。LXC 与宿主共享 diff --git a/clusters/sandbox/apps/spire-bootstrap.yaml b/clusters/sandbox/apps/spire-bootstrap.yaml new file mode 100644 index 0000000..5d42af2 --- /dev/null +++ b/clusters/sandbox/apps/spire-bootstrap.yaml @@ -0,0 +1,15 @@ +--- +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: spire-bootstrap + namespace: flux-system +spec: + interval: 10m + path: ./platform/sandbox-spire/bootstrap + prune: true + sourceRef: + kind: GitRepository + name: flux-system + timeout: 10m + wait: true diff --git a/clusters/sandbox/kustomization.yaml b/clusters/sandbox/kustomization.yaml index 30a0613..9c4837e 100644 --- a/clusters/sandbox/kustomization.yaml +++ b/clusters/sandbox/kustomization.yaml @@ -4,3 +4,4 @@ kind: Kustomization resources: - apps/monitoring-operator.yaml - apps/monitoring.yaml + - apps/spire-bootstrap.yaml diff --git a/infrastructure/sandbox-cluster/README.md b/infrastructure/sandbox-cluster/README.md index 82dd17b..6fcfa0c 100644 --- a/infrastructure/sandbox-cluster/README.md +++ b/infrastructure/sandbox-cluster/README.md @@ -108,6 +108,22 @@ homelab CA trust。Flux `v2.9.5` controllers 与 root sync 也由 Ansible 通过 server manifests 管理；root 使用 homelab CA 访问公开 Gitea 仓库，不保存 Git token。 集群内 workload 由 `clusters/sandbox/` 分阶段纳入 Flux。 +## SPIRE 跨集群 bootstrap + +Sandbox 复用 homelab 的 SPIRE Server 与 `ddupan.top` trust domain。Flux 首先安装 +SPIRE CRD，并创建权限仅为 `authentication.k8s.io/tokenreviews.create` 的 reviewer。 +在该 Kustomization Ready 后运行： + +```bash +cd infrastructure/sandbox-cluster/ansible +ansible-playbook spire-bootstrap.yml +``` + +Playbook 不把 reviewer token 或生成的 kubeconfig 落盘，而是将目标 Secret manifest +通过 stdin 交给本机 homelab `k3s kubectl`。目标 Secret +`spire-server/spire-external-kubeconfigs` 由 Ansible 单独拥有；Flux 和人工操作不得写入。 +第二次运行必须为零变更。 + ## 已验证的 Kata CI 前置条件 - Cloud Hypervisor 必须报告 `vm.info.config.memory.shared=true`； diff --git a/infrastructure/sandbox-cluster/ansible/roles/sandbox_spire_bootstrap/defaults/main.yml b/infrastructure/sandbox-cluster/ansible/roles/sandbox_spire_bootstrap/defaults/main.yml new file mode 100644 index 0000000..2456216 --- /dev/null +++ b/infrastructure/sandbox-cluster/ansible/roles/sandbox_spire_bootstrap/defaults/main.yml @@ -0,0 +1,6 @@ +--- +sandbox_spire_bootstrap_api_server: https://10.60.0.13:6443 +sandbox_spire_bootstrap_source_namespace: spire-system +sandbox_spire_bootstrap_source_secret: spire-server-token-reviewer-token +sandbox_spire_bootstrap_target_namespace: spire-server +sandbox_spire_bootstrap_target_secret: spire-external-kubeconfigs diff --git a/infrastructure/sandbox-cluster/ansible/roles/sandbox_spire_bootstrap/tasks/main.yml b/infrastructure/sandbox-cluster/ansible/roles/sandbox_spire_bootstrap/tasks/main.yml new file mode 100644 index 0000000..0cad051 --- /dev/null +++ b/infrastructure/sandbox-cluster/ansible/roles/sandbox_spire_bootstrap/tasks/main.yml @@ -0,0 +1,73 @@ +--- +- name: Wait for the sandbox SPIRE token reviewer credential + ansible.builtin.command: + argv: + - k3s + - kubectl + - --namespace + - "{{ sandbox_spire_bootstrap_source_namespace }}" + - get + - secret + - "{{ sandbox_spire_bootstrap_source_secret }}" + - --output=json + register: sandbox_spire_bootstrap_reviewer_secret + changed_when: false + retries: 60 + delay: 10 + until: + - sandbox_spire_bootstrap_reviewer_secret.rc == 0 + - (sandbox_spire_bootstrap_reviewer_secret.stdout | from_json).data.token is defined + - (sandbox_spire_bootstrap_reviewer_secret.stdout | from_json).data['ca.crt'] is defined + no_log: true + +- name: Extract the sandbox TokenReview credential data + ansible.builtin.set_fact: + sandbox_spire_bootstrap_secret_data: >- + {{ (sandbox_spire_bootstrap_reviewer_secret.stdout | from_json).data }} + no_log: true + +- name: Build the restricted sandbox TokenReview kubeconfig + ansible.builtin.set_fact: + sandbox_spire_bootstrap_kubeconfig: | + apiVersion: v1 + kind: Config + clusters: + - name: sandbox + cluster: + server: {{ sandbox_spire_bootstrap_api_server }} + certificate-authority-data: {{ sandbox_spire_bootstrap_secret_data['ca.crt'] }} + users: + - name: spire-server-token-reviewer + user: + token: {{ sandbox_spire_bootstrap_secret_data.token | b64decode }} + contexts: + - name: sandbox + context: + cluster: sandbox + user: spire-server-token-reviewer + current-context: sandbox + no_log: true + +- name: Reconcile the central SPIRE external kubeconfig Secret + ansible.builtin.command: + argv: + - k3s + - kubectl + - apply + - --filename=- + stdin: | + apiVersion: v1 + kind: Secret + metadata: + name: {{ sandbox_spire_bootstrap_target_secret }} + namespace: {{ sandbox_spire_bootstrap_target_namespace }} + type: Opaque + data: + sandbox: {{ sandbox_spire_bootstrap_kubeconfig | b64encode }} + delegate_to: localhost + become: true + register: sandbox_spire_bootstrap_target + changed_when: >- + ' created' in sandbox_spire_bootstrap_target.stdout or + ' configured' in sandbox_spire_bootstrap_target.stdout + no_log: true diff --git a/infrastructure/sandbox-cluster/ansible/site.yml b/infrastructure/sandbox-cluster/ansible/site.yml index 126144e..c0eb724 100644 --- a/infrastructure/sandbox-cluster/ansible/site.yml +++ b/infrastructure/sandbox-cluster/ansible/site.yml @@ -52,3 +52,9 @@ gather_facts: false roles: - sandbox_flux + +- name: Reconcile central SPIRE access to sandbox Kubernetes + hosts: sandbox1 + gather_facts: false + roles: + - sandbox_spire_bootstrap diff --git a/infrastructure/sandbox-cluster/ansible/spire-bootstrap.yml b/infrastructure/sandbox-cluster/ansible/spire-bootstrap.yml new file mode 100644 index 0000000..eeb53bd --- /dev/null +++ b/infrastructure/sandbox-cluster/ansible/spire-bootstrap.yml @@ -0,0 +1,6 @@ +--- +- name: Reconcile central SPIRE access to sandbox Kubernetes + hosts: sandbox1 + gather_facts: false + roles: + - sandbox_spire_bootstrap diff --git a/platform/sandbox-spire/bootstrap/crds.yaml b/platform/sandbox-spire/bootstrap/crds.yaml new file mode 100644 index 0000000..2e1fde1 --- /dev/null +++ b/platform/sandbox-spire/bootstrap/crds.yaml @@ -0,0 +1,31 @@ +--- +apiVersion: helm.toolkit.fluxcd.io/v2 +kind: HelmRelease +metadata: + name: spire-crds + namespace: spire-mgmt +spec: + chart: + spec: + chart: spire-crds + interval: 1h + sourceRef: + kind: HelmRepository + name: spiffe-hardened + version: 0.6.1 + driftDetection: + mode: enabled + install: + crds: CreateReplace + strategy: + name: RetryOnFailure + retryInterval: 5m + interval: 30m + releaseName: spire-crds + targetNamespace: spire-mgmt + timeout: 10m + upgrade: + crds: CreateReplace + strategy: + name: RetryOnFailure + retryInterval: 5m diff --git a/platform/sandbox-spire/bootstrap/kustomization.yaml b/platform/sandbox-spire/bootstrap/kustomization.yaml new file mode 100644 index 0000000..eb4ae1b --- /dev/null +++ b/platform/sandbox-spire/bootstrap/kustomization.yaml @@ -0,0 +1,8 @@ +--- +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - namespaces.yaml + - repository.yaml + - crds.yaml + - token-reviewer.yaml diff --git a/platform/sandbox-spire/bootstrap/namespaces.yaml b/platform/sandbox-spire/bootstrap/namespaces.yaml new file mode 100644 index 0000000..bfc0829 --- /dev/null +++ b/platform/sandbox-spire/bootstrap/namespaces.yaml @@ -0,0 +1,10 @@ +--- +apiVersion: v1 +kind: Namespace +metadata: + name: spire-mgmt +--- +apiVersion: v1 +kind: Namespace +metadata: + name: spire-system diff --git a/platform/sandbox-spire/bootstrap/repository.yaml b/platform/sandbox-spire/bootstrap/repository.yaml new file mode 100644 index 0000000..9ae2457 --- /dev/null +++ b/platform/sandbox-spire/bootstrap/repository.yaml @@ -0,0 +1,9 @@ +--- +apiVersion: source.toolkit.fluxcd.io/v1 +kind: HelmRepository +metadata: + name: spiffe-hardened + namespace: spire-mgmt +spec: + interval: 1h + url: https://spiffe.github.io/helm-charts-hardened/ diff --git a/platform/sandbox-spire/bootstrap/token-reviewer.yaml b/platform/sandbox-spire/bootstrap/token-reviewer.yaml new file mode 100644 index 0000000..e636f42 --- /dev/null +++ b/platform/sandbox-spire/bootstrap/token-reviewer.yaml @@ -0,0 +1,40 @@ +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: spire-server-token-reviewer + namespace: spire-system +--- +apiVersion: v1 +kind: Secret +metadata: + name: spire-server-token-reviewer-token + namespace: spire-system + annotations: + kubernetes.io/service-account.name: spire-server-token-reviewer +type: kubernetes.io/service-account-token +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: spire-server-token-reviewer +rules: + - apiGroups: + - authentication.k8s.io + resources: + - tokenreviews + verbs: + - create +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: spire-server-token-reviewer +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: spire-server-token-reviewer +subjects: + - kind: ServiceAccount + name: spire-server-token-reviewer + namespace: spire-system