diff --git a/clusters/sandbox/README.md b/clusters/sandbox/README.md index 97db2ee..03f2825 100644 --- a/clusters/sandbox/README.md +++ b/clusters/sandbox/README.md @@ -30,6 +30,12 @@ central Server reviewer。Agent ServiceAccount 留给后续 HelmRelease 创建 文件;该 Secret 准备完成后,才能启用 central external PSAT/controller-manager 和 sandbox Agent/CSI。 +External controller-manager 使用独立的 `spire-controller-manager` ServiceAccount;其 +RBAC 与上游 controller-manager 所需权限一致,用于读取 workload selectors、维护 +SPIFFE CR status/finalizer 和 leader election。它不复用只允许 TokenReview 的 Server +reviewer。Ansible 将两份 kubeconfig 写入同一个 central Secret 的不同 key,便于 central +chart 分别绑定 `sandbox` 与 `sandbox-controller`。 + ## 监控边界 这里只管理 sandbox LXC 内的 Kubernetes 监控,不负责 PVE 宿主监控。LXC 与宿主共享 diff --git a/infrastructure/sandbox-cluster/README.md b/infrastructure/sandbox-cluster/README.md index 6fcfa0c..3e47614 100644 --- a/infrastructure/sandbox-cluster/README.md +++ b/infrastructure/sandbox-cluster/README.md @@ -124,6 +124,10 @@ Playbook 不把 reviewer token 或生成的 kubeconfig 落盘,而是将目标 `spire-server/spire-external-kubeconfigs` 由 Ansible 单独拥有;Flux 和人工操作不得写入。 第二次运行必须为零变更。 +Secret 的 `sandbox` key 仅供 SPIRE Server 的 external PSAT plugin 执行 TokenReview; +`sandbox-controller` key 供 external controller-manager 读取 Pod/Node、reconcile SPIFFE +CR 及执行 leader election。两者使用不同 ServiceAccount,不得合并权限或互换。 + ## 已验证的 Kata CI 前置条件 - Cloud Hypervisor 必须报告 `vm.info.config.memory.shared=true`; diff --git a/infrastructure/sandbox-cluster/ansible/roles/sandbox_spire_bootstrap/defaults/main.yml b/infrastructure/sandbox-cluster/ansible/roles/sandbox_spire_bootstrap/defaults/main.yml index 2456216..9f13c70 100644 --- a/infrastructure/sandbox-cluster/ansible/roles/sandbox_spire_bootstrap/defaults/main.yml +++ b/infrastructure/sandbox-cluster/ansible/roles/sandbox_spire_bootstrap/defaults/main.yml @@ -2,5 +2,6 @@ sandbox_spire_bootstrap_api_server: https://10.60.0.13:6443 sandbox_spire_bootstrap_source_namespace: spire-system sandbox_spire_bootstrap_source_secret: spire-server-token-reviewer-token +sandbox_spire_bootstrap_controller_secret: spire-controller-manager-token sandbox_spire_bootstrap_target_namespace: spire-server sandbox_spire_bootstrap_target_secret: spire-external-kubeconfigs diff --git a/infrastructure/sandbox-cluster/ansible/roles/sandbox_spire_bootstrap/tasks/main.yml b/infrastructure/sandbox-cluster/ansible/roles/sandbox_spire_bootstrap/tasks/main.yml index 0cad051..ce40a31 100644 --- a/infrastructure/sandbox-cluster/ansible/roles/sandbox_spire_bootstrap/tasks/main.yml +++ b/infrastructure/sandbox-cluster/ansible/roles/sandbox_spire_bootstrap/tasks/main.yml @@ -20,10 +20,33 @@ - (sandbox_spire_bootstrap_reviewer_secret.stdout | from_json).data['ca.crt'] is defined no_log: true +- name: Wait for the sandbox SPIRE controller credential + ansible.builtin.command: + argv: + - k3s + - kubectl + - --namespace + - "{{ sandbox_spire_bootstrap_source_namespace }}" + - get + - secret + - "{{ sandbox_spire_bootstrap_controller_secret }}" + - --output=json + register: sandbox_spire_bootstrap_controller_secret_result + changed_when: false + retries: 60 + delay: 10 + until: + - sandbox_spire_bootstrap_controller_secret_result.rc == 0 + - (sandbox_spire_bootstrap_controller_secret_result.stdout | from_json).data.token is defined + - (sandbox_spire_bootstrap_controller_secret_result.stdout | from_json).data['ca.crt'] is defined + no_log: true + - name: Extract the sandbox TokenReview credential data ansible.builtin.set_fact: sandbox_spire_bootstrap_secret_data: >- {{ (sandbox_spire_bootstrap_reviewer_secret.stdout | from_json).data }} + sandbox_spire_bootstrap_controller_data: >- + {{ (sandbox_spire_bootstrap_controller_secret_result.stdout | from_json).data }} no_log: true - name: Build the restricted sandbox TokenReview kubeconfig @@ -46,6 +69,24 @@ cluster: sandbox user: spire-server-token-reviewer current-context: sandbox + sandbox_spire_bootstrap_controller_kubeconfig: | + apiVersion: v1 + kind: Config + clusters: + - name: sandbox + cluster: + server: {{ sandbox_spire_bootstrap_api_server }} + certificate-authority-data: {{ sandbox_spire_bootstrap_controller_data['ca.crt'] }} + users: + - name: spire-controller-manager + user: + token: {{ sandbox_spire_bootstrap_controller_data.token | b64decode }} + contexts: + - name: sandbox + context: + cluster: sandbox + user: spire-controller-manager + current-context: sandbox no_log: true - name: Reconcile the central SPIRE external kubeconfig Secret @@ -64,6 +105,7 @@ type: Opaque data: sandbox: {{ sandbox_spire_bootstrap_kubeconfig | b64encode }} + sandbox-controller: {{ sandbox_spire_bootstrap_controller_kubeconfig | b64encode }} delegate_to: localhost become: true register: sandbox_spire_bootstrap_target diff --git a/platform/sandbox-spire/bootstrap/controller-manager.yaml b/platform/sandbox-spire/bootstrap/controller-manager.yaml new file mode 100644 index 0000000..b473c2f --- /dev/null +++ b/platform/sandbox-spire/bootstrap/controller-manager.yaml @@ -0,0 +1,129 @@ +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: spire-controller-manager + namespace: spire-system +--- +apiVersion: v1 +kind: Secret +metadata: + name: spire-controller-manager-token + namespace: spire-system + annotations: + kubernetes.io/service-account.name: spire-controller-manager +type: kubernetes.io/service-account-token +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: spire-controller-manager +rules: + - apiGroups: + - "" + resources: + - endpoints + - namespaces + - nodes + - pods + verbs: + - get + - list + - watch + - apiGroups: + - spire.spiffe.io + resources: + - clusterfederatedtrustdomains + - clusterspiffeids + - clusterstaticentries + verbs: + - create + - delete + - get + - list + - patch + - update + - watch + - apiGroups: + - spire.spiffe.io + resources: + - clusterfederatedtrustdomains/finalizers + - clusterspiffeids/finalizers + - clusterstaticentries/finalizers + verbs: + - update + - apiGroups: + - spire.spiffe.io + resources: + - clusterfederatedtrustdomains/status + - clusterspiffeids/status + - clusterstaticentries/status + verbs: + - get + - patch + - update +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: spire-controller-manager +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: spire-controller-manager +subjects: + - kind: ServiceAccount + name: spire-controller-manager + namespace: spire-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: spire-controller-manager-leader-election + namespace: spire-server +rules: + - apiGroups: + - "" + resources: + - configmaps + verbs: + - create + - delete + - get + - list + - patch + - update + - watch + - apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - create + - delete + - get + - list + - patch + - update + - watch + - apiGroups: + - "" + resources: + - events + verbs: + - create + - patch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: spire-controller-manager-leader-election + namespace: spire-server +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: spire-controller-manager-leader-election +subjects: + - kind: ServiceAccount + name: spire-controller-manager + namespace: spire-system diff --git a/platform/sandbox-spire/bootstrap/kustomization.yaml b/platform/sandbox-spire/bootstrap/kustomization.yaml index eb4ae1b..44c2ef7 100644 --- a/platform/sandbox-spire/bootstrap/kustomization.yaml +++ b/platform/sandbox-spire/bootstrap/kustomization.yaml @@ -6,3 +6,4 @@ resources: - repository.yaml - crds.yaml - token-reviewer.yaml + - controller-manager.yaml diff --git a/platform/sandbox-spire/bootstrap/namespaces.yaml b/platform/sandbox-spire/bootstrap/namespaces.yaml index bfc0829..7efcab8 100644 --- a/platform/sandbox-spire/bootstrap/namespaces.yaml +++ b/platform/sandbox-spire/bootstrap/namespaces.yaml @@ -8,3 +8,8 @@ apiVersion: v1 kind: Namespace metadata: name: spire-system +--- +apiVersion: v1 +kind: Namespace +metadata: + name: spire-server