diff --git a/clusters/sandbox/README.md b/clusters/sandbox/README.md new file mode 100644 index 0000000..395474c --- /dev/null +++ b/clusters/sandbox/README.md @@ -0,0 +1,29 @@ +# Sandbox 集群 + +这里是 OpenSandbox、CI 和 AI Agent workload 所在双节点 k3s 集群的 Flux +reconciliation 入口。LXC、PostgreSQL、K3s、固定版本的 Flux controllers 与 root +sync 由 `infrastructure/sandbox-cluster/` 中的 Ansible 管理；本目录只组合集群内 +workload。 + +Flux 通过 `https://git.ddupan.top/panxiao81/homelab-infra.git` 读取公开仓库。 +Ansible 将 homelab CA 注入 `GitRepository/flux-system` 引用的同名 Secret，不使用 +长期 Git 凭据。root Kustomization 从 `./clusters/sandbox` 开始 reconciliation， +初始保持 `prune: false`。 + +当前 root 为空，作为 bootstrap canary。后续按依赖顺序分别引入： + +1. 监控 CRD、kube-state-metrics 以及 kubelet/cAdvisor 抓取配置； +2. SPIRE Agent、SPIFFE CSI Driver 与 workload registration； +3. Kata Containers、`block-plain` RuntimeClass； +4. OpenSandbox operator/server 及 `ci-pod`、`ci-vm` Pools。 + +每一阶段单独合并并等待对应 Flux Kustomization Ready，不在 bootstrap 时一次性部署。 + +## 监控边界 + +这里只管理 sandbox LXC 内的 Kubernetes 监控，不负责 PVE 宿主监控。LXC 与宿主共享 +内核，即使 lxcfs 虚拟化了内存和 uptime，容器内 `/proc/stat` 仍是宿主 CPU 视图； +在 LXC 内运行 node_exporter 会生成混合语义并重复采集宿主指标，因此禁止部署。 + +Sandbox 节点与 workload 指标来自 kubelet/cAdvisor 和 kube-state-metrics；K3s 或 LXC +特有但上述接口未覆盖的指标，应使用目标明确的 collector，不以 node_exporter 补齐。 diff --git a/clusters/sandbox/kustomization.yaml b/clusters/sandbox/kustomization.yaml new file mode 100644 index 0000000..fe0f332 --- /dev/null +++ b/clusters/sandbox/kustomization.yaml @@ -0,0 +1,4 @@ +--- +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: [] diff --git a/infrastructure/proxmox/ansible/roles/vyos_router/defaults/main.yml b/infrastructure/proxmox/ansible/roles/vyos_router/defaults/main.yml index 1438b22..2cea4c4 100644 --- a/infrastructure/proxmox/ansible/roles/vyos_router/defaults/main.yml +++ b/infrastructure/proxmox/ansible/roles/vyos_router/defaults/main.yml @@ -71,3 +71,20 @@ vyos_ospf_router_id: "192.168.10.2" vyos_ospf_area: "0" vyos_ospf_networks: - "192.168.10.0/24" + +# Stable K3s datastore endpoint. Only the declared writable primary is present +# in the backend; PostgreSQL health checks must never be treated as promotion. +vyos_postgresql_listen_address: "10.60.0.1" +vyos_postgresql_port: 5432 +vyos_postgresql_primary_address: "10.60.0.11" + +# Stable API endpoint for the two-node sandbox K3s control plane. HAProxy owns +# this /32 on the existing labnet interface and performs only TCP health checks. +vyos_sandbox_k3s_api_address: "10.60.0.13" +vyos_sandbox_k3s_api_interface: eth1 +vyos_sandbox_k3s_api_port: 6443 +vyos_sandbox_k3s_servers: + - name: sandbox1 + address: "10.60.0.11" + - name: sandbox2 + address: "10.60.0.12" diff --git a/infrastructure/proxmox/ansible/roles/vyos_router/tasks/main.yml b/infrastructure/proxmox/ansible/roles/vyos_router/tasks/main.yml index f791847..4fe882d 100644 --- a/infrastructure/proxmox/ansible/roles/vyos_router/tasks/main.yml +++ b/infrastructure/proxmox/ansible/roles/vyos_router/tasks/main.yml @@ -3,6 +3,23 @@ # `set` lines against the running config and issues only the differences, so # re-runs are no-ops. Supports --check and --diff. +- name: Read current VyOS configuration commands + vyos.vyos.vyos_command: + commands: + - show configuration commands + register: _vyos_running_commands + changed_when: false + +- name: Remove obsolete HAProxy PostgreSQL protocol check + vyos.vyos.vyos_config: + lines: + - delete load-balancing haproxy backend sandbox-postgresql-primary health-check + save: true + comment: Remove unsupported PostgreSQL protocol check + when: >- + "set load-balancing haproxy backend sandbox-postgresql-primary health-check 'pgsql'" + in _vyos_running_commands.stdout[0] + - name: Render the desired configuration ansible.builtin.set_fact: _vyos_lines: >- diff --git a/infrastructure/proxmox/ansible/roles/vyos_router/templates/vyos.conf.j2 b/infrastructure/proxmox/ansible/roles/vyos_router/templates/vyos.conf.j2 index 76dab92..2fec2f9 100644 --- a/infrastructure/proxmox/ansible/roles/vyos_router/templates/vyos.conf.j2 +++ b/infrastructure/proxmox/ansible/roles/vyos_router/templates/vyos.conf.j2 @@ -29,6 +29,29 @@ set protocols ospf area {{ vyos_ospf_area }} network {{ i.network }} set protocols ospf interface {{ i.iface }} passive {% endfor %} +{# --- K3s PostgreSQL datastore: fixed endpoint, explicitly selected primary --- #} +set load-balancing haproxy service sandbox-postgresql listen-address {{ vyos_postgresql_listen_address }} +set load-balancing haproxy service sandbox-postgresql port {{ vyos_postgresql_port }} +set load-balancing haproxy service sandbox-postgresql mode 'tcp' +set load-balancing haproxy service sandbox-postgresql backend 'sandbox-postgresql-primary' +set load-balancing haproxy backend sandbox-postgresql-primary mode 'tcp' +set load-balancing haproxy backend sandbox-postgresql-primary server primary address {{ vyos_postgresql_primary_address }} +set load-balancing haproxy backend sandbox-postgresql-primary server primary port {{ vyos_postgresql_port }} +set load-balancing haproxy backend sandbox-postgresql-primary server primary check + +{# --- Sandbox K3s API: stable address with both control-plane nodes --- #} +set interfaces ethernet {{ vyos_sandbox_k3s_api_interface }} address '{{ vyos_sandbox_k3s_api_address }}/32' +set load-balancing haproxy service sandbox-k3s-api listen-address {{ vyos_sandbox_k3s_api_address }} +set load-balancing haproxy service sandbox-k3s-api port {{ vyos_sandbox_k3s_api_port }} +set load-balancing haproxy service sandbox-k3s-api mode 'tcp' +set load-balancing haproxy service sandbox-k3s-api backend 'sandbox-k3s-api' +set load-balancing haproxy backend sandbox-k3s-api mode 'tcp' +{% for server in vyos_sandbox_k3s_servers %} +set load-balancing haproxy backend sandbox-k3s-api server {{ server.name }} address {{ server.address }} +set load-balancing haproxy backend sandbox-k3s-api server {{ server.name }} port {{ vyos_sandbox_k3s_api_port }} +set load-balancing haproxy backend sandbox-k3s-api server {{ server.name }} check +{% endfor %} + {# --- management --- #} set service ssh port 22 diff --git a/infrastructure/sandbox-cluster/README.md b/infrastructure/sandbox-cluster/README.md new file mode 100644 index 0000000..82dd17b --- /dev/null +++ b/infrastructure/sandbox-cluster/README.md @@ -0,0 +1,120 @@ +# Sandbox 集群 + +该目录管理供 OpenSandbox、CI 和 AI Agent workload 使用的独立双节点 k3s 集群。 +基础设施和节点生命周期由 Ansible 管理；Kubernetes API 可用后，集群内组件由 +`clusters/sandbox/` 下的 Flux desired state 管理。 + +## 所有权边界 + +Ansible 管理以下持久状态： + +- pve1/pve2 上的 privileged LXC、磁盘、固定网络与 KVM/vhost/TUN/kmsg 设备； +- LXC OS 基线、内部 CA、外部 PostgreSQL 与 K3s； +- 固定版本的 Flux controllers 和 root sync bootstrap； +- 只读状态验证和数据库切换 runbook。 + +Flux 管理以下 Kubernetes 资源： + +- Kata Containers 和 CI 专用的 `block-plain` RuntimeClass； +- SPIRE Agent、SPIFFE CSI Driver 与 workload identity 声明； +- vmagent、kube-state-metrics、kubelet/cAdvisor scrape 配置和告警； +- OpenSandbox operator/server、`ci-pod` 与 `ci-vm` Pools。 + +同一个对象只能有一个 owner。Ansible 不直接部署上述集群内 workload；Flux 不管理 +LXC、K3s datastore 或 K3s 本身。 + +监控范围仅覆盖 sandbox LXC 内的 Kubernetes 与 workload。不得在 LXC 内部署 +node_exporter：LXC 的 `/proc` 是 lxcfs 虚拟视图与宿主内核视图的混合，尤其 +`/proc/stat` 会重复暴露 PVE 宿主 CPU 数据。PVE 自身的 node_exporter 或其他宿主 +监控不属于本目录；sandbox 使用 kubelet/cAdvisor 和 kube-state-metrics。 + +## 声明拓扑 + +| 对象 | PVE 节点 | VMID | 地址 | 资源 | +|---|---:|---:|---|---| +| `sandbox1` | pve1 | 148 | `10.60.0.11/24` | 4 vCPU / 6 GiB / 2 GiB swap | +| `sandbox2` | pve2 | 149 | `10.60.0.12/24` | 4 vCPU / 4 GiB / 2 GiB swap | +| K3s API VIP | VyOS | — | `10.60.0.13:6443` | HAProxy TCP LB | + +节点与 API VIP 均位于现有 PVE `labnet`（VLAN 100，`10.60.0.0/24`），通过 +VyOS `10.60.0.1` 路由；不为 sandbox 新建 VNet，也不占用 `192.168.10.0/24` +地址。需要从 LAN 访问的服务统一经 VyOS 路由或 LB 暴露。两个 LXC 均使用 +`pve-rg-hdd` 上的 32 GiB rootfs；不为 `/var/lib/kubelet` 单独创建 volume。Kata +`block-plain` 产生的数据随 Pod 生命周期清理，当前规模没有额外磁盘故障域的需求。 +K3s API 的 `10.60.0.13/32` 由 VyOS 现有 labnet interface 持有，HAProxy 以 TCP +健康检查把 `6443` 分发到两个 control-plane 节点；它与 PostgreSQL 主从切换逻辑无关。 +LXC 内通过 `/etc/tmpfiles.d/kmsg.conf` 持久维护 `/dev/kmsg -> /dev/console`；否则 +kubelet 会因 LXC 不提供真实 host `/dev/kmsg` 而反复退出。 +每个 LXC 还以只读 bind mount 使用宿主的 `/lib/modules`。LXC 与 PVE 宿主共享内核， +guest 若看不到对应版本的模块目录，K3s 无法加载 `br_netfilter` 和 `overlay`，Flannel +也不会生成节点的 subnet 配置。 + +## PostgreSQL 写入口与切换 + +K3s 使用外部 PostgreSQL，首期采用 primary + synchronous standby。VyOS 在 labnet +gateway `10.60.0.1:5432` 提供固定 TCP 入口，backend 只包含当前声明的 primary； +不把普通 TCP 或 PostgreSQL 存活检查等同于“节点可写”，也不自动把流量切到 standby。 +VyOS 2025.11 的 PostgreSQL protocol check 会生成缺少必需 `user` 参数的 HAProxy +配置，因此这里只使用基础 TCP check；真正的可写性由 `verify.yml` 通过 SQL 验证。 + +数据库切换必须由 Ansible runbook 受控完成：先隔离旧 primary，再提升 standby，最后 +更新 VyOS backend。首期不部署 PgBouncer、Patroni、repmgr 或额外 DCS，也不宣称两节点 +PostgreSQL 能够自动 HA。以后具备第三个仲裁节点时再重新评估自动 failover。 + +数据库凭据位于 Bao `kv/infra/sandbox-postgresql`，包含 K3s 登录密码和 physical +replication 密码。运行 Ansible 前由本机 SPIFFE identity 获取短期 Bao token，再把 +两个值注入 `SANDBOX_K3S_DB_PASSWORD` 与 `SANDBOX_REPLICATION_PASSWORD`；凭据不写入 +inventory、Git 或 Ansible fact cache。首次创建 standby 只允许覆盖不含任何业务库的 +Ubuntu 默认空集群，之后重复执行不会 reseed。 + +## Ansible + +安装固定依赖： + +```bash +cd infrastructure/sandbox-cluster/ansible +uv venv .venv +uv pip install --python .venv/bin/python -r requirements.txt +source .venv/bin/activate +ansible-galaxy collection install -r requirements.yml +export PVE_API_PYTHON="$PWD/.venv/bin/python" +``` + +部署入口最终为： + +```bash +ansible-playbook site.yml +``` + +只安装或 reconcile K3s（LXC、OS baseline 和 PostgreSQL 已就绪时）： + +```bash +ansible-playbook k3s.yml +``` + +K3s 外部 datastore URI 由运行时 `SANDBOX_K3S_DB_PASSWORD` 生成，密码在 URI 中 +进行 URL 编码，最终仅持久化于节点 root 可读的 `/etc/rancher/k3s/config.yaml` +（mode `0600`）。首节点生成的 K3s join token 仅在同一次 Ansible run 内传给第二节点； +不在 inventory 或 Git 中维护副本。首期关闭内建 Traefik 和 ServiceLB。 + +部署后使用同一组运行时凭据执行只读验收： + +```bash +ansible-playbook verify.yml +``` + +当前已经声明 LXC 生命周期、最小 OS baseline、PostgreSQL 和 K3s，包括系统级 +homelab CA trust。Flux `v2.9.5` controllers 与 root sync 也由 Ansible 通过 K3s +server manifests 管理；root 使用 homelab CA 访问公开 Gitea 仓库，不保存 Git token。 +集群内 workload 由 `clusters/sandbox/` 分阶段纳入 Flux。 + +## 已验证的 Kata CI 前置条件 + +- Cloud Hypervisor 必须报告 `vm.info.config.memory.shared=true`； +- CI RuntimeClass 必须使用 `[runtime] emptydir_mode = "block-plain"`； +- dockerd bootstrap 在 guest 内创建 `/dev/kmsg`：`mknod /dev/kmsg c 1 11`； +- `/var/lib/docker` 必须是 guest block device 上的 ext4，Docker driver 必须为 + `overlay2`，不能静默退化到 `vfs` 或 `fuse-overlayfs`； +- Pod 删除后必须不存在遗留 `disk.img`、VMM 或临时 credential。 + +PoC 的完整数据和陷阱见 `../kata-lxc-lab/README.md`。 diff --git a/infrastructure/sandbox-cluster/ansible/ansible.cfg b/infrastructure/sandbox-cluster/ansible/ansible.cfg new file mode 100644 index 0000000..4a26461 --- /dev/null +++ b/infrastructure/sandbox-cluster/ansible/ansible.cfg @@ -0,0 +1,11 @@ +[defaults] +inventory = inventory/hosts.yml +host_key_checking = True +interpreter_python = auto_silent +retry_files_enabled = False +local_tmp = /tmp/ansible-sandbox-cluster-local +remote_tmp = /tmp +roles_path = roles + +[ssh_connection] +pipelining = True diff --git a/infrastructure/sandbox-cluster/ansible/inventory/group_vars/all.yml b/infrastructure/sandbox-cluster/ansible/inventory/group_vars/all.yml new file mode 100644 index 0000000..0a3e3ad --- /dev/null +++ b/infrastructure/sandbox-cluster/ansible/inventory/group_vars/all.yml @@ -0,0 +1,61 @@ +--- +sandbox_pve_api_host: "{{ lookup('env', 'PROXMOX_HOST') }}" +sandbox_pve_api_user: "{{ lookup('env', 'PROXMOX_USER') }}" +sandbox_pve_api_token_id: "{{ lookup('env', 'PROXMOX_TOKEN_ID') }}" +sandbox_pve_api_token_secret: "{{ lookup('env', 'PROXMOX_TOKEN_SECRET') }}" +sandbox_pve_validate_certs: true +sandbox_pve_api_python_interpreter: "{{ lookup('env', 'PVE_API_PYTHON') | default(ansible_playbook_python, true) }}" + +sandbox_bridge: labnet +sandbox_gateway: 10.60.0.1 +sandbox_nameserver: 192.168.10.5 +sandbox_search_domain: ad.ddupan.top +sandbox_lxc_template: laptop:vztmpl/ubuntu-24.04-standard_24.04-2_amd64.tar.zst +sandbox_rootfs_storage: pve-rg-hdd +sandbox_rootfs_gb: 32 +sandbox_lxc_cores: 4 +sandbox_lxc_swap_mb: 2048 +sandbox_lxc_pubkey: "{{ lookup('file', '~/.ssh/id_ed25519.pub') }}" +sandbox_internal_ca_url: https://bao.ad.ddupan.top:8200/v1/pki/ca/pem +sandbox_base_packages: + - ca-certificates + - curl + - jq + - openssh-server + +sandbox_postgresql_version: 16 +sandbox_postgresql_primary_address: 10.60.0.11 +sandbox_postgresql_standby_address: 10.60.0.12 +sandbox_postgresql_lb_address: 10.60.0.1 +sandbox_postgresql_database: k3s +sandbox_postgresql_user: k3s +sandbox_postgresql_replication_user: k3s_replication +sandbox_postgresql_replication_slot: sandbox2 +sandbox_postgresql_rotate_passwords: false +sandbox_postgresql_k3s_password: "{{ lookup('env', 'SANDBOX_K3S_DB_PASSWORD') }}" +sandbox_postgresql_replication_password: "{{ lookup('env', 'SANDBOX_REPLICATION_PASSWORD') }}" + +# This is rendered only at playbook runtime from the Bao-provided password. K3s +# necessarily persists its datastore credential, but config.yaml is root-only. +sandbox_k3s_datastore_password_encoded: >- + {{ sandbox_postgresql_k3s_password | urlencode | replace('/', '%2F') }} +sandbox_k3s_datastore_endpoint: >- + {{ + [ + 'postgres://', sandbox_postgresql_user, ':', sandbox_k3s_datastore_password_encoded, + '@', sandbox_postgresql_lb_address, ':5432/', sandbox_postgresql_database, '?sslmode=disable' + ] | join + }} +server_config_yaml: | + datastore-endpoint: {{ sandbox_k3s_datastore_endpoint }} + disable: + - servicelb + - traefik +user_kubectl: false + +sandbox_flux_version: v2.9.5 +sandbox_flux_install_url: >- + https://github.com/fluxcd/flux2/releases/download/{{ sandbox_flux_version }}/install.yaml +sandbox_flux_repository_url: https://git.ddupan.top/panxiao81/homelab-infra.git +sandbox_flux_repository_branch: main +sandbox_flux_path: ./clusters/sandbox diff --git a/infrastructure/sandbox-cluster/ansible/inventory/hosts.yml b/infrastructure/sandbox-cluster/ansible/inventory/hosts.yml new file mode 100644 index 0000000..5610bd6 --- /dev/null +++ b/infrastructure/sandbox-cluster/ansible/inventory/hosts.yml @@ -0,0 +1,48 @@ +--- +all: + children: + sandbox_pve: + hosts: + pve1: + ansible_host: 192.168.10.4 + sandbox_lxc: + vmid: 148 + hostname: sandbox1 + address: 10.60.0.11/24 + memory_mb: 6144 + pve2: + ansible_host: 192.168.10.7 + sandbox_lxc: + vmid: 149 + hostname: sandbox2 + address: 10.60.0.12/24 + memory_mb: 4096 + + k3s_cluster: + children: + server: + hosts: + sandbox1: + ansible_host: 10.60.0.11 + sandbox2: + ansible_host: 10.60.0.12 + agent: + hosts: {} + vars: + ansible_user: root + k3s_version: v1.36.4+k3s1 + api_endpoint: 10.60.0.13 + k3s_server_config_mode: "0600" + use_external_database: true + + postgres_primary: + hosts: + sandbox1: + + postgres_standby: + hosts: + sandbox2: + + vars: + ansible_user: root + ansible_python_interpreter: /usr/bin/python3 diff --git a/infrastructure/sandbox-cluster/ansible/k3s.yml b/infrastructure/sandbox-cluster/ansible/k3s.yml new file mode 100644 index 0000000..420a03a --- /dev/null +++ b/infrastructure/sandbox-cluster/ansible/k3s.yml @@ -0,0 +1,17 @@ +--- +- name: Validate runtime K3s datastore credential + hosts: k3s_cluster + gather_facts: false + tasks: + - name: Require the Bao-provided PostgreSQL password + ansible.builtin.assert: + that: + - sandbox_postgresql_k3s_password | length > 0 + - server_config_yaml is defined + - (server_config_yaml | from_yaml)['datastore-endpoint'] == sandbox_k3s_datastore_endpoint + fail_msg: SANDBOX_K3S_DB_PASSWORD must be populated from Bao + quiet: true + no_log: true + +- name: Install the sandbox K3s control plane + ansible.builtin.import_playbook: k3s.orchestration.site diff --git a/infrastructure/sandbox-cluster/ansible/requirements.txt b/infrastructure/sandbox-cluster/ansible/requirements.txt new file mode 100644 index 0000000..8d39016 --- /dev/null +++ b/infrastructure/sandbox-cluster/ansible/requirements.txt @@ -0,0 +1,4 @@ +ansible-core==2.21.2 +proxmoxer==2.3.0 +requests==2.34.2 +netaddr==1.3.0 diff --git a/infrastructure/sandbox-cluster/ansible/requirements.yml b/infrastructure/sandbox-cluster/ansible/requirements.yml new file mode 100644 index 0000000..0191b99 --- /dev/null +++ b/infrastructure/sandbox-cluster/ansible/requirements.yml @@ -0,0 +1,15 @@ +--- +collections: + - name: community.proxmox + version: 2.0.0 + - name: community.postgresql + version: 4.2.0 + - name: ansible.posix + version: 2.2.2 + - name: community.general + version: 13.4.0 + - name: community.library_inventory_filtering_v1 + version: 1.1.5 + - name: https://github.com/k3s-io/k3s-ansible.git + type: git + version: 1a600b60d37e0f8a6e2e79b0e474147b5b108ae5 diff --git a/infrastructure/sandbox-cluster/ansible/roles/sandbox_base/handlers/main.yml b/infrastructure/sandbox-cluster/ansible/roles/sandbox_base/handlers/main.yml new file mode 100644 index 0000000..9f18dcf --- /dev/null +++ b/infrastructure/sandbox-cluster/ansible/roles/sandbox_base/handlers/main.yml @@ -0,0 +1,5 @@ +--- +- name: Update sandbox CA certificates + ansible.builtin.command: + cmd: update-ca-certificates + changed_when: true diff --git a/infrastructure/sandbox-cluster/ansible/roles/sandbox_base/tasks/main.yml b/infrastructure/sandbox-cluster/ansible/roles/sandbox_base/tasks/main.yml new file mode 100644 index 0000000..3f7db31 --- /dev/null +++ b/infrastructure/sandbox-cluster/ansible/roles/sandbox_base/tasks/main.yml @@ -0,0 +1,58 @@ +--- +- name: Install sandbox base packages + ansible.builtin.apt: + name: "{{ sandbox_base_packages }}" + state: present + update_cache: true + cache_valid_time: 3600 + +- name: Fetch the homelab internal CA + ansible.builtin.uri: + url: "{{ sandbox_internal_ca_url }}" + return_content: true + validate_certs: true + register: sandbox_internal_ca + changed_when: false + retries: 3 + delay: 5 + until: sandbox_internal_ca is succeeded + +- name: Validate the homelab internal CA response + ansible.builtin.assert: + that: + - "'BEGIN CERTIFICATE' in sandbox_internal_ca.content" + fail_msg: OpenBao did not return a PEM certificate + quiet: true + +- name: Install the homelab internal CA + ansible.builtin.copy: + dest: /usr/local/share/ca-certificates/ddupan-internal-ca.crt + content: "{{ sandbox_internal_ca.content }}" + owner: root + group: root + mode: "0644" + notify: Update sandbox CA certificates + +- name: Enable the SSH service + ansible.builtin.service: + name: ssh + enabled: true + state: started + +# LXC does not expose the host kernel log device. Kubelet only needs a writable +# kmsg-compatible character device, and the container console is the established +# LXC mapping for this purpose. tmpfiles recreates the link after every boot. +- name: Persist the LXC kubelet kmsg mapping + ansible.builtin.copy: + dest: /etc/tmpfiles.d/kmsg.conf + content: "L+ /dev/kmsg - - - - /dev/console\n" + owner: root + group: root + mode: "0644" + +- name: Expose the LXC console as kmsg + ansible.builtin.file: + src: /dev/console + dest: /dev/kmsg + state: link + force: true diff --git a/infrastructure/sandbox-cluster/ansible/roles/sandbox_flux/tasks/main.yml b/infrastructure/sandbox-cluster/ansible/roles/sandbox_flux/tasks/main.yml new file mode 100644 index 0000000..b9251f6 --- /dev/null +++ b/infrastructure/sandbox-cluster/ansible/roles/sandbox_flux/tasks/main.yml @@ -0,0 +1,30 @@ +--- +- name: Wait for the K3s server manifest directory + ansible.builtin.wait_for: + path: /var/lib/rancher/k3s/server/manifests + state: present + timeout: 120 + +- name: Download the pinned Flux controller manifest + ansible.builtin.get_url: + url: "{{ sandbox_flux_install_url }}" + dest: /var/lib/rancher/k3s/server/manifests/flux-install.yaml + owner: root + group: root + mode: "0644" + timeout: 120 + +- name: Read the installed homelab CA + ansible.builtin.slurp: + src: /usr/local/share/ca-certificates/ddupan-internal-ca.crt + register: sandbox_flux_ca + no_log: true + +- name: Install the Flux source and root sync manifest + ansible.builtin.template: + src: flux-sync.yaml.j2 + dest: /var/lib/rancher/k3s/server/manifests/flux-sync.yaml + owner: root + group: root + mode: "0600" + no_log: true diff --git a/infrastructure/sandbox-cluster/ansible/roles/sandbox_flux/templates/flux-sync.yaml.j2 b/infrastructure/sandbox-cluster/ansible/roles/sandbox_flux/templates/flux-sync.yaml.j2 new file mode 100644 index 0000000..3fd79c7 --- /dev/null +++ b/infrastructure/sandbox-cluster/ansible/roles/sandbox_flux/templates/flux-sync.yaml.j2 @@ -0,0 +1,39 @@ +--- +apiVersion: v1 +kind: Secret +metadata: + name: flux-system + namespace: flux-system +type: Opaque +stringData: + caFile: | +{{ sandbox_flux_ca.content | b64decode | indent(4, true) }} +--- +apiVersion: source.toolkit.fluxcd.io/v1 +kind: GitRepository +metadata: + name: flux-system + namespace: flux-system +spec: + interval: 1m + ref: + branch: {{ sandbox_flux_repository_branch }} + secretRef: + name: flux-system + timeout: 60s + url: {{ sandbox_flux_repository_url }} +--- +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: flux-system + namespace: flux-system +spec: + interval: 10m + path: {{ sandbox_flux_path }} + prune: false + sourceRef: + kind: GitRepository + name: flux-system + timeout: 3m + wait: true diff --git a/infrastructure/sandbox-cluster/ansible/roles/sandbox_lxc/tasks/main.yml b/infrastructure/sandbox-cluster/ansible/roles/sandbox_lxc/tasks/main.yml new file mode 100644 index 0000000..491e2a3 --- /dev/null +++ b/infrastructure/sandbox-cluster/ansible/roles/sandbox_lxc/tasks/main.yml @@ -0,0 +1,214 @@ +--- +- name: Validate sandbox declaration and API inputs + ansible.builtin.assert: + that: + - sandbox_lxc.vmid | int > 100 + - sandbox_lxc.hostname | length > 0 + - sandbox_lxc.address is match('^10\.60\.0\.[0-9]{1,3}/24$') + - sandbox_lxc.memory_mb | int >= 4096 + - sandbox_pve_api_host | length > 0 + - sandbox_pve_api_user | length > 0 + - sandbox_pve_api_token_id | length > 0 + - sandbox_pve_api_token_secret | length > 0 + fail_msg: Invalid sandbox declaration or missing Proxmox API environment + no_log: true + +- name: Check whether the LXC already exists + ansible.builtin.stat: + path: /etc/pve/lxc/{{ sandbox_lxc.vmid }}.conf + register: sandbox_lxc_config + +- name: Read current sandbox LXC runtime state + community.proxmox.proxmox_vm_info: + api_host: "{{ sandbox_pve_api_host }}" + api_user: "{{ sandbox_pve_api_user }}" + api_token_id: "{{ sandbox_pve_api_token_id }}" + api_token_secret: "{{ sandbox_pve_api_token_secret }}" + validate_certs: "{{ sandbox_pve_validate_certs }}" + ca_path: /etc/ssl/certs/ca-certificates.crt + api_timeout: 120 + vmid: "{{ sandbox_lxc.vmid }}" + register: sandbox_lxc_info + delegate_to: localhost + no_log: true + vars: + ansible_python_interpreter: "{{ sandbox_pve_api_python_interpreter }}" + +- name: Read current sandbox LXC configuration + ansible.builtin.uri: + url: >- + https://{{ sandbox_pve_api_host }}:8006/api2/json/nodes/{{ inventory_hostname }}/lxc/{{ sandbox_lxc.vmid }}/config + headers: + Authorization: >- + PVEAPIToken={{ sandbox_pve_api_user }}!{{ sandbox_pve_api_token_id }}={{ sandbox_pve_api_token_secret }} + ca_path: /etc/ssl/certs/ca-certificates.crt + return_content: true + status_code: [200, 404, 500] + register: sandbox_lxc_current_config + delegate_to: localhost + no_log: true + +- name: Reconcile sandbox LXC through the PVE API + community.proxmox.proxmox: + api_host: "{{ sandbox_pve_api_host }}" + api_user: "{{ sandbox_pve_api_user }}" + api_token_id: "{{ sandbox_pve_api_token_id }}" + api_token_secret: "{{ sandbox_pve_api_token_secret }}" + validate_certs: "{{ sandbox_pve_validate_certs }}" + ca_path: /etc/ssl/certs/ca-certificates.crt + api_timeout: 120 + node: "{{ inventory_hostname }}" + vmid: "{{ sandbox_lxc.vmid }}" + hostname: "{{ sandbox_lxc.hostname }}" + ostemplate: "{{ sandbox_lxc_template }}" + ostype: ubuntu + cmode: shell + disk_volume: + storage: "{{ sandbox_rootfs_storage }}" + size: "{{ sandbox_rootfs_gb if not sandbox_lxc_config.stat.exists else omit }}" + cores: "{{ sandbox_lxc_cores }}" + memory: "{{ sandbox_lxc.memory_mb }}" + swap: "{{ sandbox_lxc_swap_mb }}" + netif: + net0: >- + {{ + 'name=eth0,bridge=' ~ sandbox_bridge ~ + ',ip=' ~ sandbox_lxc.address ~ + ',gw=' ~ sandbox_gateway ~ + ',type=veth' ~ + (',hwaddr=' ~ _sandbox_existing_mac if _sandbox_existing_mac | length > 0 else '') + }} + nameserver: "{{ sandbox_nameserver }}" + searchdomain: "{{ sandbox_search_domain }}" + unprivileged: false + onboot: true + startup: ["order=30", "up=30", "down=60"] + tags: [ansible, sandbox, k3s, kata] + pubkey: "{{ sandbox_lxc_pubkey }}" + update: true + state: present + timeout: 120 + register: sandbox_lxc_api + when: not sandbox_lxc_config.stat.exists or _sandbox_api_drift + delegate_to: localhost + no_log: true + vars: + ansible_python_interpreter: "{{ sandbox_pve_api_python_interpreter }}" + _sandbox_existing_mac: >- + {{ sandbox_lxc_current_config.json.data.net0.split('hwaddr=')[1].split(',')[0] + if 'hwaddr=' in (sandbox_lxc_current_config.json.data.net0 | default('')) + else '' }} + _sandbox_api_drift: >- + {{ + sandbox_lxc_current_config.status != 200 or + sandbox_lxc_current_config.json.data.hostname | default('') != sandbox_lxc.hostname or + sandbox_lxc_current_config.json.data.cores | default(0) | int != sandbox_lxc_cores | int or + sandbox_lxc_current_config.json.data.memory | default(0) | int != sandbox_lxc.memory_mb | int or + sandbox_lxc_current_config.json.data.swap | default(0) | int != sandbox_lxc_swap_mb | int or + sandbox_lxc_current_config.json.data.nameserver | default('') != sandbox_nameserver or + sandbox_lxc_current_config.json.data.searchdomain | default('') != sandbox_search_domain or + sandbox_lxc_current_config.json.data.cmode | default('') != 'shell' or + sandbox_lxc_current_config.json.data.onboot | default(0) | int != 1 or + sandbox_lxc_current_config.json.data.tags | default('') != 'ansible;k3s;kata;sandbox' or + ('bridge=' ~ sandbox_bridge) not in (sandbox_lxc_current_config.json.data.net0 | default('')) or + ('ip=' ~ sandbox_lxc.address) not in (sandbox_lxc_current_config.json.data.net0 | default('')) or + ('gw=' ~ sandbox_gateway) not in (sandbox_lxc_current_config.json.data.net0 | default('')) or + (sandbox_rootfs_storage ~ ':') not in (sandbox_lxc_current_config.json.data.rootfs | default('')) + }} + +- name: Grow existing sandbox rootfs through the PVE resize API + ansible.builtin.uri: + url: >- + https://{{ sandbox_pve_api_host }}:8006/api2/json/nodes/{{ inventory_hostname }}/lxc/{{ sandbox_lxc.vmid }}/resize + method: PUT + headers: + Authorization: >- + PVEAPIToken={{ sandbox_pve_api_user }}!{{ sandbox_pve_api_token_id }}={{ sandbox_pve_api_token_secret }} + body_format: form-urlencoded + body: + disk: rootfs + size: "{{ sandbox_rootfs_gb }}G" + ca_path: /etc/ssl/certs/ca-certificates.crt + status_code: 200 + when: + - sandbox_lxc_config.stat.exists + - _sandbox_current_rootfs_gb | int < sandbox_rootfs_gb | int + delegate_to: localhost + no_log: true + vars: + _sandbox_current_rootfs_gb: >- + {{ + sandbox_lxc_current_config.json.data.rootfs | + default('') | + regex_findall('size=([0-9]+)G') | + first | + default('0') + }} + +# These properties are not exposed by community.proxmox. Every retained line +# is required by the already validated nested Kata/kind runtime. +- name: Reconcile Kata-specific native LXC properties + ansible.builtin.lineinfile: + path: /etc/pve/lxc/{{ sandbox_lxc.vmid }}.conf + regexp: "^{{ item.key | regex_escape }}:" + line: "{{ item.key }}: {{ item.value }}" + loop: + - { key: features, value: "nesting=1,keyctl=1,fuse=1,mknod=1,force_rw_sys=1" } + - { key: dev0, value: "path=/dev/kvm,mode=0660" } + - { key: dev1, value: "path=/dev/vhost-net,mode=0660" } + - { key: dev2, value: "path=/dev/vhost-vsock,mode=0660" } + - { key: dev3, value: "path=/dev/net/tun,mode=0666" } + - { key: lxc.apparmor.profile, value: unconfined } + - { key: lxc.cgroup2.devices.allow, value: a } + - { key: lxc.cap.drop, value: "" } + - { key: lxc.mount.auto, value: "proc:rw sys:rw" } + - { key: lxc.mount.entry, value: "/lib/modules usr/lib/modules none bind,ro,create=dir 0 0" } + loop_control: + label: "{{ item.key }}" + register: sandbox_lxc_native + +- name: Start sandbox LXC + community.proxmox.proxmox: + api_host: "{{ sandbox_pve_api_host }}" + api_user: "{{ sandbox_pve_api_user }}" + api_token_id: "{{ sandbox_pve_api_token_id }}" + api_token_secret: "{{ sandbox_pve_api_token_secret }}" + validate_certs: "{{ sandbox_pve_validate_certs }}" + ca_path: /etc/ssl/certs/ca-certificates.crt + api_timeout: 120 + vmid: "{{ sandbox_lxc.vmid }}" + state: started + timeout: 120 + when: >- + not _sandbox_was_running or + not (sandbox_lxc_api is changed or sandbox_lxc_native is changed) + delegate_to: localhost + no_log: true + vars: + ansible_python_interpreter: "{{ sandbox_pve_api_python_interpreter }}" + _sandbox_was_running: >- + {{ (sandbox_lxc_info.proxmox_vms | default([]) | first | default({})).status | + default('absent') == 'running' }} + +- name: Restart existing sandbox LXC after configuration changes + community.proxmox.proxmox: + api_host: "{{ sandbox_pve_api_host }}" + api_user: "{{ sandbox_pve_api_user }}" + api_token_id: "{{ sandbox_pve_api_token_id }}" + api_token_secret: "{{ sandbox_pve_api_token_secret }}" + validate_certs: "{{ sandbox_pve_validate_certs }}" + ca_path: /etc/ssl/certs/ca-certificates.crt + api_timeout: 120 + vmid: "{{ sandbox_lxc.vmid }}" + state: restarted + timeout: 120 + when: + - _sandbox_was_running + - sandbox_lxc_api is changed or sandbox_lxc_native is changed + delegate_to: localhost + no_log: true + vars: + ansible_python_interpreter: "{{ sandbox_pve_api_python_interpreter }}" + _sandbox_was_running: >- + {{ (sandbox_lxc_info.proxmox_vms | default([]) | first | default({})).status | + default('absent') == 'running' }} diff --git a/infrastructure/sandbox-cluster/ansible/roles/sandbox_postgresql/handlers/main.yml b/infrastructure/sandbox-cluster/ansible/roles/sandbox_postgresql/handlers/main.yml new file mode 100644 index 0000000..787e8f0 --- /dev/null +++ b/infrastructure/sandbox-cluster/ansible/roles/sandbox_postgresql/handlers/main.yml @@ -0,0 +1,6 @@ +--- +- name: Restart sandbox PostgreSQL + ansible.builtin.service: + name: postgresql + enabled: true + state: restarted diff --git a/infrastructure/sandbox-cluster/ansible/roles/sandbox_postgresql/tasks/main.yml b/infrastructure/sandbox-cluster/ansible/roles/sandbox_postgresql/tasks/main.yml new file mode 100644 index 0000000..da1dd56 --- /dev/null +++ b/infrastructure/sandbox-cluster/ansible/roles/sandbox_postgresql/tasks/main.yml @@ -0,0 +1,50 @@ +--- +- name: Validate sandbox PostgreSQL inputs + ansible.builtin.assert: + that: + - sandbox_postgresql_k3s_password | length >= 32 + - sandbox_postgresql_replication_password | length >= 32 + - inventory_hostname in groups['postgres_primary'] or inventory_hostname in groups['postgres_standby'] + fail_msg: Missing Bao-provided PostgreSQL credentials or invalid inventory role + no_log: true + +- name: Install PostgreSQL packages + ansible.builtin.apt: + name: + - postgresql-{{ sandbox_postgresql_version }} + - postgresql-client-{{ sandbox_postgresql_version }} + - python3-psycopg2 + state: present + update_cache: true + cache_valid_time: 3600 + +- name: Check whether synchronous replication has completed bootstrap + ansible.builtin.stat: + path: /etc/postgresql/{{ sandbox_postgresql_version }}/main/sandbox-synchronous-ready + register: sandbox_postgresql_synchronous_marker + +- name: Install sandbox PostgreSQL access policy + ansible.builtin.template: + src: pg_hba.conf.j2 + dest: /etc/postgresql/{{ sandbox_postgresql_version }}/main/pg_hba.conf + owner: postgres + group: postgres + mode: "0640" + notify: Restart sandbox PostgreSQL + +- name: Install sandbox PostgreSQL server settings + ansible.builtin.template: + src: sandbox.conf.j2 + dest: /etc/postgresql/{{ sandbox_postgresql_version }}/main/conf.d/99-sandbox.conf + owner: postgres + group: postgres + mode: "0644" + notify: Restart sandbox PostgreSQL + +- name: Configure sandbox PostgreSQL primary + ansible.builtin.include_tasks: primary.yml + when: inventory_hostname in groups['postgres_primary'] + +- name: Configure sandbox PostgreSQL standby + ansible.builtin.include_tasks: standby.yml + when: inventory_hostname in groups['postgres_standby'] diff --git a/infrastructure/sandbox-cluster/ansible/roles/sandbox_postgresql/tasks/primary.yml b/infrastructure/sandbox-cluster/ansible/roles/sandbox_postgresql/tasks/primary.yml new file mode 100644 index 0000000..4cbf32c --- /dev/null +++ b/infrastructure/sandbox-cluster/ansible/roles/sandbox_postgresql/tasks/primary.yml @@ -0,0 +1,45 @@ +--- +- name: Apply primary PostgreSQL configuration before replication setup + ansible.builtin.meta: flush_handlers + +- name: Create K3s PostgreSQL login role + community.postgresql.postgresql_user: + name: "{{ sandbox_postgresql_user }}" + password: "{{ sandbox_postgresql_k3s_password }}" + no_password_changes: "{{ not sandbox_postgresql_rotate_passwords }}" + role_attr_flags: LOGIN,NOSUPERUSER,NOCREATEDB,NOCREATEROLE,NOREPLICATION + state: present + become: true + become_user: postgres + no_log: true + environment: + PGOPTIONS: -c password_encryption=scram-sha-256 + +- name: Create K3s PostgreSQL database + community.postgresql.postgresql_db: + name: "{{ sandbox_postgresql_database }}" + owner: "{{ sandbox_postgresql_user }}" + state: present + become: true + become_user: postgres + +- name: Create PostgreSQL replication login role + community.postgresql.postgresql_user: + name: "{{ sandbox_postgresql_replication_user }}" + password: "{{ sandbox_postgresql_replication_password }}" + no_password_changes: "{{ not sandbox_postgresql_rotate_passwords }}" + role_attr_flags: LOGIN,REPLICATION,NOSUPERUSER,NOCREATEDB,NOCREATEROLE + state: present + become: true + become_user: postgres + no_log: true + environment: + PGOPTIONS: -c password_encryption=scram-sha-256 + +- name: Create physical replication slot for sandbox2 + community.postgresql.postgresql_slot: + name: "{{ sandbox_postgresql_replication_slot }}" + slot_type: physical + state: present + become: true + become_user: postgres diff --git a/infrastructure/sandbox-cluster/ansible/roles/sandbox_postgresql/tasks/standby.yml b/infrastructure/sandbox-cluster/ansible/roles/sandbox_postgresql/tasks/standby.yml new file mode 100644 index 0000000..1c88efd --- /dev/null +++ b/infrastructure/sandbox-cluster/ansible/roles/sandbox_postgresql/tasks/standby.yml @@ -0,0 +1,114 @@ +--- +- name: Detect whether sandbox2 is already a standby + community.postgresql.postgresql_query: + login_db: postgres + query: SELECT pg_is_in_recovery() AS in_recovery + become: true + become_user: postgres + register: sandbox_postgresql_recovery_state + +- name: Inspect databases before the initial standby seed + community.postgresql.postgresql_query: + login_db: postgres + query: >- + SELECT datname FROM pg_database + WHERE NOT datistemplate AND datname <> 'postgres' + become: true + become_user: postgres + register: sandbox_postgresql_existing_databases + when: not sandbox_postgresql_recovery_state.query_result[0].in_recovery + +- name: Refuse to overwrite a non-empty PostgreSQL node + ansible.builtin.assert: + that: + - sandbox_postgresql_existing_databases.query_result | length == 0 + fail_msg: Refusing to reseed sandbox2 because it contains non-system databases + when: not sandbox_postgresql_recovery_state.query_result[0].in_recovery + +- name: Stop PostgreSQL before the initial standby seed + ansible.builtin.service: + name: postgresql + state: stopped + when: not sandbox_postgresql_recovery_state.query_result[0].in_recovery + +- name: Remove the verified-empty standby data directory + ansible.builtin.file: + path: /var/lib/postgresql/{{ sandbox_postgresql_version }}/main + state: absent + when: not sandbox_postgresql_recovery_state.query_result[0].in_recovery + +- name: Recreate the standby data directory + ansible.builtin.file: + path: /var/lib/postgresql/{{ sandbox_postgresql_version }}/main + state: directory + owner: postgres + group: postgres + mode: "0700" + when: not sandbox_postgresql_recovery_state.query_result[0].in_recovery + +- name: Install standby replication password file + ansible.builtin.copy: + dest: /var/lib/postgresql/.pgpass + content: >- + {{ sandbox_postgresql_primary_address }}:5432:*:{{ sandbox_postgresql_replication_user }}:{{ sandbox_postgresql_replication_password }} + owner: postgres + group: postgres + mode: "0600" + no_log: true + +- name: Seed sandbox2 from the PostgreSQL primary + ansible.builtin.command: + argv: + - /usr/bin/pg_basebackup + - --host={{ sandbox_postgresql_primary_address }} + - --username={{ sandbox_postgresql_replication_user }} + - --pgdata=/var/lib/postgresql/{{ sandbox_postgresql_version }}/main + - --format=plain + - --wal-method=stream + - --write-recovery-conf + - --slot={{ sandbox_postgresql_replication_slot }} + become: true + become_user: postgres + environment: + PGPASSFILE: /var/lib/postgresql/.pgpass + when: not sandbox_postgresql_recovery_state.query_result[0].in_recovery + no_log: true + +- name: Set the standby connection identity + ansible.builtin.lineinfile: + path: /var/lib/postgresql/{{ sandbox_postgresql_version }}/main/postgresql.auto.conf + regexp: ^primary_conninfo = + line: >- + primary_conninfo = 'host={{ sandbox_postgresql_primary_address }} port=5432 + user={{ sandbox_postgresql_replication_user }} application_name=sandbox2 + passfile=/var/lib/postgresql/.pgpass' + owner: postgres + group: postgres + mode: "0600" + no_log: true + +- name: Start PostgreSQL standby + ansible.builtin.service: + name: postgresql + enabled: true + state: started + +- name: Wait for sandbox2 to enter recovery + community.postgresql.postgresql_query: + login_db: postgres + query: SELECT pg_is_in_recovery() AS in_recovery + become: true + become_user: postgres + register: sandbox_postgresql_standby_ready + retries: 12 + delay: 5 + until: sandbox_postgresql_standby_ready.query_result[0].in_recovery + +- name: Mark synchronous replication bootstrap complete on the primary + ansible.builtin.copy: + dest: /etc/postgresql/{{ sandbox_postgresql_version }}/main/sandbox-synchronous-ready + content: "sandbox2\n" + owner: postgres + group: postgres + mode: "0644" + delegate_to: "{{ groups['postgres_primary'][0] }}" diff --git a/infrastructure/sandbox-cluster/ansible/roles/sandbox_postgresql/templates/pg_hba.conf.j2 b/infrastructure/sandbox-cluster/ansible/roles/sandbox_postgresql/templates/pg_hba.conf.j2 new file mode 100644 index 0000000..208657e --- /dev/null +++ b/infrastructure/sandbox-cluster/ansible/roles/sandbox_postgresql/templates/pg_hba.conf.j2 @@ -0,0 +1,5 @@ +# Ansible managed +local all postgres peer +local all all peer +host {{ sandbox_postgresql_database }} {{ sandbox_postgresql_user }} 10.60.0.0/24 scram-sha-256 +host replication {{ sandbox_postgresql_replication_user }} 10.60.0.0/24 scram-sha-256 diff --git a/infrastructure/sandbox-cluster/ansible/roles/sandbox_postgresql/templates/sandbox.conf.j2 b/infrastructure/sandbox-cluster/ansible/roles/sandbox_postgresql/templates/sandbox.conf.j2 new file mode 100644 index 0000000..f1d1277 --- /dev/null +++ b/infrastructure/sandbox-cluster/ansible/roles/sandbox_postgresql/templates/sandbox.conf.j2 @@ -0,0 +1,11 @@ +# Ansible managed +listen_addresses = '{{ ansible_host }},127.0.0.1' +password_encryption = 'scram-sha-256' +wal_level = 'replica' +max_wal_senders = 10 +max_replication_slots = 10 +hot_standby = on +{% if inventory_hostname in groups['postgres_primary'] and sandbox_postgresql_synchronous_marker.stat.exists %} +synchronous_standby_names = 'FIRST 1 (sandbox2)' +synchronous_commit = 'remote_apply' +{% endif %} diff --git a/infrastructure/sandbox-cluster/ansible/site.yml b/infrastructure/sandbox-cluster/ansible/site.yml new file mode 100644 index 0000000..126144e --- /dev/null +++ b/infrastructure/sandbox-cluster/ansible/site.yml @@ -0,0 +1,54 @@ +--- +- name: Reconcile sandbox LXC nodes + hosts: sandbox_pve + gather_facts: false + serial: 1 + roles: + - sandbox_lxc + +- name: Reconcile sandbox node OS baseline + hosts: k3s_cluster + gather_facts: false + roles: + - sandbox_base + +- name: Reconcile sandbox PostgreSQL primary + hosts: postgres_primary + gather_facts: false + roles: + - sandbox_postgresql + +- name: Reconcile sandbox PostgreSQL synchronous standby + hosts: postgres_standby + gather_facts: false + roles: + - sandbox_postgresql + +- name: Finalize synchronous PostgreSQL settings + hosts: postgres_primary + gather_facts: false + roles: + - sandbox_postgresql + +- name: Validate runtime K3s datastore credential + hosts: k3s_cluster + gather_facts: false + tasks: + - name: Require the Bao-provided PostgreSQL password + ansible.builtin.assert: + that: + - sandbox_postgresql_k3s_password | length > 0 + - server_config_yaml is defined + - (server_config_yaml | from_yaml)['datastore-endpoint'] == sandbox_k3s_datastore_endpoint + fail_msg: SANDBOX_K3S_DB_PASSWORD must be populated from Bao + quiet: true + no_log: true + +- name: Install the sandbox K3s control plane + ansible.builtin.import_playbook: k3s.orchestration.site + +- name: Reconcile Flux controllers and root sync + hosts: sandbox1 + gather_facts: false + roles: + - sandbox_flux diff --git a/infrastructure/sandbox-cluster/ansible/verify.yml b/infrastructure/sandbox-cluster/ansible/verify.yml new file mode 100644 index 0000000..b5ea94a --- /dev/null +++ b/infrastructure/sandbox-cluster/ansible/verify.yml @@ -0,0 +1,149 @@ +--- +- name: Verify the PostgreSQL datastore through the VyOS endpoint + hosts: k3s_cluster + gather_facts: false + tasks: + - name: Query the K3s database through the stable endpoint + community.postgresql.postgresql_query: + login_host: "{{ sandbox_postgresql_lb_address }}" + login_db: "{{ sandbox_postgresql_database }}" + login_user: "{{ sandbox_postgresql_user }}" + login_password: "{{ sandbox_postgresql_k3s_password }}" + query: SELECT NOT pg_is_in_recovery() AS writable + register: sandbox_datastore_endpoint + no_log: true + + - name: Assert the stable datastore endpoint is writable + ansible.builtin.assert: + that: + - sandbox_datastore_endpoint.query_result[0].writable + fail_msg: The VyOS datastore endpoint is not connected to a writable PostgreSQL primary + quiet: true + +- name: Verify synchronous PostgreSQL replication + hosts: postgres_primary + gather_facts: false + tasks: + - name: Query replication state on the primary + community.postgresql.postgresql_query: + login_db: postgres + query: >- + SELECT application_name, state, sync_state + FROM pg_stat_replication + WHERE application_name = 'sandbox2' + become: true + become_user: postgres + register: sandbox_replication_state + + - name: Assert sandbox2 is a synchronous streaming standby + ansible.builtin.assert: + that: + - sandbox_replication_state.query_result | length == 1 + - sandbox_replication_state.query_result[0].state == 'streaming' + - sandbox_replication_state.query_result[0].sync_state == 'sync' + fail_msg: sandbox2 is not synchronously streaming from sandbox1 + quiet: true + +- name: Verify PostgreSQL standby recovery state + hosts: postgres_standby + gather_facts: false + tasks: + - name: Query recovery state on the standby + community.postgresql.postgresql_query: + login_db: postgres + query: SELECT pg_is_in_recovery() AS in_recovery + become: true + become_user: postgres + register: sandbox_standby_state + + - name: Assert sandbox2 remains in recovery + ansible.builtin.assert: + that: + - sandbox_standby_state.query_result[0].in_recovery + fail_msg: sandbox2 is not operating as a PostgreSQL standby + quiet: true + +- name: Verify K3s node prerequisites + hosts: k3s_cluster + gather_facts: false + tasks: + - name: Read the kmsg mapping + ansible.builtin.command: + cmd: readlink /dev/kmsg + register: sandbox_kmsg_target + changed_when: false + + - name: Inspect the host kernel module mount + ansible.builtin.shell: + cmd: >- + set -o pipefail && + findmnt -rn -T "/lib/modules/$(uname -r)" -o OPTIONS | + grep -Eq '(^|,)ro(,|$)' + executable: /bin/bash + changed_when: false + + - name: Assert required LXC kernel integration + ansible.builtin.assert: + that: + - sandbox_kmsg_target.stdout == '/dev/console' + fail_msg: LXC is missing its persistent kmsg or read-only host module mapping + quiet: true + + - name: Read K3s service state + ansible.builtin.systemd_service: + name: k3s + register: sandbox_k3s_service + + - name: Assert K3s is active + ansible.builtin.assert: + that: + - sandbox_k3s_service.status.ActiveState == 'active' + - sandbox_k3s_service.status.SubState == 'running' + fail_msg: K3s is not running + quiet: true + +- name: Verify the K3s control plane and API VIP + hosts: sandbox1 + gather_facts: false + tasks: + - name: Wait for all declared nodes + ansible.builtin.command: + cmd: k3s kubectl wait --for=condition=Ready nodes --all --timeout=120s + changed_when: false + + - name: Read control-plane nodes + ansible.builtin.command: + cmd: >- + k3s kubectl get nodes + -l node-role.kubernetes.io/control-plane=true + -o name + register: sandbox_control_plane_nodes + changed_when: false + + - name: Assert both control-plane nodes are registered + ansible.builtin.assert: + that: + - sandbox_control_plane_nodes.stdout_lines | length == 2 + fail_msg: The sandbox cluster does not contain both control-plane nodes + quiet: true + + - name: Wait for all kube-system Pods + ansible.builtin.command: + cmd: >- + k3s kubectl wait --namespace kube-system + --for=condition=Ready pods --all --timeout=120s + changed_when: false + + - name: Verify the authenticated API path through VyOS + ansible.builtin.command: + cmd: >- + k3s kubectl --server=https://10.60.0.13:6443 get --raw=/livez + register: sandbox_api_livez + changed_when: false + + - name: Assert the API VIP is live + ansible.builtin.assert: + that: + - sandbox_api_livez.stdout == 'ok' + fail_msg: The K3s API is not healthy through the VyOS VIP + quiet: true