feat: 声明 sandbox 双节点 K3s 集群

infrastructure/sandbox-cluster/ansible/ansible.cfg

@@ -0,0 +1,11 @@
+[defaults]
+inventory = inventory/hosts.yml
+host_key_checking = True
+interpreter_python = auto_silent
+retry_files_enabled = False
+local_tmp = /tmp/ansible-sandbox-cluster-local
+remote_tmp = /tmp
+roles_path = roles
+
+[ssh_connection]
+pipelining = True

infrastructure/sandbox-cluster/ansible/inventory/group_vars/all.yml

@@ -0,0 +1,61 @@
+---
+sandbox_pve_api_host: "{{ lookup('env', 'PROXMOX_HOST') }}"
+sandbox_pve_api_user: "{{ lookup('env', 'PROXMOX_USER') }}"
+sandbox_pve_api_token_id: "{{ lookup('env', 'PROXMOX_TOKEN_ID') }}"
+sandbox_pve_api_token_secret: "{{ lookup('env', 'PROXMOX_TOKEN_SECRET') }}"
+sandbox_pve_validate_certs: true
+sandbox_pve_api_python_interpreter: "{{ lookup('env', 'PVE_API_PYTHON') | default(ansible_playbook_python, true) }}"
+
+sandbox_bridge: labnet
+sandbox_gateway: 10.60.0.1
+sandbox_nameserver: 192.168.10.5
+sandbox_search_domain: ad.ddupan.top
+sandbox_lxc_template: laptop:vztmpl/ubuntu-24.04-standard_24.04-2_amd64.tar.zst
+sandbox_rootfs_storage: pve-rg-hdd
+sandbox_rootfs_gb: 32
+sandbox_lxc_cores: 4
+sandbox_lxc_swap_mb: 2048
+sandbox_lxc_pubkey: "{{ lookup('file', '~/.ssh/id_ed25519.pub') }}"
+sandbox_internal_ca_url: https://bao.ad.ddupan.top:8200/v1/pki/ca/pem
+sandbox_base_packages:
+  - ca-certificates
+  - curl
+  - jq
+  - openssh-server
+
+sandbox_postgresql_version: 16
+sandbox_postgresql_primary_address: 10.60.0.11
+sandbox_postgresql_standby_address: 10.60.0.12
+sandbox_postgresql_lb_address: 10.60.0.1
+sandbox_postgresql_database: k3s
+sandbox_postgresql_user: k3s
+sandbox_postgresql_replication_user: k3s_replication
+sandbox_postgresql_replication_slot: sandbox2
+sandbox_postgresql_rotate_passwords: false
+sandbox_postgresql_k3s_password: "{{ lookup('env', 'SANDBOX_K3S_DB_PASSWORD') }}"
+sandbox_postgresql_replication_password: "{{ lookup('env', 'SANDBOX_REPLICATION_PASSWORD') }}"
+
+# This is rendered only at playbook runtime from the Bao-provided password. K3s
+# necessarily persists its datastore credential, but config.yaml is root-only.
+sandbox_k3s_datastore_password_encoded: >-
+  {{ sandbox_postgresql_k3s_password | urlencode | replace('/', '%2F') }}
+sandbox_k3s_datastore_endpoint: >-
+  {{
+    [
+      'postgres://', sandbox_postgresql_user, ':', sandbox_k3s_datastore_password_encoded,
+      '@', sandbox_postgresql_lb_address, ':5432/', sandbox_postgresql_database, '?sslmode=disable'
+    ] | join
+  }}
+server_config_yaml: |
+  datastore-endpoint: {{ sandbox_k3s_datastore_endpoint }}
+  disable:
+    - servicelb
+    - traefik
+user_kubectl: false
+
+sandbox_flux_version: v2.9.5
+sandbox_flux_install_url: >-
+  https://github.com/fluxcd/flux2/releases/download/{{ sandbox_flux_version }}/install.yaml
+sandbox_flux_repository_url: https://git.ddupan.top/panxiao81/homelab-infra.git
+sandbox_flux_repository_branch: main
+sandbox_flux_path: ./clusters/sandbox

infrastructure/sandbox-cluster/ansible/inventory/hosts.yml

@@ -0,0 +1,48 @@
+---
+all:
+  children:
+    sandbox_pve:
+      hosts:
+        pve1:
+          ansible_host: 192.168.10.4
+          sandbox_lxc:
+            vmid: 148
+            hostname: sandbox1
+            address: 10.60.0.11/24
+            memory_mb: 6144
+        pve2:
+          ansible_host: 192.168.10.7
+          sandbox_lxc:
+            vmid: 149
+            hostname: sandbox2
+            address: 10.60.0.12/24
+            memory_mb: 4096
+
+    k3s_cluster:
+      children:
+        server:
+          hosts:
+            sandbox1:
+              ansible_host: 10.60.0.11
+            sandbox2:
+              ansible_host: 10.60.0.12
+        agent:
+          hosts: {}
+      vars:
+        ansible_user: root
+        k3s_version: v1.36.4+k3s1
+        api_endpoint: 10.60.0.13
+        k3s_server_config_mode: "0600"
+        use_external_database: true
+
+    postgres_primary:
+      hosts:
+        sandbox1:
+
+    postgres_standby:
+      hosts:
+        sandbox2:
+
+  vars:
+    ansible_user: root
+    ansible_python_interpreter: /usr/bin/python3

infrastructure/sandbox-cluster/ansible/k3s.yml

@@ -0,0 +1,17 @@
+---
+- name: Validate runtime K3s datastore credential
+  hosts: k3s_cluster
+  gather_facts: false
+  tasks:
+    - name: Require the Bao-provided PostgreSQL password
+      ansible.builtin.assert:
+        that:
+          - sandbox_postgresql_k3s_password | length > 0
+          - server_config_yaml is defined
+          - (server_config_yaml | from_yaml)['datastore-endpoint'] == sandbox_k3s_datastore_endpoint
+        fail_msg: SANDBOX_K3S_DB_PASSWORD must be populated from Bao
+        quiet: true
+      no_log: true
+
+- name: Install the sandbox K3s control plane
+  ansible.builtin.import_playbook: k3s.orchestration.site

infrastructure/sandbox-cluster/ansible/requirements.txt

@@ -0,0 +1,4 @@
+ansible-core==2.21.2
+proxmoxer==2.3.0
+requests==2.34.2
+netaddr==1.3.0

infrastructure/sandbox-cluster/ansible/requirements.yml

@@ -0,0 +1,15 @@
+---
+collections:
+  - name: community.proxmox
+    version: 2.0.0
+  - name: community.postgresql
+    version: 4.2.0
+  - name: ansible.posix
+    version: 2.2.2
+  - name: community.general
+    version: 13.4.0
+  - name: community.library_inventory_filtering_v1
+    version: 1.1.5
+  - name: https://github.com/k3s-io/k3s-ansible.git
+    type: git
+    version: 1a600b60d37e0f8a6e2e79b0e474147b5b108ae5

infrastructure/sandbox-cluster/ansible/roles/sandbox_base/handlers/main.yml

@@ -0,0 +1,5 @@
+---
+- name: Update sandbox CA certificates
+  ansible.builtin.command:
+    cmd: update-ca-certificates
+  changed_when: true

infrastructure/sandbox-cluster/ansible/roles/sandbox_base/tasks/main.yml

@@ -0,0 +1,58 @@
+---
+- name: Install sandbox base packages
+  ansible.builtin.apt:
+    name: "{{ sandbox_base_packages }}"
+    state: present
+    update_cache: true
+    cache_valid_time: 3600
+
+- name: Fetch the homelab internal CA
+  ansible.builtin.uri:
+    url: "{{ sandbox_internal_ca_url }}"
+    return_content: true
+    validate_certs: true
+  register: sandbox_internal_ca
+  changed_when: false
+  retries: 3
+  delay: 5
+  until: sandbox_internal_ca is succeeded
+
+- name: Validate the homelab internal CA response
+  ansible.builtin.assert:
+    that:
+      - "'BEGIN CERTIFICATE' in sandbox_internal_ca.content"
+    fail_msg: OpenBao did not return a PEM certificate
+    quiet: true
+
+- name: Install the homelab internal CA
+  ansible.builtin.copy:
+    dest: /usr/local/share/ca-certificates/ddupan-internal-ca.crt
+    content: "{{ sandbox_internal_ca.content }}"
+    owner: root
+    group: root
+    mode: "0644"
+  notify: Update sandbox CA certificates
+
+- name: Enable the SSH service
+  ansible.builtin.service:
+    name: ssh
+    enabled: true
+    state: started
+
+# LXC does not expose the host kernel log device. Kubelet only needs a writable
+# kmsg-compatible character device, and the container console is the established
+# LXC mapping for this purpose. tmpfiles recreates the link after every boot.
+- name: Persist the LXC kubelet kmsg mapping
+  ansible.builtin.copy:
+    dest: /etc/tmpfiles.d/kmsg.conf
+    content: "L+ /dev/kmsg - - - - /dev/console\n"
+    owner: root
+    group: root
+    mode: "0644"
+
+- name: Expose the LXC console as kmsg
+  ansible.builtin.file:
+    src: /dev/console
+    dest: /dev/kmsg
+    state: link
+    force: true

infrastructure/sandbox-cluster/ansible/roles/sandbox_flux/tasks/main.yml

@@ -0,0 +1,30 @@
+---
+- name: Wait for the K3s server manifest directory
+  ansible.builtin.wait_for:
+    path: /var/lib/rancher/k3s/server/manifests
+    state: present
+    timeout: 120
+
+- name: Download the pinned Flux controller manifest
+  ansible.builtin.get_url:
+    url: "{{ sandbox_flux_install_url }}"
+    dest: /var/lib/rancher/k3s/server/manifests/flux-install.yaml
+    owner: root
+    group: root
+    mode: "0644"
+    timeout: 120
+
+- name: Read the installed homelab CA
+  ansible.builtin.slurp:
+    src: /usr/local/share/ca-certificates/ddupan-internal-ca.crt
+  register: sandbox_flux_ca
+  no_log: true
+
+- name: Install the Flux source and root sync manifest
+  ansible.builtin.template:
+    src: flux-sync.yaml.j2
+    dest: /var/lib/rancher/k3s/server/manifests/flux-sync.yaml
+    owner: root
+    group: root
+    mode: "0600"
+  no_log: true

infrastructure/sandbox-cluster/ansible/roles/sandbox_flux/templates/flux-sync.yaml.j2

@@ -0,0 +1,39 @@
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: flux-system
+  namespace: flux-system
+type: Opaque
+stringData:
+  caFile: |
+{{ sandbox_flux_ca.content | b64decode | indent(4, true) }}
+---
+apiVersion: source.toolkit.fluxcd.io/v1
+kind: GitRepository
+metadata:
+  name: flux-system
+  namespace: flux-system
+spec:
+  interval: 1m
+  ref:
+    branch: {{ sandbox_flux_repository_branch }}
+  secretRef:
+    name: flux-system
+  timeout: 60s
+  url: {{ sandbox_flux_repository_url }}
+---
+apiVersion: kustomize.toolkit.fluxcd.io/v1
+kind: Kustomization
+metadata:
+  name: flux-system
+  namespace: flux-system
+spec:
+  interval: 10m
+  path: {{ sandbox_flux_path }}
+  prune: false
+  sourceRef:
+    kind: GitRepository
+    name: flux-system
+  timeout: 3m
+  wait: true

infrastructure/sandbox-cluster/ansible/roles/sandbox_lxc/tasks/main.yml

@@ -0,0 +1,214 @@
+---
+- name: Validate sandbox declaration and API inputs
+  ansible.builtin.assert:
+    that:
+      - sandbox_lxc.vmid | int > 100
+      - sandbox_lxc.hostname | length > 0
+      - sandbox_lxc.address is match('^10\.60\.0\.[0-9]{1,3}/24$')
+      - sandbox_lxc.memory_mb | int >= 4096
+      - sandbox_pve_api_host | length > 0
+      - sandbox_pve_api_user | length > 0
+      - sandbox_pve_api_token_id | length > 0
+      - sandbox_pve_api_token_secret | length > 0
+    fail_msg: Invalid sandbox declaration or missing Proxmox API environment
+  no_log: true
+
+- name: Check whether the LXC already exists
+  ansible.builtin.stat:
+    path: /etc/pve/lxc/{{ sandbox_lxc.vmid }}.conf
+  register: sandbox_lxc_config
+
+- name: Read current sandbox LXC runtime state
+  community.proxmox.proxmox_vm_info:
+    api_host: "{{ sandbox_pve_api_host }}"
+    api_user: "{{ sandbox_pve_api_user }}"
+    api_token_id: "{{ sandbox_pve_api_token_id }}"
+    api_token_secret: "{{ sandbox_pve_api_token_secret }}"
+    validate_certs: "{{ sandbox_pve_validate_certs }}"
+    ca_path: /etc/ssl/certs/ca-certificates.crt
+    api_timeout: 120
+    vmid: "{{ sandbox_lxc.vmid }}"
+  register: sandbox_lxc_info
+  delegate_to: localhost
+  no_log: true
+  vars:
+    ansible_python_interpreter: "{{ sandbox_pve_api_python_interpreter }}"
+
+- name: Read current sandbox LXC configuration
+  ansible.builtin.uri:
+    url: >-
+      https://{{ sandbox_pve_api_host }}:8006/api2/json/nodes/{{ inventory_hostname }}/lxc/{{ sandbox_lxc.vmid }}/config
+    headers:
+      Authorization: >-
+        PVEAPIToken={{ sandbox_pve_api_user }}!{{ sandbox_pve_api_token_id }}={{ sandbox_pve_api_token_secret }}
+    ca_path: /etc/ssl/certs/ca-certificates.crt
+    return_content: true
+    status_code: [200, 404, 500]
+  register: sandbox_lxc_current_config
+  delegate_to: localhost
+  no_log: true
+
+- name: Reconcile sandbox LXC through the PVE API
+  community.proxmox.proxmox:
+    api_host: "{{ sandbox_pve_api_host }}"
+    api_user: "{{ sandbox_pve_api_user }}"
+    api_token_id: "{{ sandbox_pve_api_token_id }}"
+    api_token_secret: "{{ sandbox_pve_api_token_secret }}"
+    validate_certs: "{{ sandbox_pve_validate_certs }}"
+    ca_path: /etc/ssl/certs/ca-certificates.crt
+    api_timeout: 120
+    node: "{{ inventory_hostname }}"
+    vmid: "{{ sandbox_lxc.vmid }}"
+    hostname: "{{ sandbox_lxc.hostname }}"
+    ostemplate: "{{ sandbox_lxc_template }}"
+    ostype: ubuntu
+    cmode: shell
+    disk_volume:
+      storage: "{{ sandbox_rootfs_storage }}"
+      size: "{{ sandbox_rootfs_gb if not sandbox_lxc_config.stat.exists else omit }}"
+    cores: "{{ sandbox_lxc_cores }}"
+    memory: "{{ sandbox_lxc.memory_mb }}"
+    swap: "{{ sandbox_lxc_swap_mb }}"
+    netif:
+      net0: >-
+        {{
+          'name=eth0,bridge=' ~ sandbox_bridge ~
+          ',ip=' ~ sandbox_lxc.address ~
+          ',gw=' ~ sandbox_gateway ~
+          ',type=veth' ~
+          (',hwaddr=' ~ _sandbox_existing_mac if _sandbox_existing_mac | length > 0 else '')
+        }}
+    nameserver: "{{ sandbox_nameserver }}"
+    searchdomain: "{{ sandbox_search_domain }}"
+    unprivileged: false
+    onboot: true
+    startup: ["order=30", "up=30", "down=60"]
+    tags: [ansible, sandbox, k3s, kata]
+    pubkey: "{{ sandbox_lxc_pubkey }}"
+    update: true
+    state: present
+    timeout: 120
+  register: sandbox_lxc_api
+  when: not sandbox_lxc_config.stat.exists or _sandbox_api_drift
+  delegate_to: localhost
+  no_log: true
+  vars:
+    ansible_python_interpreter: "{{ sandbox_pve_api_python_interpreter }}"
+    _sandbox_existing_mac: >-
+      {{ sandbox_lxc_current_config.json.data.net0.split('hwaddr=')[1].split(',')[0]
+         if 'hwaddr=' in (sandbox_lxc_current_config.json.data.net0 | default(''))
+         else '' }}
+    _sandbox_api_drift: >-
+      {{
+        sandbox_lxc_current_config.status != 200 or
+        sandbox_lxc_current_config.json.data.hostname | default('') != sandbox_lxc.hostname or
+        sandbox_lxc_current_config.json.data.cores | default(0) | int != sandbox_lxc_cores | int or
+        sandbox_lxc_current_config.json.data.memory | default(0) | int != sandbox_lxc.memory_mb | int or
+        sandbox_lxc_current_config.json.data.swap | default(0) | int != sandbox_lxc_swap_mb | int or
+        sandbox_lxc_current_config.json.data.nameserver | default('') != sandbox_nameserver or
+        sandbox_lxc_current_config.json.data.searchdomain | default('') != sandbox_search_domain or
+        sandbox_lxc_current_config.json.data.cmode | default('') != 'shell' or
+        sandbox_lxc_current_config.json.data.onboot | default(0) | int != 1 or
+        sandbox_lxc_current_config.json.data.tags | default('') != 'ansible;k3s;kata;sandbox' or
+        ('bridge=' ~ sandbox_bridge) not in (sandbox_lxc_current_config.json.data.net0 | default('')) or
+        ('ip=' ~ sandbox_lxc.address) not in (sandbox_lxc_current_config.json.data.net0 | default('')) or
+        ('gw=' ~ sandbox_gateway) not in (sandbox_lxc_current_config.json.data.net0 | default('')) or
+        (sandbox_rootfs_storage ~ ':') not in (sandbox_lxc_current_config.json.data.rootfs | default(''))
+      }}
+
+- name: Grow existing sandbox rootfs through the PVE resize API
+  ansible.builtin.uri:
+    url: >-
+      https://{{ sandbox_pve_api_host }}:8006/api2/json/nodes/{{ inventory_hostname }}/lxc/{{ sandbox_lxc.vmid }}/resize
+    method: PUT
+    headers:
+      Authorization: >-
+        PVEAPIToken={{ sandbox_pve_api_user }}!{{ sandbox_pve_api_token_id }}={{ sandbox_pve_api_token_secret }}
+    body_format: form-urlencoded
+    body:
+      disk: rootfs
+      size: "{{ sandbox_rootfs_gb }}G"
+    ca_path: /etc/ssl/certs/ca-certificates.crt
+    status_code: 200
+  when:
+    - sandbox_lxc_config.stat.exists
+    - _sandbox_current_rootfs_gb | int < sandbox_rootfs_gb | int
+  delegate_to: localhost
+  no_log: true
+  vars:
+    _sandbox_current_rootfs_gb: >-
+      {{
+        sandbox_lxc_current_config.json.data.rootfs |
+        default('') |
+        regex_findall('size=([0-9]+)G') |
+        first |
+        default('0')
+      }}
+
+# These properties are not exposed by community.proxmox. Every retained line
+# is required by the already validated nested Kata/kind runtime.
+- name: Reconcile Kata-specific native LXC properties
+  ansible.builtin.lineinfile:
+    path: /etc/pve/lxc/{{ sandbox_lxc.vmid }}.conf
+    regexp: "^{{ item.key | regex_escape }}:"
+    line: "{{ item.key }}: {{ item.value }}"
+  loop:
+    - { key: features, value: "nesting=1,keyctl=1,fuse=1,mknod=1,force_rw_sys=1" }
+    - { key: dev0, value: "path=/dev/kvm,mode=0660" }
+    - { key: dev1, value: "path=/dev/vhost-net,mode=0660" }
+    - { key: dev2, value: "path=/dev/vhost-vsock,mode=0660" }
+    - { key: dev3, value: "path=/dev/net/tun,mode=0666" }
+    - { key: lxc.apparmor.profile, value: unconfined }
+    - { key: lxc.cgroup2.devices.allow, value: a }
+    - { key: lxc.cap.drop, value: "" }
+    - { key: lxc.mount.auto, value: "proc:rw sys:rw" }
+    - { key: lxc.mount.entry, value: "/lib/modules usr/lib/modules none bind,ro,create=dir 0 0" }
+  loop_control:
+    label: "{{ item.key }}"
+  register: sandbox_lxc_native
+
+- name: Start sandbox LXC
+  community.proxmox.proxmox:
+    api_host: "{{ sandbox_pve_api_host }}"
+    api_user: "{{ sandbox_pve_api_user }}"
+    api_token_id: "{{ sandbox_pve_api_token_id }}"
+    api_token_secret: "{{ sandbox_pve_api_token_secret }}"
+    validate_certs: "{{ sandbox_pve_validate_certs }}"
+    ca_path: /etc/ssl/certs/ca-certificates.crt
+    api_timeout: 120
+    vmid: "{{ sandbox_lxc.vmid }}"
+    state: started
+    timeout: 120
+  when: >-
+    not _sandbox_was_running or
+    not (sandbox_lxc_api is changed or sandbox_lxc_native is changed)
+  delegate_to: localhost
+  no_log: true
+  vars:
+    ansible_python_interpreter: "{{ sandbox_pve_api_python_interpreter }}"
+    _sandbox_was_running: >-
+      {{ (sandbox_lxc_info.proxmox_vms | default([]) | first | default({})).status |
+         default('absent') == 'running' }}
+
+- name: Restart existing sandbox LXC after configuration changes
+  community.proxmox.proxmox:
+    api_host: "{{ sandbox_pve_api_host }}"
+    api_user: "{{ sandbox_pve_api_user }}"
+    api_token_id: "{{ sandbox_pve_api_token_id }}"
+    api_token_secret: "{{ sandbox_pve_api_token_secret }}"
+    validate_certs: "{{ sandbox_pve_validate_certs }}"
+    ca_path: /etc/ssl/certs/ca-certificates.crt
+    api_timeout: 120
+    vmid: "{{ sandbox_lxc.vmid }}"
+    state: restarted
+    timeout: 120
+  when:
+    - _sandbox_was_running
+    - sandbox_lxc_api is changed or sandbox_lxc_native is changed
+  delegate_to: localhost
+  no_log: true
+  vars:
+    ansible_python_interpreter: "{{ sandbox_pve_api_python_interpreter }}"
+    _sandbox_was_running: >-
+      {{ (sandbox_lxc_info.proxmox_vms | default([]) | first | default({})).status |
+         default('absent') == 'running' }}

infrastructure/sandbox-cluster/ansible/roles/sandbox_postgresql/handlers/main.yml

@@ -0,0 +1,6 @@
+---
+- name: Restart sandbox PostgreSQL
+  ansible.builtin.service:
+    name: postgresql
+    enabled: true
+    state: restarted

infrastructure/sandbox-cluster/ansible/roles/sandbox_postgresql/tasks/main.yml

@@ -0,0 +1,50 @@
+---
+- name: Validate sandbox PostgreSQL inputs
+  ansible.builtin.assert:
+    that:
+      - sandbox_postgresql_k3s_password | length >= 32
+      - sandbox_postgresql_replication_password | length >= 32
+      - inventory_hostname in groups['postgres_primary'] or inventory_hostname in groups['postgres_standby']
+    fail_msg: Missing Bao-provided PostgreSQL credentials or invalid inventory role
+  no_log: true
+
+- name: Install PostgreSQL packages
+  ansible.builtin.apt:
+    name:
+      - postgresql-{{ sandbox_postgresql_version }}
+      - postgresql-client-{{ sandbox_postgresql_version }}
+      - python3-psycopg2
+    state: present
+    update_cache: true
+    cache_valid_time: 3600
+
+- name: Check whether synchronous replication has completed bootstrap
+  ansible.builtin.stat:
+    path: /etc/postgresql/{{ sandbox_postgresql_version }}/main/sandbox-synchronous-ready
+  register: sandbox_postgresql_synchronous_marker
+
+- name: Install sandbox PostgreSQL access policy
+  ansible.builtin.template:
+    src: pg_hba.conf.j2
+    dest: /etc/postgresql/{{ sandbox_postgresql_version }}/main/pg_hba.conf
+    owner: postgres
+    group: postgres
+    mode: "0640"
+  notify: Restart sandbox PostgreSQL
+
+- name: Install sandbox PostgreSQL server settings
+  ansible.builtin.template:
+    src: sandbox.conf.j2
+    dest: /etc/postgresql/{{ sandbox_postgresql_version }}/main/conf.d/99-sandbox.conf
+    owner: postgres
+    group: postgres
+    mode: "0644"
+  notify: Restart sandbox PostgreSQL
+
+- name: Configure sandbox PostgreSQL primary
+  ansible.builtin.include_tasks: primary.yml
+  when: inventory_hostname in groups['postgres_primary']
+
+- name: Configure sandbox PostgreSQL standby
+  ansible.builtin.include_tasks: standby.yml
+  when: inventory_hostname in groups['postgres_standby']

infrastructure/sandbox-cluster/ansible/roles/sandbox_postgresql/tasks/primary.yml

@@ -0,0 +1,45 @@
+---
+- name: Apply primary PostgreSQL configuration before replication setup
+  ansible.builtin.meta: flush_handlers
+
+- name: Create K3s PostgreSQL login role
+  community.postgresql.postgresql_user:
+    name: "{{ sandbox_postgresql_user }}"
+    password: "{{ sandbox_postgresql_k3s_password }}"
+    no_password_changes: "{{ not sandbox_postgresql_rotate_passwords }}"
+    role_attr_flags: LOGIN,NOSUPERUSER,NOCREATEDB,NOCREATEROLE,NOREPLICATION
+    state: present
+  become: true
+  become_user: postgres
+  no_log: true
+  environment:
+    PGOPTIONS: -c password_encryption=scram-sha-256
+
+- name: Create K3s PostgreSQL database
+  community.postgresql.postgresql_db:
+    name: "{{ sandbox_postgresql_database }}"
+    owner: "{{ sandbox_postgresql_user }}"
+    state: present
+  become: true
+  become_user: postgres
+
+- name: Create PostgreSQL replication login role
+  community.postgresql.postgresql_user:
+    name: "{{ sandbox_postgresql_replication_user }}"
+    password: "{{ sandbox_postgresql_replication_password }}"
+    no_password_changes: "{{ not sandbox_postgresql_rotate_passwords }}"
+    role_attr_flags: LOGIN,REPLICATION,NOSUPERUSER,NOCREATEDB,NOCREATEROLE
+    state: present
+  become: true
+  become_user: postgres
+  no_log: true
+  environment:
+    PGOPTIONS: -c password_encryption=scram-sha-256
+
+- name: Create physical replication slot for sandbox2
+  community.postgresql.postgresql_slot:
+    name: "{{ sandbox_postgresql_replication_slot }}"
+    slot_type: physical
+    state: present
+  become: true
+  become_user: postgres

infrastructure/sandbox-cluster/ansible/roles/sandbox_postgresql/tasks/standby.yml

@@ -0,0 +1,114 @@
+---
+- name: Detect whether sandbox2 is already a standby
+  community.postgresql.postgresql_query:
+    login_db: postgres
+    query: SELECT pg_is_in_recovery() AS in_recovery
+  become: true
+  become_user: postgres
+  register: sandbox_postgresql_recovery_state
+
+- name: Inspect databases before the initial standby seed
+  community.postgresql.postgresql_query:
+    login_db: postgres
+    query: >-
+      SELECT datname FROM pg_database
+      WHERE NOT datistemplate AND datname <> 'postgres'
+  become: true
+  become_user: postgres
+  register: sandbox_postgresql_existing_databases
+  when: not sandbox_postgresql_recovery_state.query_result[0].in_recovery
+
+- name: Refuse to overwrite a non-empty PostgreSQL node
+  ansible.builtin.assert:
+    that:
+      - sandbox_postgresql_existing_databases.query_result | length == 0
+    fail_msg: Refusing to reseed sandbox2 because it contains non-system databases
+  when: not sandbox_postgresql_recovery_state.query_result[0].in_recovery
+
+- name: Stop PostgreSQL before the initial standby seed
+  ansible.builtin.service:
+    name: postgresql
+    state: stopped
+  when: not sandbox_postgresql_recovery_state.query_result[0].in_recovery
+
+- name: Remove the verified-empty standby data directory
+  ansible.builtin.file:
+    path: /var/lib/postgresql/{{ sandbox_postgresql_version }}/main
+    state: absent
+  when: not sandbox_postgresql_recovery_state.query_result[0].in_recovery
+
+- name: Recreate the standby data directory
+  ansible.builtin.file:
+    path: /var/lib/postgresql/{{ sandbox_postgresql_version }}/main
+    state: directory
+    owner: postgres
+    group: postgres
+    mode: "0700"
+  when: not sandbox_postgresql_recovery_state.query_result[0].in_recovery
+
+- name: Install standby replication password file
+  ansible.builtin.copy:
+    dest: /var/lib/postgresql/.pgpass
+    content: >-
+      {{ sandbox_postgresql_primary_address }}:5432:*:{{ sandbox_postgresql_replication_user }}:{{ sandbox_postgresql_replication_password }}
+    owner: postgres
+    group: postgres
+    mode: "0600"
+  no_log: true
+
+- name: Seed sandbox2 from the PostgreSQL primary
+  ansible.builtin.command:
+    argv:
+      - /usr/bin/pg_basebackup
+      - --host={{ sandbox_postgresql_primary_address }}
+      - --username={{ sandbox_postgresql_replication_user }}
+      - --pgdata=/var/lib/postgresql/{{ sandbox_postgresql_version }}/main
+      - --format=plain
+      - --wal-method=stream
+      - --write-recovery-conf
+      - --slot={{ sandbox_postgresql_replication_slot }}
+  become: true
+  become_user: postgres
+  environment:
+    PGPASSFILE: /var/lib/postgresql/.pgpass
+  when: not sandbox_postgresql_recovery_state.query_result[0].in_recovery
+  no_log: true
+
+- name: Set the standby connection identity
+  ansible.builtin.lineinfile:
+    path: /var/lib/postgresql/{{ sandbox_postgresql_version }}/main/postgresql.auto.conf
+    regexp: ^primary_conninfo =
+    line: >-
+      primary_conninfo = 'host={{ sandbox_postgresql_primary_address }} port=5432
+      user={{ sandbox_postgresql_replication_user }} application_name=sandbox2
+      passfile=/var/lib/postgresql/.pgpass'
+    owner: postgres
+    group: postgres
+    mode: "0600"
+  no_log: true
+
+- name: Start PostgreSQL standby
+  ansible.builtin.service:
+    name: postgresql
+    enabled: true
+    state: started
+
+- name: Wait for sandbox2 to enter recovery
+  community.postgresql.postgresql_query:
+    login_db: postgres
+    query: SELECT pg_is_in_recovery() AS in_recovery
+  become: true
+  become_user: postgres
+  register: sandbox_postgresql_standby_ready
+  retries: 12
+  delay: 5
+  until: sandbox_postgresql_standby_ready.query_result[0].in_recovery
+
+- name: Mark synchronous replication bootstrap complete on the primary
+  ansible.builtin.copy:
+    dest: /etc/postgresql/{{ sandbox_postgresql_version }}/main/sandbox-synchronous-ready
+    content: "sandbox2\n"
+    owner: postgres
+    group: postgres
+    mode: "0644"
+  delegate_to: "{{ groups['postgres_primary'][0] }}"

infrastructure/sandbox-cluster/ansible/roles/sandbox_postgresql/templates/pg_hba.conf.j2

@@ -0,0 +1,5 @@
+# Ansible managed
+local   all             postgres                                peer
+local   all             all                                     peer
+host    {{ sandbox_postgresql_database }} {{ sandbox_postgresql_user }} 10.60.0.0/24 scram-sha-256
+host    replication     {{ sandbox_postgresql_replication_user }} 10.60.0.0/24 scram-sha-256

infrastructure/sandbox-cluster/ansible/roles/sandbox_postgresql/templates/sandbox.conf.j2

@@ -0,0 +1,11 @@
+# Ansible managed
+listen_addresses = '{{ ansible_host }},127.0.0.1'
+password_encryption = 'scram-sha-256'
+wal_level = 'replica'
+max_wal_senders = 10
+max_replication_slots = 10
+hot_standby = on
+{% if inventory_hostname in groups['postgres_primary'] and sandbox_postgresql_synchronous_marker.stat.exists %}
+synchronous_standby_names = 'FIRST 1 (sandbox2)'
+synchronous_commit = 'remote_apply'
+{% endif %}

infrastructure/sandbox-cluster/ansible/site.yml

@@ -0,0 +1,54 @@
+---
+- name: Reconcile sandbox LXC nodes
+  hosts: sandbox_pve
+  gather_facts: false
+  serial: 1
+  roles:
+    - sandbox_lxc
+
+- name: Reconcile sandbox node OS baseline
+  hosts: k3s_cluster
+  gather_facts: false
+  roles:
+    - sandbox_base
+
+- name: Reconcile sandbox PostgreSQL primary
+  hosts: postgres_primary
+  gather_facts: false
+  roles:
+    - sandbox_postgresql
+
+- name: Reconcile sandbox PostgreSQL synchronous standby
+  hosts: postgres_standby
+  gather_facts: false
+  roles:
+    - sandbox_postgresql
+
+- name: Finalize synchronous PostgreSQL settings
+  hosts: postgres_primary
+  gather_facts: false
+  roles:
+    - sandbox_postgresql
+
+- name: Validate runtime K3s datastore credential
+  hosts: k3s_cluster
+  gather_facts: false
+  tasks:
+    - name: Require the Bao-provided PostgreSQL password
+      ansible.builtin.assert:
+        that:
+          - sandbox_postgresql_k3s_password | length > 0
+          - server_config_yaml is defined
+          - (server_config_yaml | from_yaml)['datastore-endpoint'] == sandbox_k3s_datastore_endpoint
+        fail_msg: SANDBOX_K3S_DB_PASSWORD must be populated from Bao
+        quiet: true
+      no_log: true
+
+- name: Install the sandbox K3s control plane
+  ansible.builtin.import_playbook: k3s.orchestration.site
+
+- name: Reconcile Flux controllers and root sync
+  hosts: sandbox1
+  gather_facts: false
+  roles:
+    - sandbox_flux

infrastructure/sandbox-cluster/ansible/verify.yml

@@ -0,0 +1,149 @@
+---
+- name: Verify the PostgreSQL datastore through the VyOS endpoint
+  hosts: k3s_cluster
+  gather_facts: false
+  tasks:
+    - name: Query the K3s database through the stable endpoint
+      community.postgresql.postgresql_query:
+        login_host: "{{ sandbox_postgresql_lb_address }}"
+        login_db: "{{ sandbox_postgresql_database }}"
+        login_user: "{{ sandbox_postgresql_user }}"
+        login_password: "{{ sandbox_postgresql_k3s_password }}"
+        query: SELECT NOT pg_is_in_recovery() AS writable
+      register: sandbox_datastore_endpoint
+      no_log: true
+
+    - name: Assert the stable datastore endpoint is writable
+      ansible.builtin.assert:
+        that:
+          - sandbox_datastore_endpoint.query_result[0].writable
+        fail_msg: The VyOS datastore endpoint is not connected to a writable PostgreSQL primary
+        quiet: true
+
+- name: Verify synchronous PostgreSQL replication
+  hosts: postgres_primary
+  gather_facts: false
+  tasks:
+    - name: Query replication state on the primary
+      community.postgresql.postgresql_query:
+        login_db: postgres
+        query: >-
+          SELECT application_name, state, sync_state
+          FROM pg_stat_replication
+          WHERE application_name = 'sandbox2'
+      become: true
+      become_user: postgres
+      register: sandbox_replication_state
+
+    - name: Assert sandbox2 is a synchronous streaming standby
+      ansible.builtin.assert:
+        that:
+          - sandbox_replication_state.query_result | length == 1
+          - sandbox_replication_state.query_result[0].state == 'streaming'
+          - sandbox_replication_state.query_result[0].sync_state == 'sync'
+        fail_msg: sandbox2 is not synchronously streaming from sandbox1
+        quiet: true
+
+- name: Verify PostgreSQL standby recovery state
+  hosts: postgres_standby
+  gather_facts: false
+  tasks:
+    - name: Query recovery state on the standby
+      community.postgresql.postgresql_query:
+        login_db: postgres
+        query: SELECT pg_is_in_recovery() AS in_recovery
+      become: true
+      become_user: postgres
+      register: sandbox_standby_state
+
+    - name: Assert sandbox2 remains in recovery
+      ansible.builtin.assert:
+        that:
+          - sandbox_standby_state.query_result[0].in_recovery
+        fail_msg: sandbox2 is not operating as a PostgreSQL standby
+        quiet: true
+
+- name: Verify K3s node prerequisites
+  hosts: k3s_cluster
+  gather_facts: false
+  tasks:
+    - name: Read the kmsg mapping
+      ansible.builtin.command:
+        cmd: readlink /dev/kmsg
+      register: sandbox_kmsg_target
+      changed_when: false
+
+    - name: Inspect the host kernel module mount
+      ansible.builtin.shell:
+        cmd: >-
+          set -o pipefail &&
+          findmnt -rn -T "/lib/modules/$(uname -r)" -o OPTIONS |
+          grep -Eq '(^|,)ro(,|$)'
+        executable: /bin/bash
+      changed_when: false
+
+    - name: Assert required LXC kernel integration
+      ansible.builtin.assert:
+        that:
+          - sandbox_kmsg_target.stdout == '/dev/console'
+        fail_msg: LXC is missing its persistent kmsg or read-only host module mapping
+        quiet: true
+
+    - name: Read K3s service state
+      ansible.builtin.systemd_service:
+        name: k3s
+      register: sandbox_k3s_service
+
+    - name: Assert K3s is active
+      ansible.builtin.assert:
+        that:
+          - sandbox_k3s_service.status.ActiveState == 'active'
+          - sandbox_k3s_service.status.SubState == 'running'
+        fail_msg: K3s is not running
+        quiet: true
+
+- name: Verify the K3s control plane and API VIP
+  hosts: sandbox1
+  gather_facts: false
+  tasks:
+    - name: Wait for all declared nodes
+      ansible.builtin.command:
+        cmd: k3s kubectl wait --for=condition=Ready nodes --all --timeout=120s
+      changed_when: false
+
+    - name: Read control-plane nodes
+      ansible.builtin.command:
+        cmd: >-
+          k3s kubectl get nodes
+          -l node-role.kubernetes.io/control-plane=true
+          -o name
+      register: sandbox_control_plane_nodes
+      changed_when: false
+
+    - name: Assert both control-plane nodes are registered
+      ansible.builtin.assert:
+        that:
+          - sandbox_control_plane_nodes.stdout_lines | length == 2
+        fail_msg: The sandbox cluster does not contain both control-plane nodes
+        quiet: true
+
+    - name: Wait for all kube-system Pods
+      ansible.builtin.command:
+        cmd: >-
+          k3s kubectl wait --namespace kube-system
+          --for=condition=Ready pods --all --timeout=120s
+      changed_when: false
+
+    - name: Verify the authenticated API path through VyOS
+      ansible.builtin.command:
+        cmd: >-
+          k3s kubectl --server=https://10.60.0.13:6443 get --raw=/livez
+      register: sandbox_api_livez
+      changed_when: false
+
+    - name: Assert the API VIP is live
+      ansible.builtin.assert:
+        that:
+          - sandbox_api_livez.stdout == 'ok'
+        fail_msg: The K3s API is not healthy through the VyOS VIP
+        quiet: true