feat: 声明 sandbox 双节点 K3s 集群
yaml / yaml (pull_request) Successful in 18s
ansible / collection-test (pull_request) Successful in 1m16s
ansible / lint (pull_request) Successful in 2m18s

This commit is contained in:
2026-09-17 14:57:48 +00:00
parent cadfee0aea
commit 6585d8c46a
25 changed files with 1132 additions and 0 deletions
@@ -0,0 +1,5 @@
---
- name: Update sandbox CA certificates
ansible.builtin.command:
cmd: update-ca-certificates
changed_when: true
@@ -0,0 +1,58 @@
---
- name: Install sandbox base packages
ansible.builtin.apt:
name: "{{ sandbox_base_packages }}"
state: present
update_cache: true
cache_valid_time: 3600
- name: Fetch the homelab internal CA
ansible.builtin.uri:
url: "{{ sandbox_internal_ca_url }}"
return_content: true
validate_certs: true
register: sandbox_internal_ca
changed_when: false
retries: 3
delay: 5
until: sandbox_internal_ca is succeeded
- name: Validate the homelab internal CA response
ansible.builtin.assert:
that:
- "'BEGIN CERTIFICATE' in sandbox_internal_ca.content"
fail_msg: OpenBao did not return a PEM certificate
quiet: true
- name: Install the homelab internal CA
ansible.builtin.copy:
dest: /usr/local/share/ca-certificates/ddupan-internal-ca.crt
content: "{{ sandbox_internal_ca.content }}"
owner: root
group: root
mode: "0644"
notify: Update sandbox CA certificates
- name: Enable the SSH service
ansible.builtin.service:
name: ssh
enabled: true
state: started
# LXC does not expose the host kernel log device. Kubelet only needs a writable
# kmsg-compatible character device, and the container console is the established
# LXC mapping for this purpose. tmpfiles recreates the link after every boot.
- name: Persist the LXC kubelet kmsg mapping
ansible.builtin.copy:
dest: /etc/tmpfiles.d/kmsg.conf
content: "L+ /dev/kmsg - - - - /dev/console\n"
owner: root
group: root
mode: "0644"
- name: Expose the LXC console as kmsg
ansible.builtin.file:
src: /dev/console
dest: /dev/kmsg
state: link
force: true
@@ -0,0 +1,30 @@
---
- name: Wait for the K3s server manifest directory
ansible.builtin.wait_for:
path: /var/lib/rancher/k3s/server/manifests
state: present
timeout: 120
- name: Download the pinned Flux controller manifest
ansible.builtin.get_url:
url: "{{ sandbox_flux_install_url }}"
dest: /var/lib/rancher/k3s/server/manifests/flux-install.yaml
owner: root
group: root
mode: "0644"
timeout: 120
- name: Read the installed homelab CA
ansible.builtin.slurp:
src: /usr/local/share/ca-certificates/ddupan-internal-ca.crt
register: sandbox_flux_ca
no_log: true
- name: Install the Flux source and root sync manifest
ansible.builtin.template:
src: flux-sync.yaml.j2
dest: /var/lib/rancher/k3s/server/manifests/flux-sync.yaml
owner: root
group: root
mode: "0600"
no_log: true
@@ -0,0 +1,39 @@
---
apiVersion: v1
kind: Secret
metadata:
name: flux-system
namespace: flux-system
type: Opaque
stringData:
caFile: |
{{ sandbox_flux_ca.content | b64decode | indent(4, true) }}
---
apiVersion: source.toolkit.fluxcd.io/v1
kind: GitRepository
metadata:
name: flux-system
namespace: flux-system
spec:
interval: 1m
ref:
branch: {{ sandbox_flux_repository_branch }}
secretRef:
name: flux-system
timeout: 60s
url: {{ sandbox_flux_repository_url }}
---
apiVersion: kustomize.toolkit.fluxcd.io/v1
kind: Kustomization
metadata:
name: flux-system
namespace: flux-system
spec:
interval: 10m
path: {{ sandbox_flux_path }}
prune: false
sourceRef:
kind: GitRepository
name: flux-system
timeout: 3m
wait: true
@@ -0,0 +1,214 @@
---
- name: Validate sandbox declaration and API inputs
ansible.builtin.assert:
that:
- sandbox_lxc.vmid | int > 100
- sandbox_lxc.hostname | length > 0
- sandbox_lxc.address is match('^10\.60\.0\.[0-9]{1,3}/24$')
- sandbox_lxc.memory_mb | int >= 4096
- sandbox_pve_api_host | length > 0
- sandbox_pve_api_user | length > 0
- sandbox_pve_api_token_id | length > 0
- sandbox_pve_api_token_secret | length > 0
fail_msg: Invalid sandbox declaration or missing Proxmox API environment
no_log: true
- name: Check whether the LXC already exists
ansible.builtin.stat:
path: /etc/pve/lxc/{{ sandbox_lxc.vmid }}.conf
register: sandbox_lxc_config
- name: Read current sandbox LXC runtime state
community.proxmox.proxmox_vm_info:
api_host: "{{ sandbox_pve_api_host }}"
api_user: "{{ sandbox_pve_api_user }}"
api_token_id: "{{ sandbox_pve_api_token_id }}"
api_token_secret: "{{ sandbox_pve_api_token_secret }}"
validate_certs: "{{ sandbox_pve_validate_certs }}"
ca_path: /etc/ssl/certs/ca-certificates.crt
api_timeout: 120
vmid: "{{ sandbox_lxc.vmid }}"
register: sandbox_lxc_info
delegate_to: localhost
no_log: true
vars:
ansible_python_interpreter: "{{ sandbox_pve_api_python_interpreter }}"
- name: Read current sandbox LXC configuration
ansible.builtin.uri:
url: >-
https://{{ sandbox_pve_api_host }}:8006/api2/json/nodes/{{ inventory_hostname }}/lxc/{{ sandbox_lxc.vmid }}/config
headers:
Authorization: >-
PVEAPIToken={{ sandbox_pve_api_user }}!{{ sandbox_pve_api_token_id }}={{ sandbox_pve_api_token_secret }}
ca_path: /etc/ssl/certs/ca-certificates.crt
return_content: true
status_code: [200, 404, 500]
register: sandbox_lxc_current_config
delegate_to: localhost
no_log: true
- name: Reconcile sandbox LXC through the PVE API
community.proxmox.proxmox:
api_host: "{{ sandbox_pve_api_host }}"
api_user: "{{ sandbox_pve_api_user }}"
api_token_id: "{{ sandbox_pve_api_token_id }}"
api_token_secret: "{{ sandbox_pve_api_token_secret }}"
validate_certs: "{{ sandbox_pve_validate_certs }}"
ca_path: /etc/ssl/certs/ca-certificates.crt
api_timeout: 120
node: "{{ inventory_hostname }}"
vmid: "{{ sandbox_lxc.vmid }}"
hostname: "{{ sandbox_lxc.hostname }}"
ostemplate: "{{ sandbox_lxc_template }}"
ostype: ubuntu
cmode: shell
disk_volume:
storage: "{{ sandbox_rootfs_storage }}"
size: "{{ sandbox_rootfs_gb if not sandbox_lxc_config.stat.exists else omit }}"
cores: "{{ sandbox_lxc_cores }}"
memory: "{{ sandbox_lxc.memory_mb }}"
swap: "{{ sandbox_lxc_swap_mb }}"
netif:
net0: >-
{{
'name=eth0,bridge=' ~ sandbox_bridge ~
',ip=' ~ sandbox_lxc.address ~
',gw=' ~ sandbox_gateway ~
',type=veth' ~
(',hwaddr=' ~ _sandbox_existing_mac if _sandbox_existing_mac | length > 0 else '')
}}
nameserver: "{{ sandbox_nameserver }}"
searchdomain: "{{ sandbox_search_domain }}"
unprivileged: false
onboot: true
startup: ["order=30", "up=30", "down=60"]
tags: [ansible, sandbox, k3s, kata]
pubkey: "{{ sandbox_lxc_pubkey }}"
update: true
state: present
timeout: 120
register: sandbox_lxc_api
when: not sandbox_lxc_config.stat.exists or _sandbox_api_drift
delegate_to: localhost
no_log: true
vars:
ansible_python_interpreter: "{{ sandbox_pve_api_python_interpreter }}"
_sandbox_existing_mac: >-
{{ sandbox_lxc_current_config.json.data.net0.split('hwaddr=')[1].split(',')[0]
if 'hwaddr=' in (sandbox_lxc_current_config.json.data.net0 | default(''))
else '' }}
_sandbox_api_drift: >-
{{
sandbox_lxc_current_config.status != 200 or
sandbox_lxc_current_config.json.data.hostname | default('') != sandbox_lxc.hostname or
sandbox_lxc_current_config.json.data.cores | default(0) | int != sandbox_lxc_cores | int or
sandbox_lxc_current_config.json.data.memory | default(0) | int != sandbox_lxc.memory_mb | int or
sandbox_lxc_current_config.json.data.swap | default(0) | int != sandbox_lxc_swap_mb | int or
sandbox_lxc_current_config.json.data.nameserver | default('') != sandbox_nameserver or
sandbox_lxc_current_config.json.data.searchdomain | default('') != sandbox_search_domain or
sandbox_lxc_current_config.json.data.cmode | default('') != 'shell' or
sandbox_lxc_current_config.json.data.onboot | default(0) | int != 1 or
sandbox_lxc_current_config.json.data.tags | default('') != 'ansible;k3s;kata;sandbox' or
('bridge=' ~ sandbox_bridge) not in (sandbox_lxc_current_config.json.data.net0 | default('')) or
('ip=' ~ sandbox_lxc.address) not in (sandbox_lxc_current_config.json.data.net0 | default('')) or
('gw=' ~ sandbox_gateway) not in (sandbox_lxc_current_config.json.data.net0 | default('')) or
(sandbox_rootfs_storage ~ ':') not in (sandbox_lxc_current_config.json.data.rootfs | default(''))
}}
- name: Grow existing sandbox rootfs through the PVE resize API
ansible.builtin.uri:
url: >-
https://{{ sandbox_pve_api_host }}:8006/api2/json/nodes/{{ inventory_hostname }}/lxc/{{ sandbox_lxc.vmid }}/resize
method: PUT
headers:
Authorization: >-
PVEAPIToken={{ sandbox_pve_api_user }}!{{ sandbox_pve_api_token_id }}={{ sandbox_pve_api_token_secret }}
body_format: form-urlencoded
body:
disk: rootfs
size: "{{ sandbox_rootfs_gb }}G"
ca_path: /etc/ssl/certs/ca-certificates.crt
status_code: 200
when:
- sandbox_lxc_config.stat.exists
- _sandbox_current_rootfs_gb | int < sandbox_rootfs_gb | int
delegate_to: localhost
no_log: true
vars:
_sandbox_current_rootfs_gb: >-
{{
sandbox_lxc_current_config.json.data.rootfs |
default('') |
regex_findall('size=([0-9]+)G') |
first |
default('0')
}}
# These properties are not exposed by community.proxmox. Every retained line
# is required by the already validated nested Kata/kind runtime.
- name: Reconcile Kata-specific native LXC properties
ansible.builtin.lineinfile:
path: /etc/pve/lxc/{{ sandbox_lxc.vmid }}.conf
regexp: "^{{ item.key | regex_escape }}:"
line: "{{ item.key }}: {{ item.value }}"
loop:
- { key: features, value: "nesting=1,keyctl=1,fuse=1,mknod=1,force_rw_sys=1" }
- { key: dev0, value: "path=/dev/kvm,mode=0660" }
- { key: dev1, value: "path=/dev/vhost-net,mode=0660" }
- { key: dev2, value: "path=/dev/vhost-vsock,mode=0660" }
- { key: dev3, value: "path=/dev/net/tun,mode=0666" }
- { key: lxc.apparmor.profile, value: unconfined }
- { key: lxc.cgroup2.devices.allow, value: a }
- { key: lxc.cap.drop, value: "" }
- { key: lxc.mount.auto, value: "proc:rw sys:rw" }
- { key: lxc.mount.entry, value: "/lib/modules usr/lib/modules none bind,ro,create=dir 0 0" }
loop_control:
label: "{{ item.key }}"
register: sandbox_lxc_native
- name: Start sandbox LXC
community.proxmox.proxmox:
api_host: "{{ sandbox_pve_api_host }}"
api_user: "{{ sandbox_pve_api_user }}"
api_token_id: "{{ sandbox_pve_api_token_id }}"
api_token_secret: "{{ sandbox_pve_api_token_secret }}"
validate_certs: "{{ sandbox_pve_validate_certs }}"
ca_path: /etc/ssl/certs/ca-certificates.crt
api_timeout: 120
vmid: "{{ sandbox_lxc.vmid }}"
state: started
timeout: 120
when: >-
not _sandbox_was_running or
not (sandbox_lxc_api is changed or sandbox_lxc_native is changed)
delegate_to: localhost
no_log: true
vars:
ansible_python_interpreter: "{{ sandbox_pve_api_python_interpreter }}"
_sandbox_was_running: >-
{{ (sandbox_lxc_info.proxmox_vms | default([]) | first | default({})).status |
default('absent') == 'running' }}
- name: Restart existing sandbox LXC after configuration changes
community.proxmox.proxmox:
api_host: "{{ sandbox_pve_api_host }}"
api_user: "{{ sandbox_pve_api_user }}"
api_token_id: "{{ sandbox_pve_api_token_id }}"
api_token_secret: "{{ sandbox_pve_api_token_secret }}"
validate_certs: "{{ sandbox_pve_validate_certs }}"
ca_path: /etc/ssl/certs/ca-certificates.crt
api_timeout: 120
vmid: "{{ sandbox_lxc.vmid }}"
state: restarted
timeout: 120
when:
- _sandbox_was_running
- sandbox_lxc_api is changed or sandbox_lxc_native is changed
delegate_to: localhost
no_log: true
vars:
ansible_python_interpreter: "{{ sandbox_pve_api_python_interpreter }}"
_sandbox_was_running: >-
{{ (sandbox_lxc_info.proxmox_vms | default([]) | first | default({})).status |
default('absent') == 'running' }}
@@ -0,0 +1,6 @@
---
- name: Restart sandbox PostgreSQL
ansible.builtin.service:
name: postgresql
enabled: true
state: restarted
@@ -0,0 +1,50 @@
---
- name: Validate sandbox PostgreSQL inputs
ansible.builtin.assert:
that:
- sandbox_postgresql_k3s_password | length >= 32
- sandbox_postgresql_replication_password | length >= 32
- inventory_hostname in groups['postgres_primary'] or inventory_hostname in groups['postgres_standby']
fail_msg: Missing Bao-provided PostgreSQL credentials or invalid inventory role
no_log: true
- name: Install PostgreSQL packages
ansible.builtin.apt:
name:
- postgresql-{{ sandbox_postgresql_version }}
- postgresql-client-{{ sandbox_postgresql_version }}
- python3-psycopg2
state: present
update_cache: true
cache_valid_time: 3600
- name: Check whether synchronous replication has completed bootstrap
ansible.builtin.stat:
path: /etc/postgresql/{{ sandbox_postgresql_version }}/main/sandbox-synchronous-ready
register: sandbox_postgresql_synchronous_marker
- name: Install sandbox PostgreSQL access policy
ansible.builtin.template:
src: pg_hba.conf.j2
dest: /etc/postgresql/{{ sandbox_postgresql_version }}/main/pg_hba.conf
owner: postgres
group: postgres
mode: "0640"
notify: Restart sandbox PostgreSQL
- name: Install sandbox PostgreSQL server settings
ansible.builtin.template:
src: sandbox.conf.j2
dest: /etc/postgresql/{{ sandbox_postgresql_version }}/main/conf.d/99-sandbox.conf
owner: postgres
group: postgres
mode: "0644"
notify: Restart sandbox PostgreSQL
- name: Configure sandbox PostgreSQL primary
ansible.builtin.include_tasks: primary.yml
when: inventory_hostname in groups['postgres_primary']
- name: Configure sandbox PostgreSQL standby
ansible.builtin.include_tasks: standby.yml
when: inventory_hostname in groups['postgres_standby']
@@ -0,0 +1,45 @@
---
- name: Apply primary PostgreSQL configuration before replication setup
ansible.builtin.meta: flush_handlers
- name: Create K3s PostgreSQL login role
community.postgresql.postgresql_user:
name: "{{ sandbox_postgresql_user }}"
password: "{{ sandbox_postgresql_k3s_password }}"
no_password_changes: "{{ not sandbox_postgresql_rotate_passwords }}"
role_attr_flags: LOGIN,NOSUPERUSER,NOCREATEDB,NOCREATEROLE,NOREPLICATION
state: present
become: true
become_user: postgres
no_log: true
environment:
PGOPTIONS: -c password_encryption=scram-sha-256
- name: Create K3s PostgreSQL database
community.postgresql.postgresql_db:
name: "{{ sandbox_postgresql_database }}"
owner: "{{ sandbox_postgresql_user }}"
state: present
become: true
become_user: postgres
- name: Create PostgreSQL replication login role
community.postgresql.postgresql_user:
name: "{{ sandbox_postgresql_replication_user }}"
password: "{{ sandbox_postgresql_replication_password }}"
no_password_changes: "{{ not sandbox_postgresql_rotate_passwords }}"
role_attr_flags: LOGIN,REPLICATION,NOSUPERUSER,NOCREATEDB,NOCREATEROLE
state: present
become: true
become_user: postgres
no_log: true
environment:
PGOPTIONS: -c password_encryption=scram-sha-256
- name: Create physical replication slot for sandbox2
community.postgresql.postgresql_slot:
name: "{{ sandbox_postgresql_replication_slot }}"
slot_type: physical
state: present
become: true
become_user: postgres
@@ -0,0 +1,114 @@
---
- name: Detect whether sandbox2 is already a standby
community.postgresql.postgresql_query:
login_db: postgres
query: SELECT pg_is_in_recovery() AS in_recovery
become: true
become_user: postgres
register: sandbox_postgresql_recovery_state
- name: Inspect databases before the initial standby seed
community.postgresql.postgresql_query:
login_db: postgres
query: >-
SELECT datname FROM pg_database
WHERE NOT datistemplate AND datname <> 'postgres'
become: true
become_user: postgres
register: sandbox_postgresql_existing_databases
when: not sandbox_postgresql_recovery_state.query_result[0].in_recovery
- name: Refuse to overwrite a non-empty PostgreSQL node
ansible.builtin.assert:
that:
- sandbox_postgresql_existing_databases.query_result | length == 0
fail_msg: Refusing to reseed sandbox2 because it contains non-system databases
when: not sandbox_postgresql_recovery_state.query_result[0].in_recovery
- name: Stop PostgreSQL before the initial standby seed
ansible.builtin.service:
name: postgresql
state: stopped
when: not sandbox_postgresql_recovery_state.query_result[0].in_recovery
- name: Remove the verified-empty standby data directory
ansible.builtin.file:
path: /var/lib/postgresql/{{ sandbox_postgresql_version }}/main
state: absent
when: not sandbox_postgresql_recovery_state.query_result[0].in_recovery
- name: Recreate the standby data directory
ansible.builtin.file:
path: /var/lib/postgresql/{{ sandbox_postgresql_version }}/main
state: directory
owner: postgres
group: postgres
mode: "0700"
when: not sandbox_postgresql_recovery_state.query_result[0].in_recovery
- name: Install standby replication password file
ansible.builtin.copy:
dest: /var/lib/postgresql/.pgpass
content: >-
{{ sandbox_postgresql_primary_address }}:5432:*:{{ sandbox_postgresql_replication_user }}:{{ sandbox_postgresql_replication_password }}
owner: postgres
group: postgres
mode: "0600"
no_log: true
- name: Seed sandbox2 from the PostgreSQL primary
ansible.builtin.command:
argv:
- /usr/bin/pg_basebackup
- --host={{ sandbox_postgresql_primary_address }}
- --username={{ sandbox_postgresql_replication_user }}
- --pgdata=/var/lib/postgresql/{{ sandbox_postgresql_version }}/main
- --format=plain
- --wal-method=stream
- --write-recovery-conf
- --slot={{ sandbox_postgresql_replication_slot }}
become: true
become_user: postgres
environment:
PGPASSFILE: /var/lib/postgresql/.pgpass
when: not sandbox_postgresql_recovery_state.query_result[0].in_recovery
no_log: true
- name: Set the standby connection identity
ansible.builtin.lineinfile:
path: /var/lib/postgresql/{{ sandbox_postgresql_version }}/main/postgresql.auto.conf
regexp: ^primary_conninfo =
line: >-
primary_conninfo = 'host={{ sandbox_postgresql_primary_address }} port=5432
user={{ sandbox_postgresql_replication_user }} application_name=sandbox2
passfile=/var/lib/postgresql/.pgpass'
owner: postgres
group: postgres
mode: "0600"
no_log: true
- name: Start PostgreSQL standby
ansible.builtin.service:
name: postgresql
enabled: true
state: started
- name: Wait for sandbox2 to enter recovery
community.postgresql.postgresql_query:
login_db: postgres
query: SELECT pg_is_in_recovery() AS in_recovery
become: true
become_user: postgres
register: sandbox_postgresql_standby_ready
retries: 12
delay: 5
until: sandbox_postgresql_standby_ready.query_result[0].in_recovery
- name: Mark synchronous replication bootstrap complete on the primary
ansible.builtin.copy:
dest: /etc/postgresql/{{ sandbox_postgresql_version }}/main/sandbox-synchronous-ready
content: "sandbox2\n"
owner: postgres
group: postgres
mode: "0644"
delegate_to: "{{ groups['postgres_primary'][0] }}"
@@ -0,0 +1,5 @@
# Ansible managed
local all postgres peer
local all all peer
host {{ sandbox_postgresql_database }} {{ sandbox_postgresql_user }} 10.60.0.0/24 scram-sha-256
host replication {{ sandbox_postgresql_replication_user }} 10.60.0.0/24 scram-sha-256
@@ -0,0 +1,11 @@
# Ansible managed
listen_addresses = '{{ ansible_host }},127.0.0.1'
password_encryption = 'scram-sha-256'
wal_level = 'replica'
max_wal_senders = 10
max_replication_slots = 10
hot_standby = on
{% if inventory_hostname in groups['postgres_primary'] and sandbox_postgresql_synchronous_marker.stat.exists %}
synchronous_standby_names = 'FIRST 1 (sandbox2)'
synchronous_commit = 'remote_apply'
{% endif %}