diff --git a/infrastructure/dns/records.yml b/infrastructure/dns/records.yml index 96e6862..c7d3dd9 100644 --- a/infrastructure/dns/records.yml +++ b/infrastructure/dns/records.yml @@ -27,6 +27,9 @@ homelab_dns: - { zone: ad.ddupan.top, name: zot, type: A, values: [192.168.10.127] } - { zone: ad.ddupan.top, name: zot-push, type: A, values: [192.168.10.127] } + - { zone: ad.ddupan.top, name: pg-prod, type: A, values: [192.168.10.2] } + - { zone: ad.ddupan.top, name: pg-dev, type: A, values: [192.168.10.127] } + split_horizon: # backends records the current adoption boundary. obj is deliberately not # emitted to CoreDNS yet, preserving the current pod resolver behaviour. diff --git a/infrastructure/etcd/.gitignore b/infrastructure/etcd/.gitignore new file mode 100644 index 0000000..5e36e64 --- /dev/null +++ b/infrastructure/etcd/.gitignore @@ -0,0 +1,6 @@ +.terraform/ +*.tfstate* +*.tfplan +*.tfvars +!*.tfvars.example +__pycache__/ diff --git a/infrastructure/etcd/ansible/ansible.cfg b/infrastructure/etcd/ansible/ansible.cfg new file mode 100644 index 0000000..8003ffe --- /dev/null +++ b/infrastructure/etcd/ansible/ansible.cfg @@ -0,0 +1,6 @@ +[defaults] +inventory = inventory/hosts.yml +roles_path = roles +retry_files_enabled = False +host_key_checking = True +interpreter_python = auto_silent diff --git a/infrastructure/etcd/ansible/bootstrap-auth.yml b/infrastructure/etcd/ansible/bootstrap-auth.yml new file mode 100644 index 0000000..1ef20f4 --- /dev/null +++ b/infrastructure/etcd/ansible/bootstrap-auth.yml @@ -0,0 +1,60 @@ +--- +# 与常规 site 分离,防止重建/恢复时默默创建新的认证域。 +- name: 初始化共享 etcd 认证 + hosts: etcd[0] + become: true + gather_facts: false + pre_tasks: + - name: 加载共享默认参数 + ansible.builtin.import_role: + name: shared_etcd + tasks_from: context + environment: + ETCDCTL_ENDPOINTS: "https://{{ etcd_address }}:{{ etcd_client_port }}" + ETCDCTL_CACERT: "{{ etcd_config_dir }}/ca.crt" + ETCDCTL_CERT: "{{ etcd_config_dir }}/admin.crt" + ETCDCTL_KEY: "{{ etcd_config_dir }}/admin.key" + tasks: + - name: 读取认证状态 + ansible.builtin.command: + argv: ["{{ etcd_install_dir }}/etcdctl", --write-out=json, auth, status] + changed_when: false + register: etcd_auth_status + check_mode: false + + - name: 初始化管理员与认证 + when: not ((etcd_auth_status.stdout | from_json).enabled | default(false)) + block: + - name: 要求显式初始化参数 + ansible.builtin.assert: + that: etcd_bootstrap_auth | default(false) | bool + fail_msg: 首次初始化需要 -e etcd_bootstrap_auth=true;常规运行不能重建认证。 + + - name: 读取现有用户 + ansible.builtin.command: + argv: ["{{ etcd_install_dir }}/etcdctl", --write-out=json, user, list] + changed_when: false + register: etcd_users + + - name: 创建仅证书认证的 root 用户 + ansible.builtin.command: + argv: ["{{ etcd_install_dir }}/etcdctl", user, add, root, --no-password] + when: "'root' not in ((etcd_users.stdout | from_json).users | default([], true))" + changed_when: true + + - name: 读取管理员角色 + ansible.builtin.command: + argv: ["{{ etcd_install_dir }}/etcdctl", --write-out=json, user, get, root] + changed_when: false + register: etcd_root_roles + + - name: 授予 root 管理角色 + ansible.builtin.command: + argv: ["{{ etcd_install_dir }}/etcdctl", user, grant-role, root, root] + when: "'root' not in ((etcd_root_roles.stdout | from_json).roles | default([], true))" + changed_when: true + + - name: 开启认证 + ansible.builtin.command: + argv: ["{{ etcd_install_dir }}/etcdctl", auth, enable] + changed_when: true diff --git a/infrastructure/etcd/ansible/consumers.yml b/infrastructure/etcd/ansible/consumers.yml new file mode 100644 index 0000000..1e20966 --- /dev/null +++ b/infrastructure/etcd/ansible/consumers.yml @@ -0,0 +1,37 @@ +--- +- name: 收敛消费者账号及独立 prefix + hosts: etcd[0] + become: true + gather_facts: false + pre_tasks: + - name: 加载共享默认参数 + ansible.builtin.import_role: + name: shared_etcd + tasks_from: context + environment: + ETCDCTL_ENDPOINTS: "https://{{ etcd_address }}:{{ etcd_client_port }}" + ETCDCTL_CACERT: "{{ etcd_config_dir }}/ca.crt" + ETCDCTL_CERT: "{{ etcd_config_dir }}/admin.crt" + ETCDCTL_KEY: "{{ etcd_config_dir }}/admin.key" + tasks: + - name: 检查认证状态 + ansible.builtin.command: + argv: ["{{ etcd_install_dir }}/etcdctl", --write-out=json, auth, status] + register: etcd_auth_status + changed_when: false + check_mode: false + + - name: 要求认证已启用 + ansible.builtin.assert: + that: + - ((etcd_auth_status.stdout | from_json).enabled | default(false)) + - etcd_consumers | map(attribute='name') | unique | length == etcd_consumers | length + - etcd_bao_token | length > 0 + no_log: true + + - name: 逐个收敛消费者 + ansible.builtin.include_tasks: tasks/consumer.yml + loop: "{{ etcd_consumers }}" + loop_control: + loop_var: etcd_consumer + label: "{{ etcd_consumer.name }}" diff --git a/infrastructure/etcd/ansible/controller.yml b/infrastructure/etcd/ansible/controller.yml new file mode 100644 index 0000000..c0b9a0c --- /dev/null +++ b/infrastructure/etcd/ansible/controller.yml @@ -0,0 +1,74 @@ +--- +# 仅在 Terraform 身份配置完成后部署;预检失败不启用定时器。 +- name: 安装独立于人工会话的续签调度 + hosts: etcd-laptop + become: true + gather_facts: false + tasks: + - name: 创建 root 管理的程序目录 + ansible.builtin.file: + path: /opt/homelab-etcd-controller + state: directory + mode: '0755' + - name: 安装证书登录与调度程序 + ansible.builtin.copy: + src: "../controller/{{ item }}" + dest: "/opt/homelab-etcd-controller/{{ item }}" + owner: root + group: root + mode: '0755' + loop: [login.py, renew.py] + - name: 以机器身份预检(不输出 token) + ansible.builtin.command: + argv: [/usr/bin/python3, /opt/homelab-etcd-controller/login.py] + changed_when: false + no_log: true + - name: 安装固定的 Ansible 配置副本 + ansible.builtin.copy: + src: "{{ playbook_dir }}/" + dest: /opt/homelab-etcd-controller/ansible/ + owner: root + group: root + mode: preserve + - name: 创建调度状态目录 + ansible.builtin.file: + path: /var/lib/homelab-etcd-controller + state: directory + owner: panxiao81 + group: panxiao81 + mode: '0700' + - name: 安装续签 oneshot unit + ansible.builtin.copy: + dest: /etc/systemd/system/homelab-etcd-renew.service + mode: '0644' + content: | + [Unit] + Description=Renew shared etcd certificates through Bao machine identity + After=network-online.target + [Service] + Type=oneshot + User=panxiao81 + Environment=HOME=/home/panxiao81 + Environment=PATH=/home/panxiao81/.local/bin:/usr/local/bin:/usr/bin:/bin + ExecStart=/usr/bin/python3 /opt/homelab-etcd-controller/renew.py + TimeoutStartSec=30min + UMask=0077 + - name: 安装每日续签 timer + ansible.builtin.copy: + dest: /etc/systemd/system/homelab-etcd-renew.timer + mode: '0644' + content: | + [Unit] + Description=Daily shared etcd certificate renewal check + [Timer] + OnCalendar=*-*-* 04:10:00 UTC + RandomizedDelaySec=15min + Persistent=true + [Install] + WantedBy=timers.target + - name: 启用续签调度 + ansible.builtin.systemd_service: + name: homelab-etcd-renew.timer + daemon_reload: true + enabled: true + state: started diff --git a/infrastructure/etcd/ansible/inventory/group_vars/all.yml b/infrastructure/etcd/ansible/inventory/group_vars/all.yml new file mode 100644 index 0000000..3650643 --- /dev/null +++ b/infrastructure/etcd/ansible/inventory/group_vars/all.yml @@ -0,0 +1,20 @@ +--- +etcd_bao_url: https://bao.ad.ddupan.top:8200 +etcd_bao_pki_mount: pki +etcd_bao_kv_mount: kv +etcd_bao_secret_base: infra/etcd/consumers +# 只在控制端使用,不下发 Bao token。也不读取或复制历史 .vault_pass。 +etcd_bao_token: "{{ lookup('env', 'BAO_TOKEN') }}" +etcd_consumers: + - name: patroni-pg-prod + prefix: /homelab/patroni/pg-prod/ +etcd_lxc_template: laptop:vztmpl/ubuntu-24.04-standard_24.04-2_amd64.tar.zst +etcd_lxc_storage: pve-rg +etcd_lxc_bridge: labnet +etcd_lxc_gateway: 10.60.0.1 +etcd_lxc_memory: 512 +etcd_lxc_disk_gb: 8 +etcd_lxc_pubkey: "{{ lookup('file', '~/.ssh/id_ed25519.pub') }}" + +# 与现有 LAN exporter 相同的内网采集边界;禁止映射到公网。 +etcd_metrics_urls: "http://127.0.0.1:2381,http://{{ etcd_address }}:2381" diff --git a/infrastructure/etcd/ansible/inventory/hosts.yml b/infrastructure/etcd/ansible/inventory/hosts.yml new file mode 100644 index 0000000..2e69eff --- /dev/null +++ b/infrastructure/etcd/ansible/inventory/hosts.yml @@ -0,0 +1,33 @@ +--- +# 已部署成员;共置 PG standby/备份的承载预算,数据库数据用独立 HDD mp0。 +all: + vars: + ansible_user: root + children: + etcd: + hosts: + etcd-laptop: + ansible_connection: local + ansible_host: 192.168.10.127 + ansible_user: panxiao81 + etcd_address: 192.168.10.127 + etcd-pve1: + ansible_host: 10.60.0.20 + etcd_address: 10.60.0.20 + etcd-pve2: + ansible_host: 10.60.0.21 + etcd_address: 10.60.0.21 + etcd_pve: + hosts: + pve1: + ansible_host: 192.168.10.4 + etcd_lxc_vmid: 150 + etcd_lxc_hostname: etcd-pve1 + etcd_lxc_address: 10.60.0.20/24 + etcd_lxc_memory: 1536 + pve2: + ansible_host: 192.168.10.7 + etcd_lxc_vmid: 151 + etcd_lxc_hostname: etcd-pve2 + etcd_lxc_address: 10.60.0.21/24 + etcd_lxc_memory: 768 diff --git a/infrastructure/etcd/ansible/lxc.yml b/infrastructure/etcd/ansible/lxc.yml new file mode 100644 index 0000000..2d3bb1d --- /dev/null +++ b/infrastructure/etcd/ansible/lxc.yml @@ -0,0 +1,126 @@ +--- +# 只创建已声明且尚不存在的 LXC;不接管未知 VMID,不重启现有容器。 +- name: 创建独立 etcd LXC + hosts: etcd_pve + become: true + gather_facts: false + tasks: + - name: 读取全局资源,避免 VMID 在其他节点已占用 + ansible.builtin.command: + argv: [pvesh, get, /cluster/resources, --type, vm, --output-format, json] + register: etcd_pve_resources + changed_when: false + check_mode: false + + - name: 保存同号资源 + ansible.builtin.set_fact: + etcd_lxc_existing: >- + {{ etcd_pve_resources.stdout + | from_json + | selectattr('vmid', 'equalto', etcd_lxc_vmid) + | list }} + + - name: 拒绝接管未知资源 + ansible.builtin.assert: + that: + - >- + etcd_lxc_existing | length == 0 or + (etcd_lxc_existing[0].type == 'lxc' and etcd_lxc_existing[0].node == inventory_hostname + and etcd_lxc_existing[0].name == etcd_lxc_hostname + and 'shared-etcd' in (etcd_lxc_existing[0].tags | default(''))) + + - name: 新建无特权 LXC + when: etcd_lxc_existing | length == 0 and not ansible_check_mode + block: + - name: 暂存 SSH 公钥 + ansible.builtin.copy: + content: "{{ etcd_lxc_pubkey }}\n" + dest: /run/shared-etcd-bootstrap.pub + mode: '0600' + - name: 创建声明的容器 + ansible.builtin.command: + argv: + - pct + - create + - "{{ etcd_lxc_vmid }}" + - "{{ etcd_lxc_template }}" + - --hostname + - "{{ etcd_lxc_hostname }}" + - --unprivileged + - '1' + - --cores + - '1' + - --memory + - "{{ etcd_lxc_memory }}" + - --swap + - '0' + - --rootfs + - "{{ etcd_lxc_storage }}:{{ etcd_lxc_disk_gb }}" + - --net0 + - "name=eth0,bridge={{ etcd_lxc_bridge }},ip={{ etcd_lxc_address }},gw={{ etcd_lxc_gateway }},type=veth" + - --nameserver + - 192.168.10.5 + - --searchdomain + - ad.ddupan.top + - --ssh-public-keys + - /run/shared-etcd-bootstrap.pub + - --onboot + - '1' + - --tags + - ansible;shared-etcd + changed_when: true + always: + - name: 删除暂存公钥 + ansible.builtin.file: + path: /run/shared-etcd-bootstrap.pub + state: absent + + - name: 读取容器配置 + ansible.builtin.command: + argv: [pct, config, "{{ etcd_lxc_vmid }}"] + changed_when: false + register: etcd_lxc_config + when: not ansible_check_mode or etcd_lxc_existing | length > 0 + + - name: 配置漂移先报错,不直接改运行中的网络/资源 + ansible.builtin.assert: + that: + - >- + ('ip=' ~ etcd_lxc_address ~ ',') in etcd_lxc_config.stdout or + ('ip=' ~ etcd_lxc_address ~ '\n') in etcd_lxc_config.stdout + - "('bridge=' ~ etcd_lxc_bridge ~ ',') in etcd_lxc_config.stdout" + - "'unprivileged: 1' in etcd_lxc_config.stdout" + - "('memory: ' ~ etcd_lxc_memory) in etcd_lxc_config.stdout" + - "('rootfs: ' ~ etcd_lxc_storage ~ ':') in etcd_lxc_config.stdout" + when: etcd_lxc_config is not skipped + + - name: 读取容器运行状态 + ansible.builtin.command: + argv: [pct, status, "{{ etcd_lxc_vmid }}"] + changed_when: false + register: etcd_lxc_status + when: not ansible_check_mode + + - name: 启动容器 + ansible.builtin.command: + argv: [pct, start, "{{ etcd_lxc_vmid }}"] + changed_when: true + when: not ansible_check_mode and 'running' not in etcd_lxc_status.stdout + + - name: 通过可信宿主机取得容器 SSH 公钥 + ansible.builtin.command: + argv: [pct, exec, "{{ etcd_lxc_vmid }}", --, cat, /etc/ssh/ssh_host_ed25519_key.pub] + register: etcd_lxc_hostkey + changed_when: false + retries: 12 + delay: 5 + until: etcd_lxc_hostkey.rc == 0 + when: not ansible_check_mode + + - name: 保存经宿主机验证的 SSH host key + ansible.builtin.known_hosts: + name: "{{ etcd_lxc_address.split('/')[0] }}" + key: "{{ etcd_lxc_address.split('/')[0] }} {{ etcd_lxc_hostkey.stdout }}" + delegate_to: localhost + become: false + when: not ansible_check_mode diff --git a/infrastructure/etcd/ansible/move-storage.yml b/infrastructure/etcd/ansible/move-storage.yml new file mode 100644 index 0000000..8e3d278 --- /dev/null +++ b/infrastructure/etcd/ansible/move-storage.yml @@ -0,0 +1,90 @@ +--- +# 首次部署承载调整,逐节点迁移;普通 lxc.yml 不自动移动磁盘。 +- name: 逐个将新 etcd 容器迁入声明的 SSD 池 + hosts: etcd_pve + become: true + gather_facts: false + serial: 1 + any_errors_fatal: true + vars: + etcd_move_health_command: + - /opt/homelab-etcd/etcdctl + - --endpoints=https://192.168.10.127:2379,https://10.60.0.20:2379,https://10.60.0.21:2379 + - --cacert=/etc/homelab-etcd/ca.crt + - --cert=/etc/homelab-etcd/admin.crt + - --key=/etc/homelab-etcd/admin.key + - endpoint + - health + tasks: + - name: 核对容器配置 + ansible.builtin.command: + argv: [pct, config, "{{ etcd_lxc_vmid }}"] + register: etcd_move_config + changed_when: false + check_mode: false + + - name: 限定本项目新建容器与允许的源池 + ansible.builtin.assert: + that: + - etcd_lxc_vmid in [150, 151] + - "('hostname: ' ~ etcd_lxc_hostname) in etcd_move_config.stdout" + - "'shared-etcd' in etcd_move_config.stdout" + - "'rootfs: local-lvm:' in etcd_move_config.stdout or 'rootfs: pve-rg:' in etcd_move_config.stdout" + - etcd_lxc_storage == 'pve-rg' + + - name: 验证迁移前全部端点健康 + ansible.builtin.command: + argv: "{{ etcd_move_health_command }}" + delegate_to: localhost + changed_when: false + check_mode: false + + - name: 迁移当前尚在本地池的根卷 + when: "'rootfs: local-lvm:' in etcd_move_config.stdout and not ansible_check_mode" + block: + - name: 创建已验证的集群快照 + ansible.builtin.command: + argv: [systemctl, start, homelab-etcd-snapshot.service] + delegate_to: localhost + changed_when: true + + - name: 正常关闭一个容器 + ansible.builtin.command: + argv: [pct, shutdown, "{{ etcd_lxc_vmid }}", --timeout, '60'] + changed_when: true + + - name: 复制成功后移除该新建容器的原卷 + ansible.builtin.command: + argv: + - pct + - move-volume + - "{{ etcd_lxc_vmid }}" + - rootfs + - "{{ etcd_lxc_storage }}" + - --delete + - '1' + - --bwlimit + - '32768' + changed_when: true + always: + - name: 核对容器运行状态 + ansible.builtin.command: + argv: [pct, status, "{{ etcd_lxc_vmid }}"] + register: etcd_move_status + changed_when: false + - name: 重新启动容器 + ansible.builtin.command: + argv: [pct, start, "{{ etcd_lxc_vmid }}"] + when: "'running' not in etcd_move_status.stdout" + changed_when: true + + - name: 等待当前成员回归 + ansible.builtin.command: + argv: "{{ etcd_move_health_command }}" + delegate_to: localhost + changed_when: false + register: etcd_move_health + retries: 18 + delay: 5 + until: etcd_move_health.rc == 0 + when: not ansible_check_mode diff --git a/infrastructure/etcd/ansible/requirements.yml b/infrastructure/etcd/ansible/requirements.yml new file mode 100644 index 0000000..f9fe5b5 --- /dev/null +++ b/infrastructure/etcd/ansible/requirements.yml @@ -0,0 +1,4 @@ +--- +collections: + - name: community.crypto + version: 3.2.1 diff --git a/infrastructure/etcd/ansible/roles/shared_etcd/defaults/main.yml b/infrastructure/etcd/ansible/roles/shared_etcd/defaults/main.yml new file mode 100644 index 0000000..76354da --- /dev/null +++ b/infrastructure/etcd/ansible/roles/shared_etcd/defaults/main.yml @@ -0,0 +1,17 @@ +--- +etcd_version: 3.7.2 +etcd_archive_checksum: sha256:3a3679bc51a4ee9d30bccea1da7cd4fe62c6fc1d2ca1255068d2c53bf3026135 +etcd_install_dir: /opt/homelab-etcd +etcd_config_dir: /etc/homelab-etcd +etcd_data_dir: /var/lib/homelab-etcd +etcd_cluster_token: homelab-shared-etcd-v1 +etcd_client_port: 2379 +etcd_peer_port: 2380 +etcd_metrics_port: 2381 +etcd_quota_bytes: 268435456 +etcd_memory_high: 256M +etcd_memory_max: 384M +etcd_certificate_ttl: 1440h +etcd_renew_before: +14d +etcd_snapshot_dir: /var/backups/homelab-etcd +etcd_snapshot_keep: 3 diff --git a/infrastructure/etcd/ansible/roles/shared_etcd/tasks/certificate.yml b/infrastructure/etcd/ansible/roles/shared_etcd/tasks/certificate.yml new file mode 100644 index 0000000..62b9319 --- /dev/null +++ b/infrastructure/etcd/ansible/roles/shared_etcd/tasks/certificate.yml @@ -0,0 +1,93 @@ +--- +- name: 在成员本地生成私钥 + community.crypto.openssl_privatekey: + path: "{{ etcd_config_dir }}/{{ etcd_cert.name }}.key" + type: ECC + curve: secp256r1 + owner: root + group: "{{ etcd_cert.group }}" + mode: '0640' + +- name: 本地生成 CSR + community.crypto.openssl_csr: + path: "{{ etcd_config_dir }}/{{ etcd_cert.name }}.csr" + privatekey_path: "{{ etcd_config_dir }}/{{ etcd_cert.name }}.key" + common_name: "{{ etcd_cert.cn | default(omit, true) }}" + subject_alt_name: >- + {{ ['IP:' ~ etcd_address, 'DNS:' ~ inventory_hostname] if etcd_cert.name in ['server', 'peer'] + else (['DNS:' ~ inventory_hostname] if etcd_cert.name == 'gateway' else []) }} + use_common_name_for_san: false + extended_key_usage: "{{ etcd_cert.eku }}" + key_usage: [digitalSignature] + mode: '0644' + register: etcd_csr_state + +- name: 检查已有证书 + ansible.builtin.stat: + path: "{{ etcd_config_dir }}/{{ etcd_cert.name }}.crt" + register: etcd_cert_file + +- name: 检查续签窗口 + community.crypto.x509_certificate_info: + path: "{{ etcd_config_dir }}/{{ etcd_cert.name }}.crt" + valid_at: + renewal: "{{ etcd_renew_before }}" + register: etcd_cert_info + when: etcd_cert_file.stat.exists + +- name: 通过中央 CA 签发需更新的证书 + when: >- + not etcd_cert_file.stat.exists or etcd_csr_state is changed or + not (etcd_cert_info.valid_at.renewal | default(false)) + block: + - name: 仅在确需签发时要求 Bao 凭据 + ansible.builtin.assert: + that: etcd_bao_token | length > 0 + no_log: true + + - name: 读取 CSR 公共内容 + ansible.builtin.slurp: + src: "{{ etcd_config_dir }}/{{ etcd_cert.name }}.csr" + register: etcd_csr + + - name: 控制端提交 Bao 签名请求 + ansible.builtin.uri: + url: "{{ etcd_bao_url }}/v1/{{ etcd_bao_pki_mount }}/sign/{{ etcd_cert.role }}" + method: POST + headers: + X-Vault-Token: "{{ etcd_bao_token }}" + body_format: json + body: + csr: "{{ etcd_csr.content | b64decode }}" + ttl: "{{ etcd_certificate_ttl }}" + status_code: 200 + delegate_to: localhost + become: false + register: etcd_signed + no_log: true + when: not ansible_check_mode + + - name: 保存签发的证书链 + ansible.builtin.copy: + content: | + {{ etcd_signed.json.data.certificate }} + {{ etcd_signed.json.data.ca_chain | join('\n') }} + dest: "{{ etcd_config_dir }}/{{ etcd_cert.name }}.crt" + owner: root + group: "{{ etcd_cert.group }}" + mode: '0644' + when: not ansible_check_mode + + - name: 保存中央 CA 信任链 + ansible.builtin.copy: + content: | + {{ etcd_signed.json.data.ca_chain | join('\n') }} + dest: "{{ etcd_config_dir }}/ca.crt" + owner: root + group: homelab-etcd + mode: '0644' + when: not ansible_check_mode + + - name: 标记证书已更新 + ansible.builtin.set_fact: + etcd_certificates_changed: true diff --git a/infrastructure/etcd/ansible/roles/shared_etcd/tasks/context.yml b/infrastructure/etcd/ansible/roles/shared_etcd/tasks/context.yml new file mode 100644 index 0000000..2fb141d --- /dev/null +++ b/infrastructure/etcd/ansible/roles/shared_etcd/tasks/context.yml @@ -0,0 +1,3 @@ +--- +# 只加载 role defaults,供运维入口复用。 +[] diff --git a/infrastructure/etcd/ansible/roles/shared_etcd/tasks/main.yml b/infrastructure/etcd/ansible/roles/shared_etcd/tasks/main.yml new file mode 100644 index 0000000..0b6b0e9 --- /dev/null +++ b/infrastructure/etcd/ansible/roles/shared_etcd/tasks/main.yml @@ -0,0 +1,134 @@ +--- +- name: 验证拓扑和签发配置 + ansible.builtin.assert: + that: + - groups['etcd'] | length == 3 + - groups['etcd'] | map('extract', hostvars, 'etcd_address') | unique | length == 3 + - ansible_facts['architecture'] == 'x86_64' + - etcd_bao_url is match('^https://') + - etcd_archive_checksum is match('^sha256:[a-f0-9]{64}$') + no_log: true + +- name: 安装证书处理依赖 + ansible.builtin.package: + name: [python3-cryptography, openssl] + state: present + +- name: 创建独立 etcd 组 + ansible.builtin.group: + name: homelab-etcd + system: true + +- name: 创建独立 etcd 用户 + ansible.builtin.user: + name: homelab-etcd + group: homelab-etcd + system: true + shell: /usr/sbin/nologin + create_home: false + +- name: 创建受管目录 + ansible.builtin.file: + path: "{{ item.path }}" + state: directory + owner: "{{ item.owner }}" + group: homelab-etcd + mode: "{{ item.mode }}" + loop: + - {path: "{{ etcd_config_dir }}", owner: root, mode: '0750'} + - {path: "{{ etcd_data_dir }}", owner: homelab-etcd, mode: '0700'} + - {path: "{{ etcd_install_dir }}", owner: root, mode: '0755'} + +- name: 下载固定版本及校验归档 + ansible.builtin.get_url: + url: >- + https://github.com/etcd-io/etcd/releases/download/v{{ etcd_version }}/etcd-v{{ etcd_version }}-linux-amd64.tar.gz + dest: "{{ etcd_install_dir }}/etcd-v{{ etcd_version }}.tar.gz" + checksum: "{{ etcd_archive_checksum }}" + mode: '0644' + register: etcd_download + retries: 3 + delay: 5 + until: etcd_download is succeeded + +- name: 展开固定版本 + ansible.builtin.unarchive: + src: "{{ etcd_install_dir }}/etcd-v{{ etcd_version }}.tar.gz" + dest: "{{ etcd_install_dir }}" + remote_src: true + creates: "{{ etcd_install_dir }}/etcd-v{{ etcd_version }}-linux-amd64/etcd" + +- name: 安装版本链接 + ansible.builtin.file: + src: "{{ etcd_install_dir }}/etcd-v{{ etcd_version }}-linux-amd64/{{ item }}" + dest: "{{ etcd_install_dir }}/{{ item }}" + state: link + loop: [etcd, etcdctl, etcdutl] + register: etcd_binary_links + +- name: 签发成员与管理员证书 + ansible.builtin.include_tasks: certificate.yml + loop: + - {name: server, role: homelab-etcd-server, cn: "{{ inventory_hostname }}", eku: [serverAuth], group: homelab-etcd} + - {name: peer, role: homelab-etcd-peer, cn: homelab-etcd-peer, eku: [serverAuth, clientAuth], group: homelab-etcd} + - {name: gateway, role: homelab-etcd-gateway, cn: "", eku: [clientAuth], group: homelab-etcd} + - {name: admin, role: homelab-etcd-admin, cn: root, eku: [clientAuth], group: root} + loop_control: + loop_var: etcd_cert + +- name: 写入独立 etcd 配置 + ansible.builtin.template: + src: etcd.yml.j2 + dest: "{{ etcd_config_dir }}/etcd.yml" + owner: root + group: homelab-etcd + mode: '0640' + register: etcd_config_file + +- name: 写入独立 systemd unit + ansible.builtin.template: + src: homelab-etcd.service.j2 + dest: /etc/systemd/system/homelab-etcd.service + mode: '0644' + register: etcd_unit + +# 跨失败重跑记录激活状态,防止上一轮写文件后中断导致漏掉必要重启。 +- name: 计算受管文件校验和 + ansible.builtin.stat: + path: "{{ item }}" + checksum_algorithm: sha256 + loop: + - "{{ etcd_config_dir }}/etcd.yml" + - "{{ etcd_config_dir }}/server.crt" + - "{{ etcd_config_dir }}/peer.crt" + - "{{ etcd_config_dir }}/gateway.crt" + - "{{ etcd_config_dir }}/ca.crt" + - /etc/systemd/system/homelab-etcd.service + register: etcd_managed_files + +- name: 计算期望激活指纹 + ansible.builtin.set_fact: + etcd_config_fingerprint: >- + {{ ((etcd_managed_files.results | map(attribute='stat.checksum') | list | join(':')) + ~ ':' ~ etcd_version) | hash('sha256') }} + when: not ansible_check_mode + +- name: 检查已激活指纹 + ansible.builtin.stat: + path: "{{ etcd_config_dir }}/activated.sha256" + register: etcd_activated_file + +- name: 读取已激活指纹 + ansible.builtin.slurp: + src: "{{ etcd_config_dir }}/activated.sha256" + register: etcd_activated + when: etcd_activated_file.stat.exists + +- name: 判断是否需要滚动激活 + ansible.builtin.set_fact: + etcd_config_changed: >- + {{ not etcd_activated_file.stat.exists or + (etcd_activated.content | default('') | b64decode | trim) != etcd_config_fingerprint | default('check-mode') }} + +- name: 管理本地快照任务 + ansible.builtin.import_tasks: snapshot.yml diff --git a/infrastructure/etcd/ansible/roles/shared_etcd/tasks/snapshot.yml b/infrastructure/etcd/ansible/roles/shared_etcd/tasks/snapshot.yml new file mode 100644 index 0000000..71530ea --- /dev/null +++ b/infrastructure/etcd/ansible/roles/shared_etcd/tasks/snapshot.yml @@ -0,0 +1,26 @@ +--- +- name: 创建仅 root 可访问的快照目录 + ansible.builtin.file: + path: "{{ etcd_snapshot_dir }}" + state: directory + owner: root + group: root + mode: '0700' + +- name: 写入快照脚本和 systemd 任务 + ansible.builtin.template: + src: "{{ item.src }}" + dest: "{{ item.dest }}" + mode: "{{ item.mode }}" + loop: + - {src: snapshot.sh.j2, dest: "{{ etcd_install_dir }}/snapshot", mode: '0700'} + - {src: snapshot.service.j2, dest: /etc/systemd/system/homelab-etcd-snapshot.service, mode: '0644'} + - {src: snapshot.timer.j2, dest: /etc/systemd/system/homelab-etcd-snapshot.timer, mode: '0644'} + register: etcd_snapshot_units + +- name: 启用本地快照计划 + ansible.builtin.systemd_service: + name: homelab-etcd-snapshot.timer + daemon_reload: "{{ etcd_snapshot_units is changed }}" + enabled: true + state: started diff --git a/infrastructure/etcd/ansible/roles/shared_etcd/templates/etcd.yml.j2 b/infrastructure/etcd/ansible/roles/shared_etcd/templates/etcd.yml.j2 new file mode 100644 index 0000000..f1fbb38 --- /dev/null +++ b/infrastructure/etcd/ansible/roles/shared_etcd/templates/etcd.yml.j2 @@ -0,0 +1,33 @@ +# Ansible 管理;与 k3s、数据库生命周期独立。 +name: {{ inventory_hostname | to_json }} +data-dir: {{ etcd_data_dir | to_json }} +listen-client-urls: https://{{ etcd_address }}:{{ etcd_client_port }} +advertise-client-urls: https://{{ etcd_address }}:{{ etcd_client_port }} +listen-peer-urls: https://{{ etcd_address }}:{{ etcd_peer_port }} +initial-advertise-peer-urls: https://{{ etcd_address }}:{{ etcd_peer_port }} +initial-cluster: "{% for member in groups['etcd'] %}{{ member }}=https://{{ hostvars[member].etcd_address }}:{{ etcd_peer_port }}{{ ',' if not loop.last else '' }}{% endfor %}" +initial-cluster-token: {{ etcd_cluster_token | to_json }} +initial-cluster-state: new +# 已存在的数据目录优先;成员替换必须走单独 runbook,不删除数据重建。 +client-transport-security: + cert-file: {{ etcd_config_dir }}/server.crt + key-file: {{ etcd_config_dir }}/server.key + client-cert-file: {{ etcd_config_dir }}/gateway.crt + client-key-file: {{ etcd_config_dir }}/gateway.key + trusted-ca-file: {{ etcd_config_dir }}/ca.crt + client-cert-auth: true +peer-transport-security: + cert-file: {{ etcd_config_dir }}/peer.crt + key-file: {{ etcd_config_dir }}/peer.key + trusted-ca-file: {{ etcd_config_dir }}/ca.crt + client-cert-auth: true + allowed-cn: [homelab-etcd-peer] +# 独立 metrics listener 仅提供指标/健康,不开放 KV API;只绑定受管内网地址。 +listen-metrics-urls: {{ etcd_metrics_urls | default("http://127.0.0.1:" ~ etcd_metrics_port) | to_json }} +quota-backend-bytes: {{ etcd_quota_bytes }} +auto-compaction-mode: periodic +auto-compaction-retention: '1h' +heartbeat-interval: 100 +election-timeout: 1000 +logger: zap +log-level: info diff --git a/infrastructure/etcd/ansible/roles/shared_etcd/templates/homelab-etcd.service.j2 b/infrastructure/etcd/ansible/roles/shared_etcd/templates/homelab-etcd.service.j2 new file mode 100644 index 0000000..6ae5f82 --- /dev/null +++ b/infrastructure/etcd/ansible/roles/shared_etcd/templates/homelab-etcd.service.j2 @@ -0,0 +1,23 @@ +[Unit] +Description=Homelab shared etcd +Wants=network-online.target +After=network-online.target + +[Service] +User=homelab-etcd +Group=homelab-etcd +ExecStart={{ etcd_install_dir }}/etcd --config-file={{ etcd_config_dir }}/etcd.yml +Restart=on-failure +RestartSec=5 +TimeoutStopSec=60 +MemoryHigh={{ etcd_memory_high }} +MemoryMax={{ etcd_memory_max }} +UMask=0077 +NoNewPrivileges=true +ProtectSystem=strict +ProtectHome=true +PrivateTmp=true +ReadWritePaths={{ etcd_data_dir }} + +[Install] +WantedBy=multi-user.target diff --git a/infrastructure/etcd/ansible/roles/shared_etcd/templates/snapshot.service.j2 b/infrastructure/etcd/ansible/roles/shared_etcd/templates/snapshot.service.j2 new file mode 100644 index 0000000..0a3c862 --- /dev/null +++ b/infrastructure/etcd/ansible/roles/shared_etcd/templates/snapshot.service.j2 @@ -0,0 +1,11 @@ +[Unit] +Description=Snapshot homelab shared etcd +After=homelab-etcd.service + +[Service] +Type=oneshot +ExecStart={{ etcd_install_dir }}/snapshot +User=root +UMask=0077 +TimeoutStartSec=300 +Nice=10 diff --git a/infrastructure/etcd/ansible/roles/shared_etcd/templates/snapshot.sh.j2 b/infrastructure/etcd/ansible/roles/shared_etcd/templates/snapshot.sh.j2 new file mode 100644 index 0000000..f521193 --- /dev/null +++ b/infrastructure/etcd/ansible/roles/shared_etcd/templates/snapshot.sh.j2 @@ -0,0 +1,20 @@ +#!/bin/bash +set -euo pipefail +umask 077 +export ETCDCTL_ENDPOINTS="https://{{ etcd_address }}:{{ etcd_client_port }}" +export ETCDCTL_CACERT="{{ etcd_config_dir }}/ca.crt" +export ETCDCTL_CERT="{{ etcd_config_dir }}/admin.crt" +export ETCDCTL_KEY="{{ etcd_config_dir }}/admin.key" +repo="{{ etcd_snapshot_dir }}" +exec 9>"$repo/.lock" +flock -n 9 || exit 0 +output="$repo/$(date -u +%Y%m%dT%H%M%SZ).db" +trap 'rm -f "$output.partial" "$output.partial.part"' EXIT +{{ etcd_install_dir }}/etcdctl snapshot save "$output.partial" +{{ etcd_install_dir }}/etcdutl snapshot status "$output.partial" >/dev/null +mv "$output.partial" "$output" +# 只有新快照成功且验证可读才清理旧备份。目录仅由此任务管理。 +mapfile -t snapshots < <(find "$repo" -maxdepth 1 -type f -name '????????T??????Z.db' -printf '%f\n' | sort -r) +for old in "${snapshots[@]:{{ etcd_snapshot_keep }}}"; do + rm -- "$repo/$old" +done diff --git a/infrastructure/etcd/ansible/roles/shared_etcd/templates/snapshot.timer.j2 b/infrastructure/etcd/ansible/roles/shared_etcd/templates/snapshot.timer.j2 new file mode 100644 index 0000000..31dfe83 --- /dev/null +++ b/infrastructure/etcd/ansible/roles/shared_etcd/templates/snapshot.timer.j2 @@ -0,0 +1,10 @@ +[Unit] +Description=Daily homelab etcd snapshot + +[Timer] +OnCalendar=*-*-* 03:20:00 UTC +RandomizedDelaySec=300 +Persistent=true + +[Install] +WantedBy=timers.target diff --git a/infrastructure/etcd/ansible/site.yml b/infrastructure/etcd/ansible/site.yml new file mode 100644 index 0000000..c4573c3 --- /dev/null +++ b/infrastructure/etcd/ansible/site.yml @@ -0,0 +1,66 @@ +--- +# 先配置全体,再滚动启动:首次集群形成不能在第一个节点等待 quorum。 +- name: 配置 shared etcd 成员 + hosts: etcd + become: true + roles: + - shared_etcd + +- name: 滚动启动并验证 shared etcd + hosts: etcd + become: true + serial: 1 + pre_tasks: + - name: 加载共享默认参数 + ansible.builtin.import_role: + name: shared_etcd + tasks_from: context + tasks: + - name: 启动已配置的成员 + ansible.builtin.systemd_service: + name: homelab-etcd + enabled: true + daemon_reload: true + state: "{{ 'restarted' if etcd_config_changed | bool else 'started' }}" + - name: 等待本机客户端端口 + ansible.builtin.wait_for: + host: "{{ etcd_address }}" + port: "{{ etcd_client_port }}" + timeout: 60 + when: not ansible_check_mode + + - name: 已有集群每次激活后等待本成员恢复 quorum 通信 + ansible.builtin.command: + argv: + - "{{ etcd_install_dir }}/etcdctl" + - --endpoints=https://{{ etcd_address }}:{{ etcd_client_port }} + - --cacert={{ etcd_config_dir }}/ca.crt + - --cert={{ etcd_config_dir }}/admin.crt + - --key={{ etcd_config_dir }}/admin.key + - endpoint + - health + changed_when: false + register: etcd_member_health + retries: 12 + delay: 5 + until: etcd_member_health.rc == 0 + when: etcd_activated_file.stat.exists and not ansible_check_mode + +- name: 核对全部成员 + ansible.builtin.import_playbook: verify.yml + +- name: 记录已成功激活的配置 + hosts: etcd + become: true + pre_tasks: + - name: 加载共享默认参数 + ansible.builtin.import_role: + name: shared_etcd + tasks_from: context + tasks: + - name: 写入激活指纹(全体健康检查通过后) + ansible.builtin.copy: + content: "{{ etcd_config_fingerprint }}\n" + dest: "{{ etcd_config_dir }}/activated.sha256" + mode: '0644' + when: not ansible_check_mode diff --git a/infrastructure/etcd/ansible/tasks/consumer.yml b/infrastructure/etcd/ansible/tasks/consumer.yml new file mode 100644 index 0000000..815218b --- /dev/null +++ b/infrastructure/etcd/ansible/tasks/consumer.yml @@ -0,0 +1,170 @@ +--- +- name: 验证消费者范围 + ansible.builtin.assert: + that: + - etcd_consumer.name is match('^[a-z][a-z0-9-]+$') + - etcd_consumer.name != 'root' + - etcd_consumer.prefix is match('^/homelab/[a-zA-Z0-9/_-]+/$') + - etcd_consumer.prefix | length > 10 + +- name: 读取已有用户和角色 + ansible.builtin.command: + argv: ["{{ etcd_install_dir }}/etcdctl", --write-out=json, "{{ item }}", list] + loop: [user, role] + register: etcd_identities + changed_when: false + check_mode: false + +# 404 data 也可能代表被删除/销毁的旧秘密;只有 metadata 不存在才允许生成。 +- name: 控制端读取 Bao 秘密元数据 + ansible.builtin.uri: + url: "{{ etcd_bao_url }}/v1/{{ etcd_bao_kv_mount }}/metadata/{{ etcd_bao_secret_base }}/{{ etcd_consumer.name }}" + headers: + X-Vault-Token: "{{ etcd_bao_token }}" + status_code: [200, 404] + delegate_to: localhost + become: false + no_log: true + register: etcd_secret_metadata + check_mode: false + +- name: 禁止已有用户丢失秘密后自动换密码 + ansible.builtin.assert: + that: >- + etcd_secret_metadata.status == 200 or + etcd_consumer.name not in ((etcd_identities.results[0].stdout | from_json).users | default([], true)) + fail_msg: etcd 用户已存在但 Bao 秘密缺失;需恢复原秘密或执行显式轮换。 + +- name: 首次创建随机秘密且禁止覆盖已有版本 + ansible.builtin.uri: + url: "{{ etcd_bao_url }}/v1/{{ etcd_bao_kv_mount }}/data/{{ etcd_bao_secret_base }}/{{ etcd_consumer.name }}" + method: POST + headers: + X-Vault-Token: "{{ etcd_bao_token }}" + body_format: json + body: + options: {cas: 0} + data: + username: "{{ etcd_consumer.name }}" + password: "{{ lookup('ansible.builtin.password', '/dev/null', length=48, chars=['ascii_letters', 'digits']) }}" + prefix: "{{ etcd_consumer.prefix }}" + status_code: 200 + delegate_to: localhost + become: false + no_log: true + changed_when: true + when: + - etcd_secret_metadata.status == 404 + - not ansible_check_mode + +- name: 控制端读取既有秘密 + ansible.builtin.uri: + url: "{{ etcd_bao_url }}/v1/{{ etcd_bao_kv_mount }}/data/{{ etcd_bao_secret_base }}/{{ etcd_consumer.name }}" + headers: + X-Vault-Token: "{{ etcd_bao_token }}" + status_code: 200 + delegate_to: localhost + become: false + register: etcd_consumer_secret + no_log: true + when: etcd_secret_metadata.status == 200 or not ansible_check_mode + check_mode: false + +- name: 核对已保存秘密归属 + ansible.builtin.assert: + that: + - etcd_consumer_secret.json.data.data.username == etcd_consumer.name + - etcd_consumer_secret.json.data.data.prefix == etcd_consumer.prefix + - etcd_consumer_secret.json.data.data.password | length >= 32 + no_log: true + when: etcd_secret_metadata.status == 200 or not ansible_check_mode + +- name: 新建消费者用户(密码只通过 stdin 传递) + ansible.builtin.command: + argv: ["{{ etcd_install_dir }}/etcdctl", user, add, "{{ etcd_consumer.name }}", --interactive=false] + stdin: "{{ etcd_consumer_secret.json.data.data.password }}" + no_log: true + changed_when: true + when: + - etcd_consumer.name not in ((etcd_identities.results[0].stdout | from_json).users | default([], true)) + - not ansible_check_mode + +- name: 新建消费者角色 + ansible.builtin.command: + argv: ["{{ etcd_install_dir }}/etcdctl", role, add, "{{ etcd_consumer.name }}"] + changed_when: true + when: etcd_consumer.name not in ((etcd_identities.results[1].stdout | from_json).roles | default([], true)) + +- name: 读取角色权限 + ansible.builtin.command: + argv: ["{{ etcd_install_dir }}/etcdctl", --write-out=json, role, get, "{{ etcd_consumer.name }}"] + register: etcd_role_state + changed_when: false + when: >- + not ansible_check_mode or + etcd_consumer.name in ((etcd_identities.results[1].stdout | from_json).roles | default([], true)) + +- name: 拒绝不符合声明的既有权限(不自动扩大或删除) + ansible.builtin.assert: + that: + - >- + ((etcd_role_state.stdout | from_json).perm | default([], true)) in + [[], [{'permType': 2, 'key': etcd_consumer.prefix | b64encode, + 'range_end': (etcd_consumer.prefix[:-1] ~ '0') | b64encode}]] + fail_msg: 既有角色权限与声明不同,请显式审查权限迁移。 + when: etcd_role_state is not skipped + +- name: 授予唯一 prefix 读写权限 + ansible.builtin.command: + argv: + - "{{ etcd_install_dir }}/etcdctl" + - role + - grant-permission + - "{{ etcd_consumer.name }}" + - readwrite + - "{{ etcd_consumer.prefix }}" + - --prefix=true + changed_when: true + when: + - etcd_role_state is not skipped + - ((etcd_role_state.stdout | from_json).perm | default([], true)) | length == 0 + +- name: 读取用户角色 + ansible.builtin.command: + argv: ["{{ etcd_install_dir }}/etcdctl", --write-out=json, user, get, "{{ etcd_consumer.name }}"] + register: etcd_user_state + changed_when: false + when: >- + not ansible_check_mode or + etcd_consumer.name in ((etcd_identities.results[0].stdout | from_json).users | default([], true)) + +- name: 拒绝消费者已有额外角色 + ansible.builtin.assert: + that: >- + ((etcd_user_state.stdout | from_json).roles | default([], true)) + | difference([etcd_consumer.name]) | length == 0 + when: etcd_user_state is not skipped + +- name: 绑定消费者角色 + ansible.builtin.command: + argv: ["{{ etcd_install_dir }}/etcdctl", user, grant-role, "{{ etcd_consumer.name }}", "{{ etcd_consumer.name }}"] + changed_when: true + when: + - etcd_user_state is not skipped + - etcd_consumer.name not in ((etcd_user_state.stdout | from_json).roles | default([], true)) + +# 使用 gateway 的真实密码登录来核对 Bao 与 etcd 一致性,不能只看用户已存在。 +- name: 验证消费者密码可经 gateway 登录 + ansible.builtin.uri: + url: "https://{{ etcd_address }}:{{ etcd_client_port }}/v3/auth/authenticate" + method: POST + client_cert: "{{ etcd_config_dir }}/gateway.crt" + client_key: "{{ etcd_config_dir }}/gateway.key" + ca_path: "{{ etcd_config_dir }}/ca.crt" + body_format: json + body: + name: "{{ etcd_consumer.name }}" + password: "{{ etcd_consumer_secret.json.data.data.password }}" + status_code: 200 + no_log: true + when: not ansible_check_mode diff --git a/infrastructure/etcd/ansible/verify.yml b/infrastructure/etcd/ansible/verify.yml new file mode 100644 index 0000000..abf7ad4 --- /dev/null +++ b/infrastructure/etcd/ansible/verify.yml @@ -0,0 +1,29 @@ +--- +- name: 验证 shared etcd 全部端点 + hosts: etcd[0] + become: true + gather_facts: false + pre_tasks: + - name: 加载共享默认参数 + ansible.builtin.import_role: + name: shared_etcd + tasks_from: context + tasks: + - name: 通过管理员 mTLS 检查所有端点健康 + ansible.builtin.command: + argv: + - "{{ etcd_install_dir }}/etcdctl" + - >- + --endpoints={{ groups['etcd'] | map('extract', hostvars, 'etcd_address') + | map('regex_replace', '^(.*)$', 'https://\1:' ~ etcd_client_port) | join(',') }} + - --cacert={{ etcd_config_dir }}/ca.crt + - --cert={{ etcd_config_dir }}/admin.crt + - --key={{ etcd_config_dir }}/admin.key + - endpoint + - health + changed_when: false + register: etcd_health + retries: 12 + delay: 5 + until: etcd_health.rc == 0 + when: not ansible_check_mode diff --git a/infrastructure/etcd/controller/login.py b/infrastructure/etcd/controller/login.py new file mode 100644 index 0000000..347ee1e --- /dev/null +++ b/infrastructure/etcd/controller/login.py @@ -0,0 +1,30 @@ +#!/usr/bin/python3 +"""仅由受控调用者捕获 stdout;不得手动运行以免在终端输出短期 token。""" +import json +import ssl +import sys +import urllib.request + + +def main(): + context = ssl.create_default_context() + context.load_cert_chain('/etc/homelab-etcd/peer.crt', '/etc/homelab-etcd/peer.key') + request = urllib.request.Request( + 'https://bao.ad.ddupan.top:8200/v1/auth/homelab-etcd-renewal/login', + data=json.dumps({'name': 'etcd-laptop'}).encode(), + headers={'Content-Type': 'application/json'}, method='POST', + ) + with urllib.request.urlopen(request, context=context, timeout=30) as response: + token = json.load(response)['auth']['client_token'] + if not isinstance(token, str) or not token: + raise ValueError('empty token') + sys.stdout.write(token) + + +if __name__ == '__main__': + try: + main() + except Exception: + # Bao 响应和异常对象可能带敏感内容,不写入 journal。 + sys.stderr.write('Bao certificate login failed\n') + sys.exit(1) diff --git a/infrastructure/etcd/controller/renew.py b/infrastructure/etcd/controller/renew.py new file mode 100644 index 0000000..9699d7c --- /dev/null +++ b/infrastructure/etcd/controller/renew.py @@ -0,0 +1,51 @@ +#!/usr/bin/python3 +"""每日一次,机器证书换短期 token;使用现有串行 Ansible 收敛,不常驻 agent。""" +import fcntl +import os +from pathlib import Path +import subprocess +import sys +import urllib.request + +ROOT = Path('/opt/homelab-etcd-controller') + + +def main(): + with open('/var/lib/homelab-etcd-controller/renew.lock', 'a') as lock: + try: + fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB) + except BlockingIOError: + return 0 + login = subprocess.run( + ['sudo', '-n', '/usr/bin/python3', str(ROOT / 'login.py')], + capture_output=True, text=True, timeout=40, + ) + if login.returncode or not login.stdout.strip(): + print('Bao 机器证书登录失败;保留现有证书与运行中的 etcd。', file=sys.stderr) + return 1 + token = login.stdout.strip() + env = dict(os.environ, BAO_TOKEN=token) + try: + for play in ['verify.yml', 'site.yml']: + result = subprocess.run( + ['/home/panxiao81/.local/bin/ansible-playbook', play], + cwd=ROOT / 'ansible', env=env, timeout=750, + ) + if result.returncode: + return result.returncode + Path('/var/lib/homelab-etcd-controller/last-success').touch() + return 0 + finally: + request = urllib.request.Request( + 'https://bao.ad.ddupan.top:8200/v1/auth/token/revoke-self', + data=b'{}', headers={'X-Vault-Token': token}, method='POST', + ) + try: + with urllib.request.urlopen(request, timeout=15): + pass + except Exception: + print('短期 token 撤销未确认,将由 TTL 自动失效。', file=sys.stderr) + + +if __name__ == '__main__': + raise SystemExit(main()) diff --git a/infrastructure/etcd/run.py b/infrastructure/etcd/run.py new file mode 100644 index 0000000..7ad6573 --- /dev/null +++ b/infrastructure/etcd/run.py @@ -0,0 +1,43 @@ +#!/usr/bin/env python3 +"""从当前 Bao 登录会话向子进程传递凭据,不写临时秘密文件或输出秘密。""" +import json +import os +from pathlib import Path +import subprocess +import sys + +ROOT = Path(__file__).resolve().parent + + +def main(): + if len(sys.argv) < 3 or sys.argv[1] not in ('terraform', 'ansible'): + raise SystemExit('用法:python3 run.py terraform | ansible [args]') + env = dict(os.environ) + env.setdefault('BAO_ADDR', 'https://bao.ad.ddupan.top:8200') + token = env.get('BAO_TOKEN') or env.get('VAULT_TOKEN') + if not token: + token = Path('~/.vault-token').expanduser().read_text().strip() + env['BAO_TOKEN'] = env['VAULT_TOKEN'] = token + if sys.argv[1] == 'terraform': + response = subprocess.run( + ['bao', 'kv', 'get', '-format=json', 'kv/k8s/seaweedfs-s3'], + env=env, capture_output=True, text=True, + ) + if response.returncode: + raise SystemExit('读取 tfstate 受限身份失败;请检查 Bao 登录和授权。') + config = json.loads(response.stdout)['data']['data']['seaweedfs_s3_config'] + config = json.loads(config) if isinstance(config, str) else config + identities = [i for i in config['identities'] if i['name'] == 'terraform'] + if len(identities) != 1 or len(identities[0]['credentials']) != 1: + raise SystemExit('tfstate 身份不唯一,拒绝猜测凭据。') + credential = identities[0]['credentials'][0] + env['AWS_ACCESS_KEY_ID'] = credential['accessKey'] + env['AWS_SECRET_ACCESS_KEY'] = credential['secretKey'] + command, cwd = ['terraform', *sys.argv[2:]], ROOT / 'terraform' + else: + command, cwd = ['ansible-playbook', *sys.argv[2:]], ROOT / 'ansible' + raise SystemExit(subprocess.run(command, cwd=cwd, env=env).returncode) + + +if __name__ == '__main__': + main() diff --git a/infrastructure/etcd/terraform/.terraform.lock.hcl b/infrastructure/etcd/terraform/.terraform.lock.hcl new file mode 100644 index 0000000..234d8fc --- /dev/null +++ b/infrastructure/etcd/terraform/.terraform.lock.hcl @@ -0,0 +1,22 @@ +# This file is maintained automatically by "terraform init". +# Manual edits may be lost in future updates. + +provider "registry.terraform.io/hashicorp/vault" { + version = "4.8.0" + constraints = "~> 4.0" + hashes = [ + "h1:aHqgWQhDBMeZO9iUKwJYMlh4q+xNMUlMIcjRbF4d02Y=", + "zh:269ab13433f67684012ae7e15876532b0312f5d0d2002a9cf9febb1279ce5ea6", + "zh:4babc95bf0c40eb85005db1dc2ca403c46be4a71dd3e409db3711a56f7a5ca0e", + "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", + "zh:86e27c1c625ecc24446a11eeffc3ac319b36c2b4e51251db8579256a0dbcf136", + "zh:a32f31da94824009e26b077374440b52098aecb93c92ff55dc3d31dd37c4ea25", + "zh:be0a18c6c0425518bab4fbffd82078b82036a88503b5d76064de551c9f646cbf", + "zh:be5a77fdfd36863ebeec79cd12b1d13322ffad6821d157a0b279789fa06b5937", + "zh:be8317d142a3caad74c7d936039ae27076a1b2b8312ef5208e2871a5f525977c", + "zh:c94a84895a3d9954b80e983eed4603330a5cdbbd8eef5b3c99278c2d1402ef3c", + "zh:de1fb712784dd8415f011ca5346a34f87fab6046c730557615247e511dbc7d98", + "zh:e3eafae7da550f86cae395d6660b2a0e93ec8d2b0e0e5ef982ec762e961fc952", + "zh:ff35fb1ab6add288f0f368981e56f780b50405accd1937131cba1137999c8d83", + ] +} diff --git a/infrastructure/etcd/terraform/backend.tf b/infrastructure/etcd/terraform/backend.tf new file mode 100644 index 0000000..9dd1cc3 --- /dev/null +++ b/infrastructure/etcd/terraform/backend.tf @@ -0,0 +1,15 @@ +# 与既有服务复用受限 tfstate 身份,使用独立对象与原生锁;不复用 Bao 的 state。 +terraform { + backend "s3" { + bucket = "tfstate" + key = "etcd/terraform.tfstate" + endpoints = { s3 = "https://s3.ad.ddupan.top" } + region = "us-east-1" + use_path_style = true + skip_credentials_validation = true + skip_metadata_api_check = true + skip_region_validation = true + skip_requesting_account_id = true + use_lockfile = true + } +} diff --git a/infrastructure/etcd/terraform/main.tf b/infrastructure/etcd/terraform/main.tf new file mode 100644 index 0000000..e5341cb --- /dev/null +++ b/infrastructure/etcd/terraform/main.tf @@ -0,0 +1,93 @@ +terraform { + required_version = ">= 1.10" + required_providers { + vault = { + source = "hashicorp/vault" + version = "~> 4.0" + } + } +} + +provider "vault" { + address = var.bao_address +} + +variable "bao_address" { + type = string + default = "https://bao.ad.ddupan.top:8200" +} +variable "pki_mount" { + type = string + default = "pki" +} +variable "kv_mount" { + type = string + default = "kv" +} +variable "member_names" { + type = list(string) + default = ["etcd-laptop", "etcd-pve1", "etcd-pve2"] +} +variable "consumer_names" { + type = set(string) + default = ["patroni-pg-prod"] +} + +# 仅管理既有 PKI 下的新 role/policy,不纳管 CA 私钥、mount 或秘密值。 +locals { + certificate_roles = { + server = { names = var.member_names, server = true, client = false, ips = true } + peer = { names = concat(var.member_names, ["homelab-etcd-peer"]), server = true, client = true, ips = true } + admin = { names = ["root"], server = false, client = true, ips = false } + gateway = { names = var.member_names, server = false, client = true, ips = false } + client = { names = [for name in var.consumer_names : "etcd-${name}"], server = false, client = true, ips = false } + } +} +resource "vault_pki_secret_backend_role" "etcd" { + for_each = local.certificate_roles + backend = var.pki_mount + name = "homelab-etcd-${each.key}" + allowed_domains = each.value.names + allow_bare_domains = true + allow_subdomains = false + allow_glob_domains = false + allow_any_name = false + allow_localhost = false + allow_wildcard_certificates = false + allow_ip_sans = each.value.ips + server_flag = each.value.server + client_flag = each.value.client + key_type = "ec" + key_bits = 256 + ttl = 5184000 + max_ttl = 5184000 + require_cn = !contains(["gateway", "client"], each.key) + use_csr_common_name = true + use_csr_sans = true +} + +resource "vault_policy" "etcd_provisioner" { + name = "homelab-etcd-provisioner" + policy = <<-EOT + path "${var.pki_mount}/sign/homelab-etcd-*" { + capabilities = ["update"] + } + path "${var.kv_mount}/metadata/infra/etcd/consumers/*" { + capabilities = ["read"] + } + path "${var.kv_mount}/data/infra/etcd/consumers/*" { + capabilities = ["create", "read", "update"] + } + EOT +} + +# 身份绑定沿用既有控制端认证方式,创建 policy 不自动授权任何身份。 +resource "vault_policy" "etcd_consumer" { + for_each = var.consumer_names + name = "homelab-etcd-${each.key}" + policy = <<-EOT + path "${var.kv_mount}/data/infra/etcd/consumers/${each.key}" { + capabilities = ["read"] + } + EOT +} diff --git a/infrastructure/etcd/terraform/renewal.tf b/infrastructure/etcd/terraform/renewal.tf new file mode 100644 index 0000000..1fcc22f --- /dev/null +++ b/infrastructure/etcd/terraform/renewal.tf @@ -0,0 +1,28 @@ +# 独立挂载:不接管全局认证,不保存长期 token,也不授权 KV 消费者秘密读取。 +data "vault_generic_secret" "etcd_ca" { + path = "${var.pki_mount}/cert/ca" +} + +resource "vault_auth_backend" "etcd_renewal" { + type = "cert" + path = "homelab-etcd-renewal" +} + +resource "vault_policy" "etcd_renewal" { + name = "homelab-etcd-renewal" + policy = join("\n", concat([ + for kind in ["server", "peer", "admin", "gateway"] : + "path \"${var.pki_mount}/sign/homelab-etcd-${kind}\" { capabilities = [\"update\"] }" + ], ["path \"auth/token/revoke-self\" { capabilities = [\"update\"] }"])) +} + +resource "vault_cert_auth_backend_role" "etcd_renewal" { + backend = vault_auth_backend.etcd_renewal.path + name = "etcd-laptop" + certificate = data.vault_generic_secret.etcd_ca.data["certificate"] + allowed_dns_sans = ["etcd-laptop"] + token_policies = [vault_policy.etcd_renewal.name] + token_no_default_policy = true + token_ttl = 600 + token_max_ttl = 900 +} diff --git a/infrastructure/etcd/tests/integration.py b/infrastructure/etcd/tests/integration.py new file mode 100644 index 0000000..972e542 --- /dev/null +++ b/infrastructure/etcd/tests/integration.py @@ -0,0 +1,233 @@ +#!/usr/bin/env python3 +"""临时三成员 mTLS/RBAC 集成测试;仅绑定 loopback,不访问生产 Bao。""" +import base64 +import json +import os +from pathlib import Path +import re +import shutil +import ssl +import subprocess +import tempfile +import threading +import time +import urllib.error +import urllib.request +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer + +import jinja2 +import yaml + +ROOT = Path(__file__).resolve().parents[1] +BIN = Path(os.environ.get('ETCD_TEST_BIN', '/tmp/etcd-v3.7.2-linux-amd64')) + + +def run(argv, **kwargs): + return subprocess.run([str(x) for x in argv], capture_output=True, text=True, check=True, **kwargs) + + +class FakeBao(BaseHTTPRequestHandler): + records = {} + versions = {} + writes = 0 + deny = False + + def log_message(self, *_): + pass + + def respond(self, status, body): + self.send_response(status) + self.send_header('Content-Type', 'application/json') + self.end_headers() + self.wfile.write(json.dumps(body).encode()) + + def do_GET(self): + if self.deny: + return self.respond(403, {'errors': ['permission denied']}) + name = self.path.rsplit('/', 1)[-1] + if '/metadata/' in self.path and name in self.versions: + return self.respond(200, {'data': {'current_version': self.versions[name]}}) + if '/data/' in self.path and name in self.records: + return self.respond(200, {'data': {'data': self.records[name]}}) + self.respond(404, {'errors': []}) + + def do_POST(self): + body = json.loads(self.rfile.read(int(self.headers['Content-Length']))) + name = self.path.rsplit('/', 1)[-1] + if body['options']['cas'] != 0 or name in self.versions: + return self.respond(400, {'errors': ['CAS mismatch']}) + self.records[name] = body['data'] + self.versions[name] = 1 + type(self).writes += 1 + self.respond(200, {'data': {'version': 1}}) + + +def main(): + for key in ('HTTP_PROXY', 'HTTPS_PROXY', 'ALL_PROXY', 'http_proxy', 'https_proxy', 'all_proxy'): + os.environ.pop(key, None) + os.environ['NO_PROXY'] = '*' + processes = [] + handles = [] + server = None + with tempfile.TemporaryDirectory(prefix='shared-etcd-test-') as work: + w = Path(work) + w.chmod(0o700) + try: + run(['openssl', 'req', '-x509', '-newkey', 'rsa:2048', '-nodes', '-keyout', w/'ca.key', + '-out', w/'ca.crt', '-days', '1', '-subj', '/CN=isolated-test-ca']) + + def cert(name, cn, eku, ip=None): + run(['openssl', 'req', '-new', '-newkey', 'rsa:2048', '-nodes', '-keyout', w/f'{name}.key', + '-out', w/f'{name}.csr', '-subj', f'/CN={cn}' if cn else '/']) + ext = w/f'{name}.ext' + ext.write_text(f'extendedKeyUsage={eku}\n' + (f'subjectAltName=IP:{ip},IP:127.0.0.1\n' if ip else '')) + run(['openssl', 'x509', '-req', '-in', w/f'{name}.csr', '-CA', w/'ca.crt', + '-CAkey', w/'ca.key', '-CAcreateserial', '-out', w/f'{name}.crt', '-days', '1', '-extfile', ext]) + + cert('admin', 'root', 'clientAuth') + cert('client', '', 'clientAuth') + cert('gateway', '', 'clientAuth') + env = jinja2.Environment(loader=jinja2.FileSystemLoader(ROOT/'ansible/roles/shared_etcd/templates'), undefined=jinja2.StrictUndefined) + env.filters['to_json'] = json.dumps + defaults = yaml.safe_load((ROOT/'ansible/roles/shared_etcd/defaults/main.yml').read_text()) + names = ['test1', 'test2', 'test3'] + hosts = {name: {'etcd_address': f'127.0.0.{i+2}'} for i, name in enumerate(names)} + for name in names: + d = w/name + d.mkdir() + for kind, cn, eku in [('server', name, 'serverAuth'), ('peer', 'homelab-etcd-peer', 'serverAuth,clientAuth')]: + cert(f'{name}-{kind}', cn, eku, hosts[name]['etcd_address']) + for suffix in ['crt', 'key']: + shutil.copy(w/f'{name}-{kind}.{suffix}', d/f'{kind}.{suffix}') + shutil.copy(w/'ca.crt', d/'ca.crt') + for suffix in ['crt', 'key']: + shutil.copy(w/f'gateway.{suffix}', d/f'gateway.{suffix}') + values = dict(defaults, inventory_hostname=name, groups={'etcd': names}, hostvars=hosts, + etcd_address=hosts[name]['etcd_address'], etcd_config_dir=str(d), etcd_data_dir=str(d/'data'), + etcd_client_port=22379, etcd_peer_port=22380, etcd_metrics_port=0) + config = env.get_template('etcd.yml.j2').render(**values) + # 三成员共享进程命名空间,metrics 用独立 loopback IP。 + config = config.replace('http://127.0.0.1:0', f"http://{hosts[name]['etcd_address']}:22381") + (d/'config.yml').write_text(config) + handle = (d/'etcd.log').open('w') + handles.append(handle) + processes.append(subprocess.Popen([str(BIN/'etcd'), '--config-file='+str(d/'config.yml')], stdout=handle, stderr=handle)) + + base_env = dict(os.environ, ETCDCTL_ENDPOINTS='https://127.0.0.2:22379', ETCDCTL_CACERT=str(w/'ca.crt'), + ETCDCTL_CERT=str(w/'admin.crt'), ETCDCTL_KEY=str(w/'admin.key'), ETCDCTL_DIAL_TIMEOUT='2s', ETCDCTL_COMMAND_TIMEOUT='3s') + + def ctl(*args, input=None, env=None): + return run([BIN/'etcdctl', *args], input=input, env=env or base_env) + + for attempt in range(5): + try: + ctl('endpoint', 'health') + break + except subprocess.CalledProcessError: + if any(p.poll() is not None for p in processes): + + for name in names: + print((w/name/'etcd.log').read_text()[-4000:]) + raise RuntimeError('test etcd exited during startup') + time.sleep(0.2) + else: + + print((w/'test1'/'etcd.log').read_text()[-5000:]) + raise RuntimeError('test quorum did not form') + server = ThreadingHTTPServer(('127.0.0.1', 0), FakeBao) + threading.Thread(target=server.serve_forever, daemon=True).start() + inventory = w/'hosts.yml' + inventory.write_text(yaml.safe_dump({'all': {'children': {'etcd': {'hosts': {'test1': {'ansible_connection': 'local'}}}}}})) + extra = { + 'ansible_become': False, 'etcd_address': '127.0.0.2', 'etcd_client_port': 22379, + 'etcd_install_dir': str(BIN), 'etcd_config_dir': str(w), 'etcd_bootstrap_auth': True, + 'etcd_bao_url': f'http://127.0.0.1:{server.server_port}', 'etcd_bao_token': 'isolated-test-token', + 'etcd_bao_kv_mount': 'kv', 'etcd_bao_secret_base': 'infra/etcd/consumers', + 'etcd_consumers': [{'name': 'patroni-pg-prod', 'prefix': '/homelab/patroni/pg-prod/'}], + } + (w/'extra.json').write_text(json.dumps(extra)) + ansible_env = dict(os.environ, ANSIBLE_LOCAL_TEMP=str(w/'ansible-tmp'), ANSIBLE_NOCOLOR='1', + ANSIBLE_ROLES_PATH=str(ROOT/'ansible/roles')) + + def play(name, fail=False): + r = subprocess.run(['ansible-playbook', '-i', str(inventory), str(ROOT/'ansible'/name), '-e', '@'+str(w/'extra.json')], + capture_output=True, text=True, env=ansible_env) + (w/(name+'.log')).write_text(r.stdout+r.stderr) + if fail: + assert r.returncode != 0, 'expected a fail-closed playbook error' + elif r.returncode: + # Tasks use no_log for secret data; retain useful task/line diagnostic. + print(r.stdout[-5000:]);print(r.stderr[-2000:]) + raise RuntimeError(name+' failed') + return r.stdout + + play('bootstrap-auth.yml') + assert re.search(r'changed=0\s', play('bootstrap-auth.yml')) + play('consumers.yml') + secret = FakeBao.records['patroni-pg-prod']['password'] + assert len(secret) == 48 + assert re.search(r'changed=0\s', play('consumers.yml')) + assert FakeBao.writes == 1 + role = json.loads(ctl('--write-out=json', 'role', 'get', 'patroni-pg-prod').stdout) + assert role['perm'][0]['key'] == base64.b64encode(b'/homelab/patroni/pg-prod/').decode() + client_env = dict(base_env, ETCDCTL_CERT=str(w/'client.crt'), ETCDCTL_KEY=str(w/'client.key'), + ETCDCTL_USER='patroni-pg-prod', ETCDCTL_PASSWORD=secret) + ctl('put', '/homelab/patroni/pg-prod/test', 'ok', env=client_env) + try: + ctl('put', '/homelab/other/test', 'denied', env=client_env) + raise AssertionError('cross-prefix write succeeded') + except subprocess.CalledProcessError: + pass + # 模拟 Patroni 的 gateway 协议,证明 mTLS + username/password 可组合使用。 + ctx = ssl.create_default_context(cafile=str(w/'ca.crt')) + ctx.load_cert_chain(w/'client.crt', w/'client.key') + body = json.dumps({'name':'patroni-pg-prod', 'password':secret}).encode() + req = urllib.request.Request('https://127.0.0.2:22379/v3/auth/authenticate', data=body, headers={'Content-Type':'application/json'}) + with urllib.request.urlopen(req, context=ctx) as response: + token = json.load(response)['token'] + gateway_body = json.dumps({'key': base64.b64encode(b'/homelab/patroni/pg-prod/gateway').decode(), + 'value': base64.b64encode(b'ok').decode()}).encode() + req = urllib.request.Request('https://127.0.0.2:22379/v3/kv/put', data=gateway_body, + headers={'Content-Type': 'application/json', 'Authorization': token}) + with urllib.request.urlopen(req, context=ctx) as response: + assert response.status == 200 + # 对已有用户,秘密值被意外覆盖也必须失败,而不是悄悄改 etcd 密码。 + FakeBao.records['patroni-pg-prod']['password'] = 'x' * 48 + play('consumers.yml', fail=True) + FakeBao.records['patroni-pg-prod']['password'] = secret + FakeBao.deny = True + play('consumers.yml', fail=True) + FakeBao.deny = False + saved = FakeBao.records.pop('patroni-pg-prod') + play('consumers.yml', fail=True) # metadata 存在、data 已删除 + FakeBao.versions.clear() + play('consumers.yml', fail=True) # 用户存在但 metadata 丢失 + assert FakeBao.writes == 1 + FakeBao.records['patroni-pg-prod'] = saved + FakeBao.versions['patroni-pg-prod'] = 1 + ctl('snapshot', 'save', str(w/'snapshot.db')) + snapshot = json.loads(run([BIN/'etcdutl', '--write-out=json', 'snapshot', 'status', w/'snapshot.db']).stdout) + assert snapshot['totalKey'] > 0 + run([BIN/'etcdutl', 'snapshot', 'restore', w/'snapshot.db', '--data-dir='+str(w/'restored')]) + rss = [] + for process in processes: + match = re.search(r'^VmRSS:\s+(\d+)', Path(f'/proc/{process.pid}/status').read_text(), re.M) + rss.append(round(int(match[1]) / 1024, 1)) + processes[2].terminate();processes[2].wait(timeout=10) + ctl('put', '/homelab/patroni/pg-prod/after-member-loss', 'ok', env=client_env) + print('PASS: three-member mTLS, auth bootstrap/idempotence, Bao create-once/fail-closed, prefix isolation, gateway auth, snapshot/restore, one-member loss') + print('Idle test member RSS MiB (not a production capacity result):', rss) + finally: + if server: + server.shutdown();server.server_close() + for process in processes: + if process.poll() is None: + process.terminate() + try: process.wait(timeout=10) + except subprocess.TimeoutExpired: process.kill();process.wait() + for handle in handles: + handle.close() + + +if __name__ == '__main__': + main() diff --git a/infrastructure/etcd/tests/test_renewal.py b/infrastructure/etcd/tests/test_renewal.py new file mode 100644 index 0000000..bc60faf --- /dev/null +++ b/infrastructure/etcd/tests/test_renewal.py @@ -0,0 +1,71 @@ +"""续签调度失败关闭测试;不会连接真实 Bao 或运行 Ansible。""" +import importlib.util +import subprocess +import tempfile +import unittest +from pathlib import Path +from unittest.mock import patch, Mock + +spec = importlib.util.spec_from_file_location('renew', Path(__file__).parents[1] / 'controller/renew.py') +renew = importlib.util.module_from_spec(spec) +spec.loader.exec_module(renew) + + +class RenewalTests(unittest.TestCase): + def execute(self, results): + with tempfile.TemporaryDirectory() as directory: + lock = Path(directory) / 'lock' + success = Path(directory) / 'last-success' + original_open = open + response = Mock() + response.__enter__ = Mock(return_value=response) + response.__exit__ = Mock(return_value=False) + with patch('builtins.open', side_effect=lambda *_: original_open(lock, 'a')), \ + patch.object(renew.subprocess, 'run', side_effect=results) as run, \ + patch.object(renew.urllib.request, 'urlopen', return_value=response) as revoke, \ + patch.object(renew, 'Path', return_value=success): + rc = renew.main() + return rc, run.call_args_list, revoke.call_count, success.exists() + + def test_login_failure_never_changes_members(self): + rc, calls, revokes, success = self.execute([subprocess.CompletedProcess([], 1, '', '')]) + self.assertEqual(rc, 1) + self.assertEqual(len(calls), 1) + self.assertEqual(revokes, 0) + self.assertFalse(success) + + def test_unhealthy_cluster_never_runs_site(self): + rc, calls, revokes, success = self.execute([ + subprocess.CompletedProcess([], 0, 'test-only-token'), + subprocess.CompletedProcess([], 2), + ]) + self.assertEqual(rc, 2) + self.assertEqual(calls[1].args[0][-1], 'verify.yml') + self.assertEqual(len(calls), 2) + self.assertEqual(revokes, 1) + self.assertFalse(success) + + def test_success_checks_then_converges_and_revokes(self): + rc, calls, revokes, success = self.execute([ + subprocess.CompletedProcess([], 0, 'test-only-token'), + subprocess.CompletedProcess([], 0), + subprocess.CompletedProcess([], 0), + ]) + self.assertEqual(rc, 0) + self.assertEqual([c.args[0][-1] for c in calls[1:]], ['verify.yml', 'site.yml']) + self.assertEqual(revokes, 1) + self.assertTrue(success) + + def test_site_failure_is_not_success(self): + rc, calls, revokes, success = self.execute([ + subprocess.CompletedProcess([], 0, 'test-only-token'), + subprocess.CompletedProcess([], 0), + subprocess.CompletedProcess([], 2), + ]) + self.assertEqual(rc, 2) + self.assertEqual(revokes, 1) + self.assertFalse(success) + + +if __name__ == '__main__': + unittest.main() diff --git a/infrastructure/shared-postgresql/.gitignore b/infrastructure/shared-postgresql/.gitignore new file mode 100644 index 0000000..6326aa5 --- /dev/null +++ b/infrastructure/shared-postgresql/.gitignore @@ -0,0 +1,8 @@ +.terraform/ +*.tfstate* +*.tfplan +*.tfvars +__pycache__/ + +# 此文件是从 Bao 读取凭据的 playbook,不包含秘密值。 +!ansible/credentials.yml diff --git a/infrastructure/shared-postgresql/ansible/ansible.cfg b/infrastructure/shared-postgresql/ansible/ansible.cfg new file mode 100644 index 0000000..634674c --- /dev/null +++ b/infrastructure/shared-postgresql/ansible/ansible.cfg @@ -0,0 +1,6 @@ +[defaults] +inventory = inventory/hosts.yml +roles_path = roles +host_key_checking = True +retry_files_enabled = False +interpreter_python = auto_silent diff --git a/infrastructure/shared-postgresql/ansible/ayatori-credentials.yml b/infrastructure/shared-postgresql/ansible/ayatori-credentials.yml new file mode 100644 index 0000000..99ba604 --- /dev/null +++ b/infrastructure/shared-postgresql/ansible/ayatori-credentials.yml @@ -0,0 +1,13 @@ +--- +- name: 读取现有实例管理凭据 + ansible.builtin.import_playbook: credentials.yml +- name: 将 Ayatori 身份交付到独立路径(不携带 superuser 凭据) + hosts: localhost + connection: local + gather_facts: false + tasks: + - name: 逐实例 CAS 创建并核对,只接受一致的已有值 + ansible.builtin.include_tasks: tasks/ayatori-credential.yml + loop: [prod, dev] + loop_control: + loop_var: pg_ayatori_instance diff --git a/infrastructure/shared-postgresql/ansible/backup.yml b/infrastructure/shared-postgresql/ansible/backup.yml new file mode 100644 index 0000000..4e9b27a --- /dev/null +++ b/infrastructure/shared-postgresql/ansible/backup.yml @@ -0,0 +1,160 @@ +--- +- name: 配置独立备份仓库账号与工具 + hosts: pg_repository + become: true + gather_facts: false + any_errors_fatal: true + tasks: + - name: 必须先挂载专属数据盘 + ansible.builtin.command: + argv: [findmnt, --mountpoint, "{{ pg_repo_path }}", --noheadings] + changed_when: false + check_mode: false + - name: 配置与数据库相同的软件源 + ansible.builtin.import_role: + name: pg_packages + tasks_from: repository + - name: 安装备份工具(不安装数据库) + ansible.builtin.apt: + name: [pgbackrest, openssh-client] + state: present + install_recommends: false + - name: 创建仓库组 + ansible.builtin.group: + name: pgbackup + system: true + - name: 创建仓库账号 + ansible.builtin.user: + name: pgbackup + group: pgbackup + home: "{{ pg_repo_path }}" + create_home: false + system: true + shell: /bin/bash + - name: 设置仓库目录权限 + ansible.builtin.file: + path: "{{ pg_repo_path }}" + state: directory + owner: pgbackup + group: pgbackup + mode: '0700' + +- name: 建立专用 SSH 传输身份 + hosts: pg_hosts:pg_repository + become: true + gather_facts: false + any_errors_fatal: true + vars: + pg_transport_user: "{{ 'pgbackup' if inventory_hostname in groups['pg_repository'] else 'pgprod' }}" + pg_transport_home: "{{ pg_repo_path if inventory_hostname in groups['pg_repository'] else pg_data_root ~ '/prod' }}" + tasks: + - name: 创建 SSH 目录 + ansible.builtin.file: + path: "{{ pg_transport_home }}/.ssh" + state: directory + owner: "{{ pg_transport_user }}" + group: "{{ pg_transport_user }}" + mode: '0700' + # community.crypto 的跨文件系统 preserved_copy 会把 ext4 属性带到 ZFS,产生 chattr 错误。 + - name: 仅在缺失时原地生成专属传输私钥 + ansible.builtin.command: + argv: [ssh-keygen, -q, -t, ed25519, -N, '', -f, "{{ pg_transport_home }}/.ssh/id_ed25519"] + creates: "{{ pg_transport_home }}/.ssh/id_ed25519" + - name: 收敛私钥属主与权限 + ansible.builtin.file: + path: "{{ pg_transport_home }}/.ssh/id_ed25519" + owner: "{{ pg_transport_user }}" + group: "{{ pg_transport_user }}" + mode: '0600' + - name: 从既有私钥派生公开部分,不重新生成密钥 + ansible.builtin.command: + argv: [ssh-keygen, -y, -f, "{{ pg_transport_home }}/.ssh/id_ed25519"] + register: pg_transport_key + changed_when: false + - name: 保存对应公钥 + ansible.builtin.copy: + content: "{{ pg_transport_key.stdout }}\n" + dest: "{{ pg_transport_home }}/.ssh/id_ed25519.pub" + owner: "{{ pg_transport_user }}" + group: "{{ pg_transport_user }}" + mode: '0644' + - name: 经已信任 Ansible 通道读取 SSH 主机公钥 + ansible.builtin.slurp: + src: /etc/ssh/ssh_host_ed25519_key.pub + register: pg_transport_host_key + +- name: 配置专用传输授权(不允许转发和 PTY) + hosts: pg_hosts:pg_repository + become: true + gather_facts: false + any_errors_fatal: true + vars: + pg_transport_user: "{{ 'pgbackup' if inventory_hostname in groups['pg_repository'] else 'pgprod' }}" + pg_transport_home: "{{ pg_repo_path if inventory_hostname in groups['pg_repository'] else pg_data_root ~ '/prod' }}" + pg_transport_peers: >- + {{ groups['pg_hosts'] if inventory_hostname in groups['pg_repository'] else groups['pg_repository'] }} + tasks: + - name: 限定专用账号的对端密钥 + ansible.builtin.copy: + content: | + {% for peer in pg_transport_peers %} + restrict {{ hostvars[peer].pg_transport_key.stdout }} + {% endfor %} + dest: "{{ pg_transport_home }}/.ssh/authorized_keys" + owner: "{{ pg_transport_user }}" + group: "{{ pg_transport_user }}" + mode: '0600' + - name: 保持专用 SSH 信任文件只由 IaC 管理 + ansible.builtin.copy: + content: | + Host {{ pg_transport_peers | map('extract', hostvars, 'ansible_host') | join(' ') }} + StrictHostKeyChecking yes + UpdateHostKeys no + IdentitiesOnly yes + IdentityFile {{ pg_transport_home }}/.ssh/id_ed25519 + UserKnownHostsFile {{ pg_transport_home }}/.ssh/known_hosts + dest: "{{ pg_transport_home }}/.ssh/config" + owner: "{{ pg_transport_user }}" + group: "{{ pg_transport_user }}" + mode: '0600' + - name: 固定对端 SSH 主机身份 + ansible.builtin.copy: + content: | + {% for peer in pg_transport_peers %} + {{ hostvars[peer].ansible_host }} {{ hostvars[peer].pg_transport_host_key.content | b64decode | trim }} + {% endfor %} + dest: "{{ pg_transport_home }}/.ssh/known_hosts" + owner: "{{ pg_transport_user }}" + group: "{{ pg_transport_user }}" + mode: '0600' + +- name: 配置主库感知的仓库与每日备份任务(不提前启用) + hosts: pg_repository + become: true + gather_facts: false + any_errors_fatal: true + vars: + pg_backup_members: + - {address: '192.168.10.127'} + - {address: '10.60.0.20'} + tasks: + - name: 创建配置目录 + ansible.builtin.file: + path: /etc/homelab-pgbackrest + state: directory + owner: root + group: pgbackup + mode: '0750' + - name: 安装仓库配置 + ansible.builtin.template: + src: roles/pg_instance/templates/repository.conf.j2 + dest: /etc/homelab-pgbackrest/repository.conf + owner: root + group: pgbackup + mode: '0640' + - name: 安装备份服务和 timer + ansible.builtin.template: + src: "roles/pg_instance/templates/backup.{{ item }}.j2" + dest: "/etc/systemd/system/homelab-postgresql-backup.{{ item }}" + mode: '0644' + loop: [service, timer] diff --git a/infrastructure/shared-postgresql/ansible/bootstrap-backup.yml b/infrastructure/shared-postgresql/ansible/bootstrap-backup.yml new file mode 100644 index 0000000..e53f497 --- /dev/null +++ b/infrastructure/shared-postgresql/ansible/bootstrap-backup.yml @@ -0,0 +1,40 @@ +--- +- name: 在已启动数据库上验证首备后启用调度 + hosts: pg_repository + become: true + become_user: pgbackup + gather_facts: false + vars: + pg_backup_command: [pgbackrest, '--config=/etc/homelab-pgbackrest/repository.conf', '--stanza=prod'] + tasks: + - name: 检查 stanza 元数据 + ansible.builtin.stat: + path: "{{ pg_repo_path }}/backup/prod/backup.info" + register: pg_stanza + - name: 首次初始化 stanza + ansible.builtin.command: + argv: "{{ pg_backup_command + ['stanza-create'] }}" + when: not pg_stanza.stat.exists + changed_when: true + - name: 验证真实 WAL 归档(会触发 WAL 切换) + ansible.builtin.command: + argv: "{{ pg_backup_command + ['check'] }}" + changed_when: false + - name: 查询已有备份 + ansible.builtin.command: + argv: "{{ pg_backup_command + ['--output=json', 'info'] }}" + register: pg_backup_info + changed_when: false + - name: 没有成功备份时创建首个 full + ansible.builtin.command: + argv: "{{ pg_backup_command + ['--type=full', 'backup'] }}" + when: (pg_backup_info.stdout | from_json)[0].backup | length == 0 + changed_when: true + - name: 验证成功后才启用 timer + ansible.builtin.systemd_service: + name: homelab-postgresql-backup.timer + daemon_reload: true + enabled: true + state: started + become_user: root + become: true diff --git a/infrastructure/shared-postgresql/ansible/bootstrap.yml b/infrastructure/shared-postgresql/ansible/bootstrap.yml new file mode 100644 index 0000000..47551e7 --- /dev/null +++ b/infrastructure/shared-postgresql/ansible/bootstrap.yml @@ -0,0 +1,20 @@ +--- +- name: 读取已有凭据 + ansible.builtin.import_playbook: credentials.yml +- name: 检查承载和所有权 + ansible.builtin.import_playbook: preflight.yml +- name: 显式初始化新实例,laptop 先启动 + hosts: pg_hosts + become: true + serial: 1 + any_errors_fatal: true + tasks: + - name: 要求显式首次初始化开关 + ansible.builtin.assert: + that: pg_allow_initialize | bool + fail_msg: 仅首次初始化使用 -e pg_allow_initialize=true;恢复和升级不走此入口。 + - name: 初始化声明的实例 + ansible.builtin.include_tasks: tasks/bootstrap-instance.yml + loop: "{{ pg_instances }}" + loop_control: + loop_var: pg_instance diff --git a/infrastructure/shared-postgresql/ansible/controller.yml b/infrastructure/shared-postgresql/ansible/controller.yml new file mode 100644 index 0000000..e993f05 --- /dev/null +++ b/infrastructure/shared-postgresql/ansible/controller.yml @@ -0,0 +1,70 @@ +--- +- name: 安装每日证书续签控制任务 + hosts: pg-laptop + become: true + gather_facts: false + tasks: + - name: 创建 root 管理的程序目录 + ansible.builtin.file: + path: /opt/homelab-postgresql-controller + state: directory + mode: '0755' + - name: 安装短期身份包装器 + ansible.builtin.copy: + src: ../run.py + dest: /opt/homelab-postgresql-controller/run.py + owner: root + group: root + mode: '0755' + - name: 安装固定 Ansible 配置副本 + ansible.builtin.copy: + src: "{{ playbook_dir }}/" + dest: /opt/homelab-postgresql-controller/ansible/ + owner: root + group: root + mode: preserve + - name: 使用 SPIFFE 执行一次续签预检(不输出 token) + ansible.builtin.command: + argv: [python3, /opt/homelab-postgresql-controller/run.py, --spiffe, ansible, renew-certificates.yml] + environment: + PG_BAO_SPIFFE_ROLE: homelab-postgresql + PATH: /home/panxiao81/.local/bin:/usr/local/bin:/usr/bin:/bin + become: true + become_user: panxiao81 + changed_when: false + - name: 安装续签 service + ansible.builtin.copy: + dest: /etc/systemd/system/homelab-postgresql-renew.service + mode: '0644' + content: | + [Unit] + Description=Renew shared PostgreSQL certificates using short-lived SPIFFE identity + After=network-online.target + [Service] + Type=oneshot + User=panxiao81 + Environment=HOME=/home/panxiao81 + Environment=PATH=/home/panxiao81/.local/bin:/usr/local/bin:/usr/bin:/bin + Environment=PG_BAO_SPIFFE_ROLE=homelab-postgresql + ExecStart=/usr/bin/python3 /opt/homelab-postgresql-controller/run.py --spiffe ansible renew-certificates.yml + TimeoutStartSec=14min + UMask=0077 + - name: 安装每日 timer + ansible.builtin.copy: + dest: /etc/systemd/system/homelab-postgresql-renew.timer + mode: '0644' + content: | + [Unit] + Description=Daily shared PostgreSQL certificate renewal + [Timer] + OnCalendar=*-*-* 04:30:00 UTC + RandomizedDelaySec=15min + Persistent=true + [Install] + WantedBy=timers.target + - name: 预检通过后启用调度 + ansible.builtin.systemd_service: + name: homelab-postgresql-renew.timer + daemon_reload: true + enabled: true + state: started diff --git a/infrastructure/shared-postgresql/ansible/credentials.yml b/infrastructure/shared-postgresql/ansible/credentials.yml new file mode 100644 index 0000000..8075e9b --- /dev/null +++ b/infrastructure/shared-postgresql/ansible/credentials.yml @@ -0,0 +1,59 @@ +--- +# 只读收敛入口;秘密首次创建、删除或轮换不放进 site.yml。 +- name: 在任何主机变更前读取并校验已有凭据 + hosts: localhost + connection: local + gather_facts: false + tasks: + - name: 要求有效的显式 Bao 身份 + ansible.builtin.assert: + that: + - pg_bao_token | length > 0 + - pg_bao_url is match('^https://') + no_log: true + - name: 读取两个实例的管理凭据 + ansible.builtin.uri: + url: "{{ pg_bao_url }}/v1/kv/data/{{ pg_secret_paths[item] }}" + headers: + X-Vault-Token: "{{ pg_bao_token }}" + status_code: 200 + loop: [prod, dev] + register: pg_secret_response + no_log: true + - name: 整理并验证凭据结构 + ansible.builtin.set_fact: + pg_loaded_credentials: "{{ pg_loaded_credentials | default({}) | combine({item.item: item.json.data.data}) }}" + loop: "{{ pg_secret_response.results }}" + no_log: true + - name: 拒绝缺失或短密码 + ansible.builtin.assert: + that: + - pg_loaded_credentials[item].superuser_password | length >= 32 + - pg_loaded_credentials[item].ayatori_password | length >= 32 + - pg_loaded_credentials[item].ayatori_username == 'ayatori' + loop: [prod, dev] + no_log: true + - name: 校验生产复制与 API 凭据 + ansible.builtin.assert: + that: + - pg_loaded_credentials.prod.replication_password | length >= 32 + - pg_loaded_credentials.prod.rest_password | length >= 32 + no_log: true + - name: 读取已有 etcd 消费者凭据 + ansible.builtin.uri: + url: "{{ pg_bao_url }}/v1/kv/data/{{ pg_etcd_secret_path }}" + headers: + X-Vault-Token: "{{ pg_bao_token }}" + status_code: 200 + register: pg_etcd_response + no_log: true + - name: 校验 etcd prefix 并交付内存引用 + ansible.builtin.assert: + that: + - pg_etcd_response.json.data.data.username == 'patroni-pg-prod' + - pg_etcd_response.json.data.data.prefix == '/homelab/patroni/pg-prod/' + no_log: true + - name: 保存短生命周期内存引用 + ansible.builtin.set_fact: + pg_loaded_etcd: "{{ pg_etcd_response.json.data.data }}" + no_log: true diff --git a/infrastructure/shared-postgresql/ansible/initialize-secrets.yml b/infrastructure/shared-postgresql/ansible/initialize-secrets.yml new file mode 100644 index 0000000..d47c941 --- /dev/null +++ b/infrastructure/shared-postgresql/ansible/initialize-secrets.yml @@ -0,0 +1,31 @@ +--- +# 首次创建专用密码;已存在的数据与丢失秘密不能组合为自动重置。 +- name: 检查已有实例数据 + hosts: pg_hosts + become: true + gather_facts: false + tasks: + - name: 检查各实例 PG_VERSION + ansible.builtin.stat: + path: "{{ pg_data_root }}/{{ item }}/data/PG_VERSION" + loop: "{{ pg_instances }}" + register: pg_secret_data_files + - name: 记录现有数据边界 + ansible.builtin.set_fact: + pg_has_data: "{{ pg_secret_data_files.results | selectattr('stat.exists') | list | length > 0 }}" +- name: 首次将随机实例凭据写入 Bao + hosts: localhost + connection: local + gather_facts: false + tasks: + - name: 要求显式初始化与管理身份 + ansible.builtin.assert: + that: + - pg_allow_initialize | bool + - pg_bao_token | length > 0 + no_log: true + - name: 逐实例确认元数据和 CAS 创建 + ansible.builtin.include_tasks: tasks/initialize-secret.yml + loop: [prod, dev] + loop_control: + loop_var: pg_secret_instance diff --git a/infrastructure/shared-postgresql/ansible/inventory/group_vars/all.yml b/infrastructure/shared-postgresql/ansible/inventory/group_vars/all.yml new file mode 100644 index 0000000..b760ebd --- /dev/null +++ b/infrastructure/shared-postgresql/ansible/inventory/group_vars/all.yml @@ -0,0 +1,40 @@ +--- +pg_major: 18 +pg_patroni_version: 4.1.5 +pg_bao_url: https://bao.ad.ddupan.top:8200 +pg_bao_token: "{{ lookup('env', 'BAO_TOKEN') }}" +pg_config_root: /etc/homelab-postgresql +pg_data_root: /var/lib/homelab-postgresql +pg_bin_dir: /usr/lib/postgresql/18/bin +pg_patroni_bin: /opt/homelab-patroni/bin/patroni +pg_allow_initialize: false +# 仅存放 Bao 路径,值必须在控制端 no_log 读取;不从失效读请求推断秘密不存在。 +pg_secret_paths: + prod: infra/postgresql/prod + dev: infra/postgresql/dev +pg_etcd_secret_path: infra/etcd/consumers/patroni-pg-prod +pg_etcd_endpoints: ['192.168.10.127:2379', '10.60.0.20:2379', '10.60.0.21:2379'] +pg_profiles: + prod: + user: pgprod + port: 5432 + api_port: 8008 + dns: pg-prod.ad.ddupan.top + memory_high: 768M + memory_max: 1G + shared_buffers: 128MB + max_connections: 80 + dev: + user: pgdev + port: 5433 + dns: pg-dev.ad.ddupan.top + memory_high: 256M + memory_max: 384M + shared_buffers: 32MB + max_connections: 30 +# 客户端网络范围;HBA 仍需独立的 TLS + SCRAM 身份认证。 +pg_client_cidrs: ['192.168.10.0/24', '10.60.0.0/24', '10.42.0.0/16'] +pg_replication_addresses: ['192.168.10.127/32', '10.60.0.20/32'] +pg_repo_host: 10.60.0.21 +pg_repo_user: pgbackup +pg_repo_path: /var/lib/homelab-pgbackrest diff --git a/infrastructure/shared-postgresql/ansible/inventory/hosts.yml b/infrastructure/shared-postgresql/ansible/inventory/hosts.yml new file mode 100644 index 0000000..546c1b3 --- /dev/null +++ b/infrastructure/shared-postgresql/ansible/inventory/hosts.yml @@ -0,0 +1,44 @@ +--- +all: + vars: + ansible_user: root + children: + pg_hosts: + hosts: + pg-laptop: + ansible_connection: local + ansible_user: panxiao81 + ansible_host: 192.168.10.127 + pg_address: 192.168.10.127 + pg_instances: [prod, dev] + pg-pve1: + ansible_host: 10.60.0.20 + pg_address: 10.60.0.20 + pg_instances: [prod] + pg_repository: + hosts: + pg-pve2: + ansible_host: 10.60.0.21 + pg_pve: + hosts: + pve1: + ansible_host: 192.168.10.4 + pg_lxc_id: 150 + pg_lxc_hostname: etcd-pve1 + pg_lxc_memory: 1536 + pg_lxc_mount: /var/lib/homelab-postgresql/prod + pg_lxc_disk_gb: 16 + pve2: + ansible_host: 192.168.10.7 + pg_lxc_id: 151 + pg_lxc_hostname: etcd-pve2 + pg_lxc_memory: 768 + pg_lxc_mount: /var/lib/homelab-pgbackrest + pg_lxc_disk_gb: 32 + pg_proxy: + hosts: + pg-proxy-vyos: + ansible_host: 192.168.10.2 + ansible_user: vyos + ansible_connection: ssh + ansible_python_interpreter: /usr/bin/python3 diff --git a/infrastructure/shared-postgresql/ansible/limit-resync.yml b/infrastructure/shared-postgresql/ansible/limit-resync.yml new file mode 100644 index 0000000..22e8ace --- /dev/null +++ b/infrastructure/shared-postgresql/ansible/limit-resync.yml @@ -0,0 +1,10 @@ +--- +# 新 HDD 卷初始同步曾伴随共享链路 PingAck 超时;仅约束本项目 mp0,不改全局 quorum/协议。 +- name: 限制本项目新 HDD 卷的后台同步速率 + hosts: pg_pve + become: true + gather_facts: false + serial: 1 + tasks: + - name: 收敛本项目数据卷同步限额 + ansible.builtin.include_tasks: tasks/limit-resync.yml diff --git a/infrastructure/shared-postgresql/ansible/packages.yml b/infrastructure/shared-postgresql/ansible/packages.yml new file mode 100644 index 0000000..af4af92 --- /dev/null +++ b/infrastructure/shared-postgresql/ansible/packages.yml @@ -0,0 +1,23 @@ +--- +- name: 检查承载前提 + ansible.builtin.import_playbook: preflight.yml +- name: 准备数据库软件,不配置或初始化实例 + hosts: pg_hosts + become: true + serial: 1 + any_errors_fatal: true + roles: + - pg_packages + post_tasks: + - name: 核对包安装没有自动创建 PG18 默认实例 + ansible.builtin.stat: + path: /var/lib/postgresql/18/main/PG_VERSION + register: pg_package_default_cluster + - name: 默认实例应保持未初始化 + ansible.builtin.assert: + that: not pg_package_default_cluster.stat.exists + fail_msg: 发现默认 PG18 main 实例,停止后续初始化;本任务不会删除已有数据。 + - name: 验证独立 Patroni 环境可运行 + ansible.builtin.command: + argv: ["{{ pg_patroni_bin }}", --version] + changed_when: false diff --git a/infrastructure/shared-postgresql/ansible/preflight.yml b/infrastructure/shared-postgresql/ansible/preflight.yml new file mode 100644 index 0000000..434df98 --- /dev/null +++ b/infrastructure/shared-postgresql/ansible/preflight.yml @@ -0,0 +1,28 @@ +--- +- name: 只读核对数据库宿主与数据挂载 + hosts: pg_hosts + become: true + tasks: + - name: 限定已验证的平台 + ansible.builtin.assert: + that: + - ansible_facts.distribution == 'Ubuntu' + - ansible_facts.distribution_version == '24.04' + - ansible_facts.architecture == 'x86_64' + - pg_instances | difference(['prod', 'dev']) | length == 0 + - name: 核对共置内存预算 + ansible.builtin.assert: + that: + - ansible_facts.memtotal_mb >= (2048 if 'dev' in pg_instances else 1536) + fail_msg: PG 与 etcd 共置前需通过承载 IaC 扩容;不能以当前 512 MiB LXC 直接上线。 + - name: 核对每个数据父目录已单独挂载 + ansible.builtin.command: + argv: [findmnt, --mountpoint, "{{ pg_data_root }}/{{ item }}", --noheadings, --output, 'TARGET,FSTYPE,SOURCE'] + loop: "{{ pg_instances }}" + changed_when: false + check_mode: false + - name: 核对已有数据所有权 + ansible.builtin.include_tasks: tasks/preflight-instance.yml + loop: "{{ pg_instances }}" + loop_control: + loop_var: pg_instance diff --git a/infrastructure/shared-postgresql/ansible/proxy.yml b/infrastructure/shared-postgresql/ansible/proxy.yml new file mode 100644 index 0000000..0211c01 --- /dev/null +++ b/infrastructure/shared-postgresql/ansible/proxy.yml @@ -0,0 +1,87 @@ +--- +# 单独 HAProxy 实例复用 VyOS 的二进制与 HA 承载,避免修改发行版生成的全局模板。 +- name: 发布已经有唯一可写主库的生产入口 + hosts: pg_proxy + become: true + gather_facts: false + vars: + pg_proxy_bind: 192.168.10.2:5432 + pg_proxy_members: + - {name: laptop, address: '192.168.10.127', port: 5432, api_port: 8008} + - {name: pve1, address: '10.60.0.20', port: 5432, api_port: 8008} + tasks: + - name: 前置核对角色端点,不为未就绪实例发布入口 + ansible.builtin.uri: + url: "http://{{ item.address }}:{{ item.api_port }}/primary" + status_code: [200, 503] + loop: "{{ pg_proxy_members }}" + register: pg_proxy_primary + check_mode: false + - name: 必须有且只有一个 primary + ansible.builtin.assert: + that: pg_proxy_primary.results | selectattr('status', 'equalto', 200) | list | length == 1 + - name: 创建跨 VyOS 镜像升级保留的配置目录 + ansible.builtin.file: + path: /config/homelab-pg + state: directory + owner: root + group: root + mode: '0755' + - name: 写入并校验角色感知 HAProxy 配置 + ansible.builtin.template: + src: roles/pg_instance/templates/haproxy.cfg.j2 + dest: /config/homelab-pg/haproxy.cfg + mode: '0644' + validate: /usr/sbin/haproxy -c -f %s + notify: 重载数据库代理 + - name: 保存独立服务定义 + ansible.builtin.copy: + dest: /config/homelab-pg/homelab-pg-proxy.service + mode: '0644' + content: | + [Unit] + Description=Shared PostgreSQL role-aware TCP entry point + After=network-online.target + Wants=network-online.target + [Service] + Type=simple + DynamicUser=true + ExecStart=/usr/sbin/haproxy -W -db -f /config/homelab-pg/haproxy.cfg + ExecReload=/bin/kill -USR2 $MAINPID + Restart=on-failure + RestartSec=3 + MemoryMax=64M + LimitNOFILE=2048 + NoNewPrivileges=true + PrivateTmp=true + ProtectSystem=strict + ProtectHome=true + [Install] + WantedBy=multi-user.target + notify: 重载数据库代理 + - name: 安装服务链接 + ansible.builtin.file: + src: /config/homelab-pg/homelab-pg-proxy.service + dest: /etc/systemd/system/homelab-pg-proxy.service + state: link + - name: 通过既有 postconfig hook 在系统升级后恢复服务链接 + ansible.builtin.blockinfile: + path: /config/scripts/vyos-postconfig-bootup.script + marker: '# {mark} ANSIBLE HOMELAB PG PROXY' + insertbefore: '^exit 0' + block: | + ln -sfn /config/homelab-pg/homelab-pg-proxy.service /etc/systemd/system/homelab-pg-proxy.service + systemctl daemon-reload + systemctl enable --now homelab-pg-proxy.service + - name: 启动独立入口 + ansible.builtin.systemd_service: + name: homelab-pg-proxy + daemon_reload: true + enabled: true + state: started + handlers: + - name: 重载数据库代理 + ansible.builtin.systemd_service: + name: homelab-pg-proxy + daemon_reload: true + state: reloaded diff --git a/infrastructure/shared-postgresql/ansible/pve-storage.yml b/infrastructure/shared-postgresql/ansible/pve-storage.yml new file mode 100644 index 0000000..da0d8d0 --- /dev/null +++ b/infrastructure/shared-postgresql/ansible/pve-storage.yml @@ -0,0 +1,85 @@ +--- +# 仅扩展已经受管的两个 shared-etcd LXC;不移动/删除已有 rootfs 或数据卷。 +- name: 串行准备 standby 与备份仓库的独立 HDD 卷 + hosts: pg_pve + become: true + gather_facts: false + serial: 1 + any_errors_fatal: true + vars: + pg_etcd_health: + - /opt/homelab-etcd/etcdctl + - --endpoints=https://192.168.10.127:2379,https://10.60.0.20:2379,https://10.60.0.21:2379 + - --cacert=/etc/homelab-etcd/ca.crt + - --cert=/etc/homelab-etcd/admin.crt + - --key=/etc/homelab-etcd/admin.key + - endpoint + - health + tasks: + - name: 核对受管容器 + ansible.builtin.command: + argv: [pct, config, "{{ pg_lxc_id }}"] + register: pg_lxc_config + changed_when: false + check_mode: false + - name: 拒绝未知容器或已有冲突挂载 + ansible.builtin.assert: + that: + - pg_lxc_id in [150, 151] + - "'shared-etcd' in pg_lxc_config.stdout" + - "('hostname: ' ~ pg_lxc_hostname) in pg_lxc_config.stdout" + - >- + 'mp0:' not in pg_lxc_config.stdout or + ('mp=' ~ pg_lxc_mount ~ ',') in (pg_lxc_config.stdout ~ ',') + - "'mp0:' not in pg_lxc_config.stdout or 'mp0: pve-rg-hdd:' in pg_lxc_config.stdout" + - name: 验证维护前全部 etcd 成员健康 + ansible.builtin.command: + argv: "{{ pg_etcd_health }}" + delegate_to: localhost + changed_when: false + check_mode: false + - name: 首次添加独立卷 + when: "'mp0:' not in pg_lxc_config.stdout and not ansible_check_mode" + block: + - name: 正常停止当前单个成员 + ansible.builtin.command: + argv: [pct, shutdown, "{{ pg_lxc_id }}", --timeout, '60'] + changed_when: true + - name: 从指定 HDD 池创建新的 mp0 + ansible.builtin.command: + argv: + - pct + - set + - "{{ pg_lxc_id }}" + - --mp0 + - "pve-rg-hdd:{{ pg_lxc_disk_gb }},mp={{ pg_lxc_mount }},backup=1" + changed_when: true + - name: 容器启动前限制新卷的初始同步速率 + ansible.builtin.include_tasks: tasks/limit-resync.yml + always: + - name: 查询当前状态 + ansible.builtin.command: + argv: [pct, status, "{{ pg_lxc_id }}"] + register: pg_lxc_status + changed_when: false + - name: 恢复容器运行 + ansible.builtin.command: + argv: [pct, start, "{{ pg_lxc_id }}"] + changed_when: true + when: "'running' not in pg_lxc_status.stdout" + - name: 仅上调不足的内存限额,不缩容既有预算 + ansible.builtin.command: + argv: [pct, set, "{{ pg_lxc_id }}", --memory, "{{ pg_lxc_memory }}"] + when: >- + ((pg_lxc_config.stdout | regex_findall('(?m)^memory: ([0-9]+)$')) | first | int) < pg_lxc_memory + changed_when: true + - name: 确认当前成员恢复再处理下一台 + ansible.builtin.command: + argv: "{{ pg_etcd_health }}" + delegate_to: localhost + changed_when: false + register: pg_etcd_after + retries: 18 + delay: 5 + until: pg_etcd_after.rc == 0 + when: not ansible_check_mode diff --git a/infrastructure/shared-postgresql/ansible/renew-certificates.yml b/infrastructure/shared-postgresql/ansible/renew-certificates.yml new file mode 100644 index 0000000..36a725f --- /dev/null +++ b/infrastructure/shared-postgresql/ansible/renew-certificates.yml @@ -0,0 +1,14 @@ +--- +# 只续签和重载证书,不安装包、不变更数据库参数或重启服务。 +- name: 串行检查实例证书续签窗口 + hosts: pg_hosts + become: true + gather_facts: false + serial: 1 + any_errors_fatal: true + tasks: + - name: 逐实例续签 + ansible.builtin.include_tasks: tasks/renew-instance.yml + loop: "{{ pg_instances }}" + loop_control: + loop_var: pg_instance diff --git a/infrastructure/shared-postgresql/ansible/requirements.yml b/infrastructure/shared-postgresql/ansible/requirements.yml new file mode 100644 index 0000000..f9fe5b5 --- /dev/null +++ b/infrastructure/shared-postgresql/ansible/requirements.yml @@ -0,0 +1,4 @@ +--- +collections: + - name: community.crypto + version: 3.2.1 diff --git a/infrastructure/shared-postgresql/ansible/roles/pg_instance/tasks/certificate.yml b/infrastructure/shared-postgresql/ansible/roles/pg_instance/tasks/certificate.yml new file mode 100644 index 0000000..0461c3d --- /dev/null +++ b/infrastructure/shared-postgresql/ansible/roles/pg_instance/tasks/certificate.yml @@ -0,0 +1,69 @@ +--- +- name: 本地生成私钥 + community.crypto.openssl_privatekey: + path: "{{ pg_config_dir }}/{{ pg_cert_name }}.key" + type: ECC + curve: secp256r1 + owner: root + group: "{{ pg_profile.user }}" + mode: '0640' +- name: 本地生成 CSR + community.crypto.openssl_csr: + path: "{{ pg_config_dir }}/{{ pg_cert_name }}.csr" + privatekey_path: "{{ pg_config_dir }}/{{ pg_cert_name }}.key" + common_name: "{{ pg_cert_cn | default(omit, true) }}" + use_common_name_for_san: false + subject_alt_name: >- + {{ ['DNS:' ~ pg_profile.dns, 'IP:' ~ pg_address] if pg_cert_name == 'server' + else ['DNS:etcd-patroni-pg-prod'] }} + extended_key_usage: "{{ pg_cert_eku }}" + key_usage: [digitalSignature] + mode: '0644' + register: pg_csr +- name: 检查证书文件 + ansible.builtin.stat: + path: "{{ pg_config_dir }}/{{ pg_cert_name }}.crt" + register: pg_cert_file +- name: 检查续签窗口 + community.crypto.x509_certificate_info: + path: "{{ pg_config_dir }}/{{ pg_cert_name }}.crt" + valid_at: {renewal: '+14d'} + register: pg_cert_info + when: pg_cert_file.stat.exists +- name: 需要时请求中央 CA 签发 + when: not pg_cert_file.stat.exists or pg_csr is changed or not (pg_cert_info.valid_at.renewal | default(false)) + block: + - name: 读取公开 CSR + ansible.builtin.slurp: + src: "{{ pg_config_dir }}/{{ pg_cert_name }}.csr" + register: pg_csr_content + - name: 请求签发(凭据仅在控制端) + ansible.builtin.uri: + url: "{{ pg_bao_url }}/v1/pki/sign/{{ pg_cert_role }}" + method: POST + headers: + X-Vault-Token: "{{ pg_bao_token }}" + body_format: json + body: + csr: "{{ pg_csr_content.content | b64decode }}" + ttl: 1440h + status_code: 200 + delegate_to: localhost + become: false + register: pg_signed + no_log: true + - name: 保存证书和信任链 + ansible.builtin.copy: + content: "{{ item.content }}" + dest: "{{ pg_config_dir }}/{{ item.name }}" + owner: root + group: "{{ pg_profile.user }}" + mode: '0644' + loop: + - name: "{{ pg_cert_name }}.crt" + content: "{{ pg_signed.json.data.certificate }}\n{{ pg_signed.json.data.ca_chain | join('\n') }}\n" + - name: ca.crt + content: "{{ pg_signed.json.data.ca_chain | join('\n') }}\n" + - name: 记录本轮已签发证书以供显式续签入口重载 + ansible.builtin.set_fact: + pg_certificates_changed: true diff --git a/infrastructure/shared-postgresql/ansible/roles/pg_instance/tasks/main.yml b/infrastructure/shared-postgresql/ansible/roles/pg_instance/tasks/main.yml new file mode 100644 index 0000000..8d5b4ce --- /dev/null +++ b/infrastructure/shared-postgresql/ansible/roles/pg_instance/tasks/main.yml @@ -0,0 +1,80 @@ +--- +- name: 派生实例专用路径 + ansible.builtin.set_fact: + pg_profile: "{{ pg_profiles[pg_instance] }}" + pg_config_dir: "{{ pg_config_root }}/{{ pg_instance }}" + pg_parent_dir: "{{ pg_data_root }}/{{ pg_instance }}" + pg_data_dir: "{{ pg_data_root }}/{{ pg_instance }}/data" + pg_socket_dir: "/run/homelab-postgresql-{{ pg_instance }}" + pg_member: "{{ inventory_hostname }}" + pg_credentials: "{{ hostvars['localhost'].pg_loaded_credentials[pg_instance] }}" + pg_etcd_credentials: "{{ hostvars['localhost'].pg_loaded_etcd }}" + no_log: true +- name: 创建实例操作系统组 + ansible.builtin.group: + name: "{{ pg_profile.user }}" + system: true +- name: 创建实例操作系统账号 + ansible.builtin.user: + name: "{{ pg_profile.user }}" + group: "{{ pg_profile.user }}" + home: "{{ pg_parent_dir }}" + create_home: false + shell: /bin/bash + system: true +- name: 创建隔离目录 + ansible.builtin.file: + path: "{{ item }}" + state: directory + owner: "{{ pg_profile.user }}" + group: "{{ pg_profile.user }}" + mode: '0700' + loop: + - "{{ pg_parent_dir }}" + - "{{ pg_data_dir }}" + - "{{ pg_socket_dir }}" + - "{{ pg_parent_dir }}/backup-lock" + - "{{ pg_parent_dir }}/backup-spool" +- name: 保持共享配置父目录可遍历,实例目录各自受限 + ansible.builtin.file: + path: "{{ pg_config_root }}" + state: directory + owner: root + group: root + mode: '0755' +- name: 创建 root 管理的配置目录 + ansible.builtin.file: + path: "{{ pg_config_dir }}" + state: directory + owner: root + group: "{{ pg_profile.user }}" + mode: '0750' +- name: 签发实例服务器证书 + ansible.builtin.include_tasks: certificate.yml + vars: + pg_cert_name: server + pg_cert_cn: "{{ pg_profile.dns }}" + pg_cert_role: "homelab-pg-{{ pg_instance }}" + pg_cert_eku: [serverAuth] +- name: 签发 Patroni 独立无 CN 客户端证书 + ansible.builtin.include_tasks: certificate.yml + vars: + pg_cert_name: etcd + pg_cert_cn: '' + pg_cert_role: homelab-etcd-client + pg_cert_eku: [clientAuth] + when: pg_instance == 'prod' +- name: 写入实例配置 + ansible.builtin.template: + src: "{{ item }}.j2" + dest: "{{ pg_config_dir }}/{{ item }}" + owner: root + group: "{{ pg_profile.user }}" + mode: '0640' + loop: "{{ ['patroni.yml', 'pgbackrest.conf'] if pg_instance == 'prod' else ['postgresql.conf', 'pg_hba.conf'] }}" + no_log: true +- name: 安装实例专用服务(不自动启动或重启) + ansible.builtin.template: + src: instance.service.j2 + dest: "/etc/systemd/system/homelab-postgresql-{{ pg_instance }}.service" + mode: '0644' diff --git a/infrastructure/shared-postgresql/ansible/roles/pg_instance/templates/backup.service.j2 b/infrastructure/shared-postgresql/ansible/roles/pg_instance/templates/backup.service.j2 new file mode 100644 index 0000000..a5a7b55 --- /dev/null +++ b/infrastructure/shared-postgresql/ansible/roles/pg_instance/templates/backup.service.j2 @@ -0,0 +1,14 @@ +[Unit] +Description=Shared PostgreSQL primary-aware full backup +After=network-online.target +RequiresMountsFor={{ pg_repo_path }} +[Service] +Type=oneshot +User=pgbackup +Group=pgbackup +ExecStart=/usr/bin/pgbackrest --config=/etc/homelab-pgbackrest/repository.conf --stanza=prod --type=full backup +TimeoutStartSec=2h +UMask=0077 +Nice=10 +IOSchedulingClass=best-effort +IOSchedulingPriority=7 diff --git a/infrastructure/shared-postgresql/ansible/roles/pg_instance/templates/backup.timer.j2 b/infrastructure/shared-postgresql/ansible/roles/pg_instance/templates/backup.timer.j2 new file mode 100644 index 0000000..bdf9091 --- /dev/null +++ b/infrastructure/shared-postgresql/ansible/roles/pg_instance/templates/backup.timer.j2 @@ -0,0 +1,8 @@ +[Unit] +Description=Daily shared PostgreSQL full backup +[Timer] +OnCalendar=*-*-* 03:40:00 UTC +RandomizedDelaySec=10min +Persistent=true +[Install] +WantedBy=timers.target diff --git a/infrastructure/shared-postgresql/ansible/roles/pg_instance/templates/haproxy.cfg.j2 b/infrastructure/shared-postgresql/ansible/roles/pg_instance/templates/haproxy.cfg.j2 new file mode 100644 index 0000000..33b947a --- /dev/null +++ b/infrastructure/shared-postgresql/ansible/roles/pg_instance/templates/haproxy.cfg.j2 @@ -0,0 +1,19 @@ +# 参考 Pigsty v4.5.0 的 Patroni 角色检查与旧连接摘除行为。 +global + maxconn 256 + log stdout format raw local0 + stats socket /tmp/homelab-pg-haproxy.sock mode 600 level admin +defaults + mode tcp + timeout connect 3s + timeout client 1h + timeout server 1h + timeout check 3s +listen pg-prod + bind {{ pg_proxy_bind }} + option httpchk GET /primary + http-check expect status 200 + default-server inter 2s fall 3 rise 2 on-marked-down shutdown-sessions +{% for member in pg_proxy_members %} + server {{ member.name }} {{ member.address }}:{{ member.port }} check port {{ member.api_port }} +{% endfor %} diff --git a/infrastructure/shared-postgresql/ansible/roles/pg_instance/templates/instance.service.j2 b/infrastructure/shared-postgresql/ansible/roles/pg_instance/templates/instance.service.j2 new file mode 100644 index 0000000..60b4fa6 --- /dev/null +++ b/infrastructure/shared-postgresql/ansible/roles/pg_instance/templates/instance.service.j2 @@ -0,0 +1,33 @@ +[Unit] +Description=Homelab PostgreSQL {{ pg_instance }}{{ ' through Patroni' if pg_instance == 'prod' else '' }} +After=network-online.target +Wants=network-online.target +RequiresMountsFor={{ pg_parent_dir }} +[Service] +User={{ pg_profile.user }} +Group={{ pg_profile.user }} +UMask=0077 +RuntimeDirectory=homelab-postgresql-{{ pg_instance }} +RuntimeDirectoryMode=0700 +{% if pg_instance == 'prod' %} +Type=simple +ExecStart={{ pg_patroni_bin }} {{ pg_config_dir }}/patroni.yml +KillMode=process +{% else %} +Type=simple +ExecStart={{ pg_bin_dir }}/postgres -D {{ pg_data_dir }} -c config_file={{ pg_config_dir }}/postgresql.conf +KillSignal=SIGINT +{% endif %} +Restart=on-failure +RestartSec=5 +TimeoutStopSec=120 +MemoryHigh={{ pg_profile.memory_high }} +MemoryMax={{ pg_profile.memory_max }} +OOMPolicy=stop +NoNewPrivileges=true +ProtectSystem=strict +ProtectHome=true +PrivateTmp=true +ReadWritePaths={{ pg_parent_dir }} {{ pg_config_dir }} {{ pg_socket_dir }} +[Install] +WantedBy=multi-user.target diff --git a/infrastructure/shared-postgresql/ansible/roles/pg_instance/templates/patroni.yml.j2 b/infrastructure/shared-postgresql/ansible/roles/pg_instance/templates/patroni.yml.j2 new file mode 100644 index 0000000..81c18a9 --- /dev/null +++ b/infrastructure/shared-postgresql/ansible/roles/pg_instance/templates/patroni.yml.j2 @@ -0,0 +1,99 @@ +# 实例化部署;不引用 Pigsty 的全局 /pg 或宿主内存自动调优。 +name: {{ pg_member | to_json }} +namespace: /homelab/patroni/ +scope: pg-prod +restapi: + listen: {{ pg_address }}:{{ pg_profile.api_port }} + connect_address: {{ pg_address }}:{{ pg_profile.api_port }} + authentication: + username: patroni + password: {{ pg_credentials.rest_password | to_json }} +etcd3: + hosts: {{ pg_etcd_endpoints | to_json }} + protocol: https + username: {{ pg_etcd_credentials.username | to_json }} + password: {{ pg_etcd_credentials.password | to_json }} + cacert: {{ pg_config_dir }}/ca.crt + cert: {{ pg_config_dir }}/etcd.crt + key: {{ pg_config_dir }}/etcd.key +bootstrap: + dcs: + ttl: 30 + loop_wait: 10 + retry_timeout: 10 + maximum_lag_on_failover: 1048576 + check_timeline: true + synchronous_mode: false + failsafe_mode: false + postgresql: + use_pg_rewind: true + use_slots: true + parameters: + max_connections: {{ pg_profile.max_connections }} + wal_level: replica + hot_standby: 'on' + wal_log_hints: 'on' + max_wal_senders: 5 + max_replication_slots: 5 + max_slot_wal_keep_size: 2GB + max_wal_size: 1GB + min_wal_size: 80MB + archive_mode: 'on' + archive_timeout: 300s + archive_command: "pgbackrest --config={{ pg_config_dir }}/pgbackrest.conf --stanza=prod archive-push %p" + initdb: + - encoding: UTF8 + - locale: C.UTF-8 + - data-checksums +postgresql: + listen: {{ pg_address }}:{{ pg_profile.port }} + connect_address: {{ pg_address }}:{{ pg_profile.port }} + data_dir: {{ pg_data_dir }} + bin_dir: {{ pg_bin_dir }} + pgpass: {{ pg_parent_dir }}/pgpass + use_unix_socket: true + use_unix_socket_repl: true + authentication: + superuser: + username: {{ pg_profile.user }} + password: {{ pg_credentials.superuser_password | to_json }} + replication: + username: replicator + password: {{ pg_credentials.replication_password | to_json }} + sslmode: verify-full + sslrootcert: {{ pg_config_dir }}/ca.crt + parameters: + unix_socket_directories: {{ pg_socket_dir }} + unix_socket_permissions: '0700' + password_encryption: scram-sha-256 + shared_buffers: {{ pg_profile.shared_buffers }} + work_mem: 4MB + maintenance_work_mem: 64MB + ssl: 'on' + ssl_cert_file: {{ pg_config_dir }}/server.crt + ssl_key_file: {{ pg_config_dir }}/server.key + ssl_ca_file: {{ pg_config_dir }}/ca.crt + log_statement: none + log_min_error_statement: panic + pg_hba: + - local all {{ pg_profile.user }} peer + - local replication {{ pg_profile.user }} peer + - local replication replicator scram-sha-256 +{% for address in pg_replication_addresses %} + - hostssl replication replicator {{ address }} scram-sha-256 + - hostssl all {{ pg_profile.user }} {{ address }} scram-sha-256 +{% endfor %} +{% for cidr in pg_client_cidrs %} + - hostssl all all {{ cidr }} scram-sha-256 +{% endfor %} + - host all all 0.0.0.0/0 reject + - host all all ::/0 reject + remove_data_directory_on_rewind_failure: false + remove_data_directory_on_diverged_timelines: false +watchdog: + mode: 'off' +tags: + nofailover: false + noloadbalance: false + clonefrom: false + nosync: false diff --git a/infrastructure/shared-postgresql/ansible/roles/pg_instance/templates/pg_hba.conf.j2 b/infrastructure/shared-postgresql/ansible/roles/pg_instance/templates/pg_hba.conf.j2 new file mode 100644 index 0000000..cfd0be5 --- /dev/null +++ b/infrastructure/shared-postgresql/ansible/roles/pg_instance/templates/pg_hba.conf.j2 @@ -0,0 +1,6 @@ +local all {{ pg_profile.user }} peer +{% for cidr in pg_client_cidrs %} +hostssl all all {{ cidr }} scram-sha-256 +{% endfor %} +host all all 0.0.0.0/0 reject +host all all ::/0 reject diff --git a/infrastructure/shared-postgresql/ansible/roles/pg_instance/templates/pgbackrest.conf.j2 b/infrastructure/shared-postgresql/ansible/roles/pg_instance/templates/pgbackrest.conf.j2 new file mode 100644 index 0000000..771bb25 --- /dev/null +++ b/infrastructure/shared-postgresql/ansible/roles/pg_instance/templates/pgbackrest.conf.j2 @@ -0,0 +1,22 @@ +[global] +{% if pg_repo_remote | default(true) %} +repo1-host={{ pg_repo_host }} +repo1-host-user={{ pg_repo_user }} +repo1-host-config=/etc/homelab-pgbackrest/repository.conf +{% endif %} +repo1-path={{ pg_repo_path }} +repo1-retention-full=3 +process-max=1 +compress-type=zst +compress-level=3 +start-fast=y +log-level-console=warn +log-level-file=off +lock-path={{ pg_parent_dir }}/backup-lock +spool-path={{ pg_parent_dir }}/backup-spool +# 不设置 archive-push-queue-max;超限丢弃 WAL 会破坏恢复链。 +[prod] +pg1-path={{ pg_data_dir }} +pg1-port={{ pg_profile.port }} +pg1-socket-path={{ pg_socket_dir }} +pg1-user={{ pg_profile.user }} diff --git a/infrastructure/shared-postgresql/ansible/roles/pg_instance/templates/postgresql.conf.j2 b/infrastructure/shared-postgresql/ansible/roles/pg_instance/templates/postgresql.conf.j2 new file mode 100644 index 0000000..2f07377 --- /dev/null +++ b/infrastructure/shared-postgresql/ansible/roles/pg_instance/templates/postgresql.conf.j2 @@ -0,0 +1,18 @@ +listen_addresses = '{{ pg_address }}' +port = {{ pg_profile.port }} +unix_socket_directories = '{{ pg_socket_dir }}' +unix_socket_permissions = 0700 +hba_file = '{{ pg_config_dir }}/pg_hba.conf' +password_encryption = 'scram-sha-256' +shared_buffers = '{{ pg_profile.shared_buffers }}' +max_connections = {{ pg_profile.max_connections }} +work_mem = '4MB' +maintenance_work_mem = '32MB' +max_wal_size = '512MB' +ssl = on +ssl_cert_file = '{{ pg_config_dir }}/server.crt' +ssl_key_file = '{{ pg_config_dir }}/server.key' +ssl_ca_file = '{{ pg_config_dir }}/ca.crt' +archive_mode = off +log_statement = none +log_min_error_statement = panic diff --git a/infrastructure/shared-postgresql/ansible/roles/pg_instance/templates/repository.conf.j2 b/infrastructure/shared-postgresql/ansible/roles/pg_instance/templates/repository.conf.j2 new file mode 100644 index 0000000..41f06d1 --- /dev/null +++ b/infrastructure/shared-postgresql/ansible/roles/pg_instance/templates/repository.conf.j2 @@ -0,0 +1,21 @@ +# 备份从仓库发起,发现实际 primary,避免切换后定时器仍依赖旧主。 +[global] +repo1-path={{ pg_repo_path }} +repo1-retention-full=3 +process-max=1 +compress-type=zst +compress-level=3 +start-fast=y +log-level-console=info +log-level-file=off +lock-path={{ pg_repo_path }}/lock +[prod] +{% for member in pg_backup_members %} +pg{{ loop.index }}-host={{ member.address }} +pg{{ loop.index }}-host-user=pgprod +pg{{ loop.index }}-host-config=/etc/homelab-postgresql/prod/pgbackrest.conf +pg{{ loop.index }}-path=/var/lib/homelab-postgresql/prod/data +pg{{ loop.index }}-port=5432 +pg{{ loop.index }}-socket-path=/run/homelab-postgresql-prod +pg{{ loop.index }}-user=pgprod +{% endfor %} diff --git a/infrastructure/shared-postgresql/ansible/roles/pg_packages/tasks/main.yml b/infrastructure/shared-postgresql/ansible/roles/pg_packages/tasks/main.yml new file mode 100644 index 0000000..bcf5a57 --- /dev/null +++ b/infrastructure/shared-postgresql/ansible/roles/pg_packages/tasks/main.yml @@ -0,0 +1,40 @@ +--- +- name: 配置官方软件源 + ansible.builtin.import_tasks: repository.yml +- name: 安装集群管理基础包(不安装默认主版本元包) + ansible.builtin.apt: + name: postgresql-common + state: present +- name: 查询已安装包以避免重复制造临时配置变更 + ansible.builtin.package_facts: + manager: apt +- name: 临时禁止包安装自动建立默认 main 实例 + when: >- + ['postgresql-18', 'postgresql-client-18', 'pgbackrest'] + | difference(ansible_facts.packages.keys() | list) | length > 0 + block: + - name: 确保包配置片段目录存在 + ansible.builtin.file: + path: /etc/postgresql-common/createcluster.d + state: directory + mode: '0755' + - name: 禁止本次 apt postinst 自动初始化 + ansible.builtin.copy: + dest: /etc/postgresql-common/createcluster.d/90-homelab-no-auto.conf + content: "create_main_cluster = false\n" + mode: '0644' + - name: 安装明确的 PG 主版本与备份工具 + ansible.builtin.apt: + name: ['postgresql-18', 'postgresql-client-18', pgbackrest] + state: present + install_recommends: false + always: + - name: 撤回临时包安装选项 + ansible.builtin.file: + path: /etc/postgresql-common/createcluster.d/90-homelab-no-auto.conf + state: absent +- name: 在独立 venv 安装固定 Patroni 版本 + ansible.builtin.pip: + name: 'patroni[etcd3,psycopg3]=={{ pg_patroni_version }}' + virtualenv: /opt/homelab-patroni + virtualenv_command: python3 -m venv diff --git a/infrastructure/shared-postgresql/ansible/roles/pg_packages/tasks/repository.yml b/infrastructure/shared-postgresql/ansible/roles/pg_packages/tasks/repository.yml new file mode 100644 index 0000000..79d0db4 --- /dev/null +++ b/infrastructure/shared-postgresql/ansible/roles/pg_packages/tasks/repository.yml @@ -0,0 +1,51 @@ +--- +# 共用宿主机上其他第三方源故障不能阻塞本项目;仅刷新 Ubuntu/PGDG,仍强制签名验证。 +- name: 仅刷新 Ubuntu 官方基础软件源 # noqa: command-instead-of-module + ansible.builtin.command: + argv: + - apt-get + - -o + - Dir::Etc::sourcelist=/etc/apt/sources.list.d/ubuntu.sources + - -o + - Dir::Etc::sourceparts=- + - -o + - APT::Get::List-Cleanup=0 + - -o + - APT::Update::Error-Mode=any + - update + changed_when: false +- name: 安装软件源及 Python 依赖 + ansible.builtin.apt: + name: [ca-certificates, python3-debian, python3-venv, python3-cryptography, acl] + state: present +- name: 安装官方 PGDG 公钥 + ansible.builtin.get_url: + url: https://www.postgresql.org/media/keys/ACCC4CF8.asc + dest: /usr/share/keyrings/homelab-pgdg.asc + mode: '0644' + register: pg_repo_key + retries: 3 + delay: 5 + until: pg_repo_key is succeeded +- name: 声明 PGDG 官方软件源 + ansible.builtin.deb822_repository: + name: homelab-pgdg + uris: https://apt.postgresql.org/pub/repos/apt + suites: noble-pgdg + components: main + architectures: amd64 + signed_by: /usr/share/keyrings/homelab-pgdg.asc +- name: 仅刷新本项目 PGDG 软件源 # noqa: command-instead-of-module + ansible.builtin.command: + argv: + - apt-get + - -o + - Dir::Etc::sourcelist=/etc/apt/sources.list.d/homelab-pgdg.sources + - -o + - Dir::Etc::sourceparts=- + - -o + - APT::Get::List-Cleanup=0 + - -o + - APT::Update::Error-Mode=any + - update + changed_when: false diff --git a/infrastructure/shared-postgresql/ansible/site.yml b/infrastructure/shared-postgresql/ansible/site.yml new file mode 100644 index 0000000..4f49233 --- /dev/null +++ b/infrastructure/shared-postgresql/ansible/site.yml @@ -0,0 +1,17 @@ +--- +- name: 检查承载前提 + ansible.builtin.import_playbook: preflight.yml +- name: 验证秘密可用 + ansible.builtin.import_playbook: credentials.yml +- name: 配置数据库实例(不初始化、不启动、不自动重启) + hosts: pg_hosts + become: true + roles: + - pg_packages + tasks: + - name: 配置声明的实例 + ansible.builtin.include_role: + name: pg_instance + loop: "{{ pg_instances }}" + loop_control: + loop_var: pg_instance diff --git a/infrastructure/shared-postgresql/ansible/storage.yml b/infrastructure/shared-postgresql/ansible/storage.yml new file mode 100644 index 0000000..345ae72 --- /dev/null +++ b/infrastructure/shared-postgresql/ansible/storage.yml @@ -0,0 +1,57 @@ +--- +# 只创建专属 dataset;PVE 新挂载需串行重启 etcd 成员,单独处理,不混入日常数据库收敛。 +- name: 创建 laptop 独立生产与开发 dataset + hosts: pg-laptop + become: true + gather_facts: false + tasks: + - name: 查询专属 ZFS dataset + ansible.builtin.command: + argv: [zfs, list, -H, -o, name, "data/homelab-pg-{{ item }}"] + loop: [prod, dev] + register: pg_datasets + changed_when: false + failed_when: pg_datasets.rc not in [0, 1] + check_mode: false + - name: 检查新 dataset 挂载路径未被占用 + ansible.builtin.stat: + path: "{{ pg_data_root }}/{{ item.item }}" + loop: "{{ pg_datasets.results }}" + when: item.rc == 1 + register: pg_new_mounts + - name: 禁止覆盖已有目录 + ansible.builtin.assert: + that: not item.stat.exists + loop: "{{ pg_new_mounts.results }}" + when: not (item.skipped | default(false)) + - name: 新建本项目专用 ZFS dataset + ansible.builtin.command: + argv: + - zfs + - create + - -o + - "mountpoint={{ pg_data_root }}/{{ item.item }}" + - -o + - recordsize=16K + - -o + - compression=lz4 + - -o + - atime=off + - -o + - "quota={{ '32G' if item.item == 'prod' else '8G' }}" + - -o + - org.homelab:owner=shared-postgresql + - "data/homelab-pg-{{ item.item }}" + loop: "{{ pg_datasets.results }}" + when: item.rc == 1 + changed_when: true + - name: 核对所有权与挂载路径 + ansible.builtin.command: + argv: [zfs, get, -H, -o, value, 'org.homelab:owner,mountpoint', "data/homelab-pg-{{ item }}"] + loop: [prod, dev] + register: pg_dataset_owner + changed_when: false + failed_when: >- + pg_dataset_owner.rc != 0 or + pg_dataset_owner.stdout_lines != ['shared-postgresql', pg_data_root ~ '/' ~ item] + when: not ansible_check_mode diff --git a/infrastructure/shared-postgresql/ansible/tasks/ayatori-credential.yml b/infrastructure/shared-postgresql/ansible/tasks/ayatori-credential.yml new file mode 100644 index 0000000..4612914 --- /dev/null +++ b/infrastructure/shared-postgresql/ansible/tasks/ayatori-credential.yml @@ -0,0 +1,37 @@ +--- +- name: 读取独立管理凭据元数据 + ansible.builtin.uri: + url: "{{ pg_bao_url }}/v1/kv/metadata/infra/postgresql/ayatori/{{ pg_ayatori_instance }}" + headers: {X-Vault-Token: "{{ pg_bao_token }}"} + status_code: [200, 404] + register: pg_admin_metadata + no_log: true +- name: 只在明确缺失时写入独立的用户名密码 + ansible.builtin.uri: + url: "{{ pg_bao_url }}/v1/kv/data/infra/postgresql/ayatori/{{ pg_ayatori_instance }}" + method: POST + headers: {X-Vault-Token: "{{ pg_bao_token }}"} + body_format: json + body: + options: {cas: 0} + data: + username: "{{ pg_loaded_credentials[pg_ayatori_instance].ayatori_username }}" + password: "{{ pg_loaded_credentials[pg_ayatori_instance].ayatori_password }}" + status_code: 200 + when: pg_admin_metadata.status == 404 + changed_when: true + no_log: true +- name: 回读已有管理凭据 + ansible.builtin.uri: + url: "{{ pg_bao_url }}/v1/kv/data/infra/postgresql/ayatori/{{ pg_ayatori_instance }}" + headers: {X-Vault-Token: "{{ pg_bao_token }}"} + status_code: 200 + register: pg_admin_readback + no_log: true +- name: 不隐式覆盖或轮换不一致凭据 + ansible.builtin.assert: + that: + - pg_admin_readback.json.data.data.keys() | sort == ['password', 'username'] + - pg_admin_readback.json.data.data.username == pg_loaded_credentials[pg_ayatori_instance].ayatori_username + - pg_admin_readback.json.data.data.password == pg_loaded_credentials[pg_ayatori_instance].ayatori_password + no_log: true diff --git a/infrastructure/shared-postgresql/ansible/tasks/bootstrap-instance.yml b/infrastructure/shared-postgresql/ansible/tasks/bootstrap-instance.yml new file mode 100644 index 0000000..9a6531d --- /dev/null +++ b/infrastructure/shared-postgresql/ansible/tasks/bootstrap-instance.yml @@ -0,0 +1,172 @@ +--- +- name: 设置实例初始化参数 + ansible.builtin.set_fact: + pg_profile: "{{ pg_profiles[pg_instance] }}" + pg_parent_dir: "{{ pg_data_root }}/{{ pg_instance }}" + pg_data_dir: "{{ pg_data_root }}/{{ pg_instance }}/data" + pg_config_dir: "{{ pg_config_root }}/{{ pg_instance }}" + pg_bootstrap_password: "{{ hostvars['localhost'].pg_loaded_credentials[pg_instance].ayatori_password }}" + pg_socket_dir: "/run/homelab-postgresql-{{ pg_instance }}" + no_log: true +- name: 检查版本标记 + ansible.builtin.stat: + path: "{{ pg_data_dir }}/PG_VERSION" + register: pg_version_file +- name: 核对未初始化目录为空 + ansible.builtin.find: + paths: "{{ pg_data_dir }}" + hidden: true + file_type: any + register: pg_initial_files + when: not pg_version_file.stat.exists +- name: 禁止覆盖不完整或未知数据 + ansible.builtin.assert: + that: pg_version_file.stat.exists or pg_initial_files.matched == 0 +- name: 初始化独立开发实例 + when: pg_instance == 'dev' and not pg_version_file.stat.exists + block: + - name: 临时写入开发管理员密码 + ansible.builtin.copy: + content: "{{ hostvars['localhost'].pg_loaded_credentials.dev.superuser_password }}\n" + dest: "{{ pg_parent_dir }}/.init-password" + owner: "{{ pg_profile.user }}" + mode: '0600' + no_log: true + - name: 仅在空目录初始化 + ansible.builtin.command: + argv: + - "{{ pg_bin_dir }}/initdb" + - -D + - "{{ pg_data_dir }}" + - --username={{ pg_profile.user }} + - --pwfile={{ pg_parent_dir }}/.init-password + - --auth-local=peer + - --auth-host=scram-sha-256 + - --encoding=UTF8 + - --locale=C.UTF-8 + - --data-checksums + become: true + become_user: "{{ pg_profile.user }}" + changed_when: true + no_log: true + always: + - name: 删除临时初始化密码文件 + ansible.builtin.file: + path: "{{ pg_parent_dir }}/.init-password" + state: absent +- name: 登记专属数据目录(启动失败后仍可辨认来源) + ansible.builtin.copy: + content: "shared-postgresql/{{ pg_instance }}\n" + dest: "{{ pg_parent_dir }}/.homelab-owned" + owner: root + mode: '0644' +- name: 启动配置完成的新实例 + ansible.builtin.systemd_service: + name: "homelab-postgresql-{{ pg_instance }}" + daemon_reload: true + enabled: true + state: started +- name: 等待本机 SQL 可连接 + ansible.builtin.command: + argv: + - '{{ pg_bin_dir }}/psql' + - -X + - -At + - -h + - '{{ pg_socket_dir }}' + - -p + - '{{ pg_profile.port | string }}' + - -d + - postgres + - -c + - SELECT 1 + become: true + become_user: "{{ pg_profile.user }}" + register: pg_ready + until: pg_ready.rc == 0 + retries: 30 + delay: 2 + changed_when: false +- name: 核对本节点是否可写 + ansible.builtin.command: + argv: + - '{{ pg_bin_dir }}/psql' + - -X + - -At + - -h + - '{{ pg_socket_dir }}' + - -p + - '{{ pg_profile.port | string }}' + - -d + - postgres + - -c + - SELECT NOT pg_is_in_recovery() + become: true + become_user: "{{ pg_profile.user }}" + register: pg_is_primary + changed_when: false +- name: 在可写实例创建 Ayatori 最小管理账号 + when: pg_is_primary.stdout | trim == 't' + block: + - name: 查询 Ayatori 账号 + ansible.builtin.command: + argv: + - '{{ pg_bin_dir }}/psql' + - -X + - -At + - -h + - '{{ pg_socket_dir }}' + - -p + - '{{ pg_profile.port | string }}' + - -d + - postgres + - -c + - SELECT count(*) FROM pg_roles WHERE rolname = 'ayatori' + become: true + become_user: "{{ pg_profile.user }}" + register: pg_ayatori_exists + changed_when: false + - name: 首次建立受限角色(不静默轮换已存在密码) + ansible.builtin.command: + argv: + - '{{ pg_bin_dir }}/psql' + - -X + - -v + - ON_ERROR_STOP=1 + - -h + - '{{ pg_socket_dir }}' + - -p + - '{{ pg_profile.port | string }}' + - -d + - postgres + stdin: >- + CREATE ROLE ayatori LOGIN NOSUPERUSER CREATEDB CREATEROLE NOREPLICATION NOBYPASSRLS + PASSWORD '{{ pg_bootstrap_password | replace("'", "''") }}'; + become: true + become_user: "{{ pg_profile.user }}" + when: pg_ayatori_exists.stdout | trim == '0' + changed_when: true + no_log: true + - name: 以实际保存密码及 verify-full 验证权限 + ansible.builtin.command: + argv: + - "{{ pg_bin_dir }}/psql" + - -X + - -At + - -c + - >- + SELECT NOT rolsuper AND rolcreatedb AND rolcreaterole AND NOT pg_is_in_recovery() + FROM pg_roles WHERE rolname = current_user + environment: + PGHOST: "{{ pg_profile.dns }}" + PGHOSTADDR: "{{ pg_address }}" + PGPORT: "{{ pg_profile.port | string }}" + PGDATABASE: postgres + PGUSER: ayatori + PGPASSWORD: "{{ hostvars['localhost'].pg_loaded_credentials[pg_instance].ayatori_password }}" + PGSSLMODE: verify-full + PGSSLROOTCERT: "{{ pg_config_dir }}/ca.crt" + register: pg_ayatori_check + changed_when: false + failed_when: pg_ayatori_check.rc != 0 or pg_ayatori_check.stdout | trim != 't' + no_log: true diff --git a/infrastructure/shared-postgresql/ansible/tasks/initialize-secret.yml b/infrastructure/shared-postgresql/ansible/tasks/initialize-secret.yml new file mode 100644 index 0000000..13785e7 --- /dev/null +++ b/infrastructure/shared-postgresql/ansible/tasks/initialize-secret.yml @@ -0,0 +1,46 @@ +--- +- name: 核对秘密元数据(非 404 错误不能视为不存在) + ansible.builtin.uri: + url: "{{ pg_bao_url }}/v1/kv/metadata/{{ pg_secret_paths[pg_secret_instance] }}" + headers: + X-Vault-Token: "{{ pg_bao_token }}" + status_code: [200, 404] + register: pg_secret_metadata + no_log: true +- name: 明确不存在时创建一次 + when: pg_secret_metadata.status == 404 + block: + - name: 拒绝为已有实例重造丢失秘密 + ansible.builtin.assert: + that: >- + groups['pg_hosts'] | map('extract', hostvars, 'pg_has_data') + | select('equalto', true) | list | length == 0 + - name: 随机生成只在本次执行内存存在的值 + ansible.builtin.set_fact: + pg_new_secret: + ayatori_username: ayatori + ayatori_password: "{{ lookup('password', '/dev/null', length=48, chars=['ascii_letters', 'digits']) }}" + superuser_password: "{{ lookup('password', '/dev/null', length=48, chars=['ascii_letters', 'digits']) }}" + replication_password: "{{ lookup('password', '/dev/null', length=48, chars=['ascii_letters', 'digits']) }}" + rest_password: "{{ lookup('password', '/dev/null', length=48, chars=['ascii_letters', 'digits']) }}" + no_log: true + - name: 使用 KV v2 CAS=0 防覆盖写入 + ansible.builtin.uri: + url: "{{ pg_bao_url }}/v1/kv/data/{{ pg_secret_paths[pg_secret_instance] }}" + method: POST + headers: + X-Vault-Token: "{{ pg_bao_token }}" + body_format: json + body: + options: {cas: 0} + data: "{{ pg_new_secret }}" + status_code: 200 + changed_when: true + no_log: true +- name: 确认已有秘密数据可读,软删除必须失败 + ansible.builtin.uri: + url: "{{ pg_bao_url }}/v1/kv/data/{{ pg_secret_paths[pg_secret_instance] }}" + headers: + X-Vault-Token: "{{ pg_bao_token }}" + status_code: 200 + no_log: true diff --git a/infrastructure/shared-postgresql/ansible/tasks/limit-resync.yml b/infrastructure/shared-postgresql/ansible/tasks/limit-resync.yml new file mode 100644 index 0000000..7e2e9ec --- /dev/null +++ b/infrastructure/shared-postgresql/ansible/tasks/limit-resync.yml @@ -0,0 +1,34 @@ +--- +- name: 从受管容器配置定位资源 + ansible.builtin.command: + argv: [pct, config, "{{ pg_lxc_id }}"] + register: pg_resync_config + changed_when: false + check_mode: false +- name: 核对容器和专属挂载 + ansible.builtin.assert: + that: + - pg_lxc_id in [150, 151] + - "'shared-etcd' in pg_resync_config.stdout" + - "('mp=' ~ pg_lxc_mount ~ ',') in pg_resync_config.stdout" + - "pg_resync_config.stdout | regex_findall('(?m)^mp0: pve-rg-hdd:(pm-[0-9a-f]+)_') | length == 1" +- name: 提取明确匹配的 LINSTOR resource definition + ansible.builtin.set_fact: + pg_resync_resource: "{{ (pg_resync_config.stdout | regex_findall('(?m)^mp0: pve-rg-hdd:(pm-[0-9a-f]+)_'))[0] }}" +- name: 读取本资源已有配置 + ansible.builtin.command: + argv: [linstor, resource-definition, list-properties, "{{ pg_resync_resource }}"] + register: pg_resync_properties + changed_when: false + check_mode: false +- name: 每个新卷将自适应同步速率上限设为 10 MiB/s + ansible.builtin.command: + argv: + - linstor + - resource-definition + - set-property + - "{{ pg_resync_resource }}" + - DrbdOptions/PeerDevice/c-max-rate + - '10240' + when: pg_resync_properties.stdout is not search('c-max-rate\s*│\s*10240\s*│') + changed_when: true diff --git a/infrastructure/shared-postgresql/ansible/tasks/preflight-instance.yml b/infrastructure/shared-postgresql/ansible/tasks/preflight-instance.yml new file mode 100644 index 0000000..6f51e9a --- /dev/null +++ b/infrastructure/shared-postgresql/ansible/tasks/preflight-instance.yml @@ -0,0 +1,33 @@ +--- +- name: 检查现有 PGDATA + ansible.builtin.stat: + path: "{{ pg_data_root }}/{{ pg_instance }}/data/PG_VERSION" + register: pg_existing_version +- name: 检查本项目所有权标记 + ansible.builtin.stat: + path: "{{ pg_data_root }}/{{ pg_instance }}/.homelab-owned" + register: pg_owned_marker +- name: 拒绝接管未知现有实例 + ansible.builtin.assert: + that: not pg_existing_version.stat.exists or pg_owned_marker.stat.exists + fail_msg: 发现未登记 PGDATA,停止;导入和恢复必须使用独立流程。 +- name: 核对已登记实例 + when: pg_owned_marker.stat.exists + block: + - name: 读取实例标记 + ansible.builtin.slurp: + src: "{{ pg_data_root }}/{{ pg_instance }}/.homelab-owned" + register: pg_marker_content + - name: 验证实例身份 + ansible.builtin.assert: + that: (pg_marker_content.content | b64decode | trim) == ('shared-postgresql/' ~ pg_instance) +- name: 核对已有实例主版本,禁止隐式升级 + when: pg_existing_version.stat.exists + block: + - name: 读取 PG_VERSION + ansible.builtin.slurp: + src: "{{ pg_data_root }}/{{ pg_instance }}/data/PG_VERSION" + register: pg_existing_version_content + - name: 主版本必须匹配 + ansible.builtin.assert: + that: (pg_existing_version_content.content | b64decode | trim) == (pg_major | string) diff --git a/infrastructure/shared-postgresql/ansible/tasks/renew-instance.yml b/infrastructure/shared-postgresql/ansible/tasks/renew-instance.yml new file mode 100644 index 0000000..2b6b99b --- /dev/null +++ b/infrastructure/shared-postgresql/ansible/tasks/renew-instance.yml @@ -0,0 +1,56 @@ +--- +- name: 设置现有实例路径 + ansible.builtin.set_fact: + pg_profile: "{{ pg_profiles[pg_instance] }}" + pg_config_dir: "{{ pg_config_root }}/{{ pg_instance }}" + pg_socket_dir: "/run/homelab-postgresql-{{ pg_instance }}" + pg_certificates_changed: false +- name: 续签前必须存在运行中的受管服务 + ansible.builtin.command: + argv: [systemctl, is-active, "homelab-postgresql-{{ pg_instance }}.service"] + changed_when: false +- name: 检查服务器证书续签 + ansible.builtin.include_role: + name: pg_instance + tasks_from: certificate + vars: + pg_cert_name: server + pg_cert_cn: "{{ pg_profile.dns }}" + pg_cert_role: "homelab-pg-{{ pg_instance }}" + pg_cert_eku: [serverAuth] +- name: 检查生产 DCS 客户端证书续签 + ansible.builtin.include_role: + name: pg_instance + tasks_from: certificate + vars: + pg_cert_name: etcd + pg_cert_cn: '' + pg_cert_role: homelab-etcd-client + pg_cert_eku: [clientAuth] + when: pg_instance == 'prod' +- name: 重载已续签实例(Patroni 或 PostgreSQL 主进程接收 HUP) + ansible.builtin.command: + argv: [systemctl, kill, --kill-whom=main, --signal=HUP, "homelab-postgresql-{{ pg_instance }}.service"] + when: pg_certificates_changed + changed_when: true +- name: 检查重载后的本机 SQL 可用 + ansible.builtin.command: + argv: + - "{{ pg_bin_dir }}/psql" + - -X + - -At + - -h + - "{{ pg_socket_dir }}" + - -p + - "{{ pg_profile.port | string }}" + - -d + - postgres + - -c + - SELECT 1 + become: true + become_user: "{{ pg_profile.user }}" + changed_when: false + register: pg_renew_sql + retries: 6 + delay: 2 + until: pg_renew_sql.rc == 0 and pg_renew_sql.stdout | trim == '1' diff --git a/infrastructure/shared-postgresql/ayatori/admin-credentials.yaml b/infrastructure/shared-postgresql/ayatori/admin-credentials.yaml new file mode 100644 index 0000000..44849d9 --- /dev/null +++ b/infrastructure/shared-postgresql/ayatori/admin-credentials.yaml @@ -0,0 +1,45 @@ +# SecretStore 由独立控制面的部署管理,必须只读下述两个管理凭据路径。 +apiVersion: external-secrets.io/v1 +kind: ExternalSecret +metadata: + name: homelab-postgresql-prod-admin +spec: + refreshInterval: 1h + secretStoreRef: + name: homelab-postgresql-admin + kind: SecretStore + target: + name: homelab-postgresql-prod-admin + creationPolicy: Owner + data: + - secretKey: username + remoteRef: + key: infra/postgresql/ayatori/prod + property: username + - secretKey: password + remoteRef: + key: infra/postgresql/ayatori/prod + property: password +--- +# SecretStore 由独立控制面的部署管理,必须只读下述两个管理凭据路径。 +apiVersion: external-secrets.io/v1 +kind: ExternalSecret +metadata: + name: homelab-postgresql-dev-admin +spec: + refreshInterval: 1h + secretStoreRef: + name: homelab-postgresql-admin + kind: SecretStore + target: + name: homelab-postgresql-dev-admin + creationPolicy: Owner + data: + - secretKey: username + remoteRef: + key: infra/postgresql/ayatori/dev + property: username + - secretKey: password + remoteRef: + key: infra/postgresql/ayatori/dev + property: password diff --git a/infrastructure/shared-postgresql/ayatori/ca.crt b/infrastructure/shared-postgresql/ayatori/ca.crt new file mode 100644 index 0000000..4d106c8 --- /dev/null +++ b/infrastructure/shared-postgresql/ayatori/ca.crt @@ -0,0 +1,20 @@ +-----BEGIN CERTIFICATE----- +MIIDMzCCAhugAwIBAgIUMs0iV657yC9UhA2p2vomLIbFnzgwDQYJKoZIhvcNAQEL +BQAwITEfMB0GA1UEAxMWZGR1cGFuLnRvcCBJbnRlcm5hbCBDQTAeFw0yNjA3MjQy +MDE1MDFaFw0zNjA3MjEyMDE1MzFaMCExHzAdBgNVBAMTFmRkdXBhbi50b3AgSW50 +ZXJuYWwgQ0EwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQC6QWlwBe6f +t7Ca3KCTvr4Pz+jVO60WrMBoEDYYM8Mp04btBHzhAQHf9Pp8+15aEW9iUcQhqqm+ +2vT6H0JEhIbplyCWY6Guv0mTu8f+lvFknJIl2b3JqnMLHJKjh/rBrsE12XZ3i17M +2tCr34BWcei85IZyQl5HMW6dB8lAE6bdom+YynK4oLJdej9DD6bSyM8WcL0OsneZ +NsjwOlNMy3zjbtaH6mH71SgbFinxLp3AAAuLVe1DIKhFxuTQeVr/WaPum5y/oOsc +0gJp9If6nsC33lpRGcPLiZE9kfFZa4fPe8laCaN8q1K253qZ0rjRiDhbTAppW4Fy +r5P67h+2D+TbAgMBAAGjYzBhMA4GA1UdDwEB/wQEAwIBBjAPBgNVHRMBAf8EBTAD +AQH/MB0GA1UdDgQWBBSOgk1fR0qhz/Bo4wD9g2BnOAzDXzAfBgNVHSMEGDAWgBSO +gk1fR0qhz/Bo4wD9g2BnOAzDXzANBgkqhkiG9w0BAQsFAAOCAQEANm5kKkts1Ar2 +7IlS+TxLFrZ/C9yhIdGcBk2SL5E+5E8S3skQWLEPGLRwvV4RmiB8gQ2V6UyGLrCx +1MuuSmCDaSYL9G66sGX1MIHlQ0F0bHIOxxtsTwIYzb5Sl8h3MfsARabmOhE3xUkn +jaAT9YUweHhjF4vi0U1Q4F8oOSvu4eJp5dMx1r7b2bLN90A1xh9sfdkEenSBX0tm +xK82ROYXI2Ejv/EO+lPUIn3jfqbqrS2itw75Xz/ECHjIfSxvW98puP69U54a1gf6 +gWdXslr0pGkyMHqxw4dmaecpK0QK3jvqCNycNwNBfMdCypS2QRy03adcUosEAP3O +LZU7Kd8aeg== +-----END CERTIFICATE----- \ No newline at end of file diff --git a/infrastructure/shared-postgresql/ayatori/eso-policy.hcl b/infrastructure/shared-postgresql/ayatori/eso-policy.hcl new file mode 100644 index 0000000..0d3772c --- /dev/null +++ b/infrastructure/shared-postgresql/ayatori/eso-policy.hcl @@ -0,0 +1,3 @@ +# 供未来独立控制面管理 SecretStore 的 ESO 身份绑定;本文件尚未应用。 +path "kv/data/infra/postgresql/ayatori/prod" { capabilities = ["read"] } +path "kv/data/infra/postgresql/ayatori/dev" { capabilities = ["read"] } diff --git a/infrastructure/shared-postgresql/ayatori/instances.yaml b/infrastructure/shared-postgresql/ayatori/instances.yaml new file mode 100644 index 0000000..11c849f --- /dev/null +++ b/infrastructure/shared-postgresql/ayatori/instances.yaml @@ -0,0 +1,28 @@ +# 尚未 apply:须先准备管理 Secret、CA bundle 和角色感知的生产稳定入口。 +apiVersion: database.ayatori.ddupan.top/v1alpha1 +kind: PostgreSQLInstance +metadata: + name: homelab-prod +spec: + endpoint: + host: pg-prod.ad.ddupan.top + hostaddr: 192.168.10.2 + port: 5432 + database: postgres + sslMode: verify-full + adminCredentialRef: + name: homelab-postgresql-prod-admin +--- +apiVersion: database.ayatori.ddupan.top/v1alpha1 +kind: PostgreSQLInstance +metadata: + name: homelab-dev +spec: + endpoint: + host: pg-dev.ad.ddupan.top + hostaddr: 192.168.10.127 + port: 5433 + database: postgres + sslMode: verify-full + adminCredentialRef: + name: homelab-postgresql-dev-admin diff --git a/infrastructure/shared-postgresql/ayatori/kustomization.yaml b/infrastructure/shared-postgresql/ayatori/kustomization.yaml new file mode 100644 index 0000000..8c4d43d --- /dev/null +++ b/infrastructure/shared-postgresql/ayatori/kustomization.yaml @@ -0,0 +1,12 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +# 仅供未来独立 Ayatori 控制面;此 namespace 必须与 manager 的 Secret namespace 一致。 +namespace: ayatori-system +resources: + - instances.yaml + - admin-credentials.yaml +configMapGenerator: + - name: homelab-postgresql-ca + files: [ca.crt] +generatorOptions: + disableNameSuffixHash: true diff --git a/infrastructure/shared-postgresql/run.py b/infrastructure/shared-postgresql/run.py new file mode 100644 index 0000000..8bb3a81 --- /dev/null +++ b/infrastructure/shared-postgresql/run.py @@ -0,0 +1,88 @@ +#!/usr/bin/env python3 +"""从显式登录会话或本机 SPIFFE 获取 Bao 凭据,仅传入子进程内存。""" +import json +import os +from pathlib import Path +import subprocess +import sys +import urllib.request +import urllib.error + +ROOT = Path(__file__).resolve().parent + + +def bao_request(address, path, body, token=None): + headers = {'Content-Type': 'application/json'} + if token: + headers['X-Vault-Token'] = token + request = urllib.request.Request( + address.rstrip('/') + '/v1/' + path, + data=json.dumps(body).encode(), headers=headers, + ) + with urllib.request.urlopen(request, timeout=20) as response: + return json.load(response) if response.status != 204 else None + + +def spiffe_token(address): + response = subprocess.run( + ['spire-agent', 'api', 'fetch', 'jwt', '-audience', 'openbao', + '-socketPath', '/run/spire/agent-sockets/spire-agent.sock', '-output', 'json'], + capture_output=True, text=True, + ) + if response.returncode: + raise SystemExit('本机 SPIFFE 身份获取失败;未回退到其他身份。') + try: + jwt = json.loads(response.stdout)[0]['svids'][0]['svid'] + return bao_request(address, 'auth/jwt-spire/login', + {'role': os.environ.get('PG_BAO_SPIFFE_ROLE', 'local-development'), 'jwt': jwt})['auth']['client_token'] + except (ValueError, KeyError, IndexError, urllib.error.URLError): + raise SystemExit('本机 SPIFFE 登录 Bao 失败;未输出秘密。') from None + + +def main(): + args = sys.argv[1:] + use_spiffe = bool(args and args[0] == '--spiffe') + if use_spiffe: + args.pop(0) + if len(args) < 2 or args[0] not in ('terraform', 'ansible'): + raise SystemExit('用法:python3 run.py [--spiffe] terraform | ansible [args]') + env = dict(os.environ) + env.setdefault('BAO_ADDR', 'https://bao.ad.ddupan.top:8200') + token = spiffe_token(env['BAO_ADDR']) if use_spiffe else ( + env.get('BAO_TOKEN') or env.get('VAULT_TOKEN') + or Path('~/.vault-token').expanduser().read_text().strip()) + try: + return run_command(args, env, token) + finally: + if use_spiffe: + try: + bao_request(env['BAO_ADDR'], 'auth/token/revoke-self', {}, token) + except urllib.error.URLError: + print('短期会话撤销失败,将按服务器 TTL 到期。', file=sys.stderr) + + +def run_command(args, env, token): + env['BAO_TOKEN'] = env['VAULT_TOKEN'] = token + if args[0] == 'terraform': + response = subprocess.run( + ['bao', 'kv', 'get', '-format=json', 'kv/k8s/seaweedfs-s3'], + env=env, capture_output=True, text=True, + ) + if response.returncode: + raise SystemExit('读取 tfstate 受限身份失败;请检查 Bao 登录和授权。') + config = json.loads(response.stdout)['data']['data']['seaweedfs_s3_config'] + config = json.loads(config) if isinstance(config, str) else config + identities = [i for i in config['identities'] if i['name'] == 'terraform'] + if len(identities) != 1 or len(identities[0]['credentials']) != 1: + raise SystemExit('tfstate 身份不唯一,拒绝猜测凭据。') + credential = identities[0]['credentials'][0] + env['AWS_ACCESS_KEY_ID'] = credential['accessKey'] + env['AWS_SECRET_ACCESS_KEY'] = credential['secretKey'] + command, cwd = ['terraform', *args[1:]], ROOT / 'terraform' + else: + command, cwd = ['ansible-playbook', *args[1:]], ROOT / 'ansible' + return subprocess.run(command, cwd=cwd, env=env).returncode + + +if __name__ == '__main__': + raise SystemExit(main()) diff --git a/infrastructure/shared-postgresql/terraform/.terraform.lock.hcl b/infrastructure/shared-postgresql/terraform/.terraform.lock.hcl new file mode 100644 index 0000000..234d8fc --- /dev/null +++ b/infrastructure/shared-postgresql/terraform/.terraform.lock.hcl @@ -0,0 +1,22 @@ +# This file is maintained automatically by "terraform init". +# Manual edits may be lost in future updates. + +provider "registry.terraform.io/hashicorp/vault" { + version = "4.8.0" + constraints = "~> 4.0" + hashes = [ + "h1:aHqgWQhDBMeZO9iUKwJYMlh4q+xNMUlMIcjRbF4d02Y=", + "zh:269ab13433f67684012ae7e15876532b0312f5d0d2002a9cf9febb1279ce5ea6", + "zh:4babc95bf0c40eb85005db1dc2ca403c46be4a71dd3e409db3711a56f7a5ca0e", + "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", + "zh:86e27c1c625ecc24446a11eeffc3ac319b36c2b4e51251db8579256a0dbcf136", + "zh:a32f31da94824009e26b077374440b52098aecb93c92ff55dc3d31dd37c4ea25", + "zh:be0a18c6c0425518bab4fbffd82078b82036a88503b5d76064de551c9f646cbf", + "zh:be5a77fdfd36863ebeec79cd12b1d13322ffad6821d157a0b279789fa06b5937", + "zh:be8317d142a3caad74c7d936039ae27076a1b2b8312ef5208e2871a5f525977c", + "zh:c94a84895a3d9954b80e983eed4603330a5cdbbd8eef5b3c99278c2d1402ef3c", + "zh:de1fb712784dd8415f011ca5346a34f87fab6046c730557615247e511dbc7d98", + "zh:e3eafae7da550f86cae395d6660b2a0e93ec8d2b0e0e5ef982ec762e961fc952", + "zh:ff35fb1ab6add288f0f368981e56f780b50405accd1937131cba1137999c8d83", + ] +} diff --git a/infrastructure/shared-postgresql/terraform/backend.tf b/infrastructure/shared-postgresql/terraform/backend.tf new file mode 100644 index 0000000..b295fed --- /dev/null +++ b/infrastructure/shared-postgresql/terraform/backend.tf @@ -0,0 +1,15 @@ +# 与既有服务复用受限 tfstate 身份,使用独立对象与原生锁;不复用 Bao 的 state。 +terraform { + backend "s3" { + bucket = "tfstate" + key = "shared-postgresql/terraform.tfstate" + endpoints = { s3 = "https://s3.ad.ddupan.top" } + region = "us-east-1" + use_path_style = true + skip_credentials_validation = true + skip_metadata_api_check = true + skip_region_validation = true + skip_requesting_account_id = true + use_lockfile = true + } +} diff --git a/infrastructure/shared-postgresql/terraform/identity.tf b/infrastructure/shared-postgresql/terraform/identity.tf new file mode 100644 index 0000000..be96347 --- /dev/null +++ b/infrastructure/shared-postgresql/terraform/identity.tf @@ -0,0 +1,35 @@ +# 复用现有 JWT-SVID 验证挂载,独立身份仅授权此服务所需路径。 +resource "vault_policy" "postgresql" { + name = "homelab-postgresql-provisioner" + policy = <<-EOT + path "pki/sign/homelab-pg-prod" { capabilities = ["update"] } + path "pki/sign/homelab-pg-dev" { capabilities = ["update"] } + path "pki/sign/homelab-etcd-client" { capabilities = ["update"] } + path "kv/data/infra/postgresql/prod" { capabilities = ["create", "read", "update"] } + path "kv/data/infra/postgresql/dev" { capabilities = ["create", "read", "update"] } + path "kv/metadata/infra/postgresql/prod" { capabilities = ["read"] } + path "kv/metadata/infra/postgresql/dev" { capabilities = ["read"] } + path "kv/data/infra/etcd/consumers/patroni-pg-prod" { capabilities = ["read"] } + path "kv/data/infra/postgresql/ayatori/prod" { capabilities = ["create", "read", "update"] } + path "kv/data/infra/postgresql/ayatori/dev" { capabilities = ["create", "read", "update"] } + path "kv/metadata/infra/postgresql/ayatori/prod" { capabilities = ["read"] } + path "kv/metadata/infra/postgresql/ayatori/dev" { capabilities = ["read"] } + path "auth/token/lookup-self" { capabilities = ["read"] } + path "auth/token/revoke-self" { capabilities = ["update"] } + EOT +} + +resource "vault_jwt_auth_backend_role" "postgresql" { + backend = "jwt-spire" + role_name = "homelab-postgresql" + role_type = "jwt" + user_claim = "sub" + bound_audiences = ["openbao"] + bound_claims = { + sub = "spiffe://ddupan.top/dev/panxiao81" + } + token_policies = [vault_policy.postgresql.name] + token_no_default_policy = true + token_ttl = 900 + token_max_ttl = 900 +} diff --git a/infrastructure/shared-postgresql/terraform/main.tf b/infrastructure/shared-postgresql/terraform/main.tf new file mode 100644 index 0000000..b47f013 --- /dev/null +++ b/infrastructure/shared-postgresql/terraform/main.tf @@ -0,0 +1,30 @@ +terraform { + required_version = ">= 1.10" + required_providers { + vault = { source = "hashicorp/vault", version = "~> 4.0" } + } +} +provider "vault" { + address = "https://bao.ad.ddupan.top:8200" + # 使用包装器取得的短期 token;它负责执行后撤销,不要求额外创建子 token。 + skip_child_token = true +} +# 只管理新实例的签发角色;不保存 CA 私钥、实例密码或 etcd 账号。 +resource "vault_pki_secret_backend_role" "postgresql" { + for_each = toset(["prod", "dev"]) + backend = "pki" + name = "homelab-pg-${each.key}" + allowed_domains = ["pg-${each.key}.ad.ddupan.top"] + allow_bare_domains = true + allow_subdomains = false + allow_any_name = false + allow_localhost = false + allow_wildcard_certificates = false + allow_ip_sans = true + server_flag = true + client_flag = false + key_type = "ec" + key_bits = 256 + ttl = 5184000 + max_ttl = 5184000 +} diff --git a/infrastructure/shared-postgresql/tests/Dockerfile b/infrastructure/shared-postgresql/tests/Dockerfile new file mode 100644 index 0000000..82eb8a8 --- /dev/null +++ b/infrastructure/shared-postgresql/tests/Dockerfile @@ -0,0 +1,4 @@ +FROM postgres:18-bookworm +RUN apt-get update && apt-get install -y --no-install-recommends python3-venv pgbackrest openssh-client haproxy && rm -rf /var/lib/apt/lists/* +RUN python3 -m venv /opt/patroni && /opt/patroni/bin/pip install --no-cache-dir 'patroni[etcd3,psycopg3]==4.1.5' +ENTRYPOINT [] diff --git a/infrastructure/shared-postgresql/tests/integration.py b/infrastructure/shared-postgresql/tests/integration.py new file mode 100644 index 0000000..98bfeb5 --- /dev/null +++ b/infrastructure/shared-postgresql/tests/integration.py @@ -0,0 +1,195 @@ +#!/usr/bin/env python3 +"""真实 PG/Patroni/HAProxy/pgBackRest;仅 loopback、临时测试 CA/密码,不访问生产。""" +import json +import os +from pathlib import Path +import shutil +import ssl +import subprocess +import tempfile +import time +import urllib.request + +import jinja2 +import yaml + +ROOT = Path(__file__).resolve().parents[1] +ETCD = Path(os.environ.get('ETCD_TEST_BIN', '/tmp/etcd-v3.7.2-linux-amd64')) +IMAGE = os.environ.get('PG_TEST_IMAGE', 'homelab-pg-test:18-patroni4.1.5') + + +def run(args, **kw): + return subprocess.run([str(x) for x in args], capture_output=True, text=True, check=True, **kw) + + +def wait_for(check, timeout=90): + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + try: + result = check() + if result: + return result + except (subprocess.CalledProcessError, OSError, ValueError): + pass + time.sleep(1) + raise RuntimeError('timed out waiting for fixture state') + + +def main(): + os.environ['NO_PROXY'] = '*' + processes, handles, containers = [], [], [] + with tempfile.TemporaryDirectory(prefix='shared-pg-test-') as tmp: + w = Path(tmp) + pg = w/'pg'; pg.mkdir(); pg.chmod(0o755) + prefix = f'shared-pg-test-{os.getpid()}' + env = jinja2.Environment(loader=jinja2.FileSystemLoader(ROOT/'ansible/roles/pg_instance/templates'), undefined=jinja2.StrictUndefined) + env.filters['to_json'] = json.dumps + try: + run(['openssl', 'req', '-x509', '-newkey', 'rsa:2048', '-nodes', '-keyout', w/'ca.key', '-out', w/'ca.crt', '-days', '1', '-subj', '/CN=pg-isolated-ca']) + + def cert(directory, name, cn, eku, sans=''): + directory.mkdir(parents=True, exist_ok=True) + run(['openssl','req','-new','-newkey','rsa:2048','-nodes','-keyout',directory/f'{name}.key','-out',directory/f'{name}.csr','-subj',f'/CN={cn}' if cn else '/']) + ext = directory/f'{name}.ext'; ext.write_text(f'extendedKeyUsage={eku}\n'+(f'subjectAltName={sans}\n' if sans else '')) + run(['openssl','x509','-req','-in',directory/f'{name}.csr','-CA',w/'ca.crt','-CAkey',w/'ca.key','-CAcreateserial','-out',directory/f'{name}.crt','-days','1','-extfile',ext]) + if directory != w: + shutil.copy(w/'ca.crt',directory/'ca.crt') + + # 独立三成员 DCS,绝不复用生产 prefix 或端点。 + names = ['e1','e2','e3'] + hosts = {n:{'etcd_address':f'127.0.1.{i+1}'} for i,n in enumerate(names)} + et = jinja2.Environment(loader=jinja2.FileSystemLoader(ROOT.parent/'etcd/ansible/roles/shared_etcd/templates'),undefined=jinja2.StrictUndefined) + et.filters['to_json']=json.dumps + defaults=yaml.safe_load((ROOT.parent/'etcd/ansible/roles/shared_etcd/defaults/main.yml').read_text()) + cert(w,'admin','root','clientAuth') + for name in names: + d=w/name; ip=hosts[name]['etcd_address'] + cert(d,'server',name,'serverAuth',f'IP:{ip},IP:127.0.0.1') + cert(d,'peer','homelab-etcd-peer','serverAuth,clientAuth',f'IP:{ip},IP:127.0.0.1') + cert(d,'gateway','','clientAuth') + cfg=et.get_template('etcd.yml.j2').render(**dict(defaults,inventory_hostname=name,groups={'etcd':names},hostvars=hosts,etcd_address=ip,etcd_config_dir=str(d),etcd_data_dir=str(d/'data'),etcd_client_port=23379,etcd_peer_port=23380,etcd_metrics_urls=f'http://{ip}:23381')) + (d/'config.yml').write_text(cfg); log=(d/'log').open('w');handles.append(log) + processes.append(subprocess.Popen([str(ETCD/'etcd'),'--config-file='+str(d/'config.yml')],stdout=log,stderr=log)) + ctlbase=[ETCD/'etcdctl','--endpoints=https://127.0.1.1:23379','--cacert='+str(w/'ca.crt'),'--cert='+str(w/'admin.crt'),'--key='+str(w/'admin.key'),'--command-timeout=3s'] + wait_for(lambda: run(ctlbase+['endpoint','health'])) + run(ctlbase+['user','add','root','--no-password']);run(ctlbase+['user','grant-role','root','root']);run(ctlbase+['auth','enable']) + run(ctlbase+['user','add','patroni-pg-prod','--interactive=false'],input='fixture-etcd-password\n') + run(ctlbase+['role','add','patroni-pg-prod']);run(ctlbase+['role','grant-permission','patroni-pg-prod','readwrite','/homelab/patroni/pg-prod/','--prefix=true']);run(ctlbase+['user','grant-role','patroni-pg-prod','patroni-pg-prod']) + values={} + for index,name in enumerate(['p1','p2','dev']): + d=pg/name;cfg=d/'config';cfg.mkdir(parents=True) + address=f'127.0.2.{index+1}' + cert(cfg,'server','pg-fixture.test','serverAuth',f'DNS:pg-fixture.test,IP:{address}') + cert(cfg,'etcd','','clientAuth','DNS:etcd-patroni-pg-prod') + v=dict(pg_instance='dev' if name=='dev' else 'prod',pg_member=name,pg_address=address, + pg_profile=dict(user='postgres',port=25432,api_port=28008,shared_buffers='32MB',max_connections=30), + pg_credentials=dict(superuser_password='fixture-superuser-password',replication_password='fixture-replication-password',rest_password='fixture-rest-password'), + pg_etcd_credentials=dict(username='patroni-pg-prod',password='fixture-etcd-password'), + pg_etcd_endpoints=[f'{h["etcd_address"]}:23379' for h in hosts.values()], + pg_config_dir='/node/config',pg_parent_dir='/node',pg_data_dir='/node/data',pg_socket_dir='/node/socket', + pg_bin_dir='/usr/lib/postgresql/18/bin',pg_replication_addresses=['127.0.0.0/8'],pg_client_cidrs=['127.0.0.0/8'], + pg_repo_remote=False,pg_repo_path='/repo') + values[name]=v + for template in (['postgresql.conf','pg_hba.conf'] if name=='dev' else ['patroni.yml','pgbackrest.conf']): + (cfg/template).write_text(env.get_template(template+'.j2').render(**v)) + for folder in ['socket','backup-lock','backup-spool']:(d/folder).mkdir() + (pg/'repo').mkdir();(pg/'restore').mkdir();(pg/'pitr').mkdir() + uid=os.getuid();gid=os.getgid() + run(['docker','run','--rm','-v',f'{pg}:/fixture',IMAGE,'chown','-R','999:999','/fixture']) + + def start(name, command): + cname=prefix+'-'+name + args=['docker','run','-d','--name',cname,'--network','host','--user','999:999','--memory','512m','--cpus','1', + '-v',f'{pg/name}:/node','-v',f'{pg/"repo"}:/repo','-v',f'{pg/"restore"}:/restore','-v',f'{pg/"pitr"}:/pitr',IMAGE,*command] + run(args);containers.append(cname);return cname + + def sql(name, query, tcp=False, password='fixture-superuser-password', user='postgres'): + command=['docker','exec','-i',prefix+'-'+name,'psql','-X','-v','ON_ERROR_STOP=1','-At'] + if tcp: + command+=['host=pg-fixture.test hostaddr='+values[name]['pg_address']+' port=25432 user='+user+' dbname=postgres sslmode=verify-full sslrootcert=/node/config/ca.crt password='+password] + else: command+=['-h','/node/socket','-p','25432','-U','postgres','postgres'] + return run(command,input=query).stdout.strip() + + start('p1',['/opt/patroni/bin/patroni','/node/config/patroni.yml']) + wait_for(lambda: sql('p1','SELECT NOT pg_is_in_recovery()')=='t') + start('p2',['/opt/patroni/bin/patroni','/node/config/patroni.yml']) + wait_for(lambda: sql('p2','SELECT pg_is_in_recovery()')=='t') + assert sql('p1','SHOW max_connections')=='30' + print('PASS: PG 18 primary and streaming standby through authenticated mTLS etcd3',flush=True) + # 开发实例直接使用同套配置模板;与生产不共享数据目录或 DCS。 + run(['docker','run','--rm','--user','999:999','-v',f'{pg/"dev"}:/node',IMAGE,'/usr/lib/postgresql/18/bin/initdb','-D','/node/data','--username=postgres','--auth-local=peer','--auth-host=scram-sha-256']) + start('dev',['/usr/lib/postgresql/18/bin/postgres','-D','/node/data','-c','config_file=/node/config/postgresql.conf']) + wait_for(lambda: sql('dev','SELECT 1')=='1') + sql('p1',"CREATE TABLE probe(id integer primary key); INSERT INTO probe VALUES (1); CREATE ROLE ayatori LOGIN NOSUPERUSER CREATEDB CREATEROLE PASSWORD 'fixture-ayatori-password';") + assert sql('p1',"SELECT NOT rolsuper AND rolcreatedb AND rolcreaterole FROM pg_roles WHERE rolname=current_user",True,'fixture-ayatori-password','ayatori')=='t' + sql('p1',"CREATE ROLE tenant_owner NOLOGIN; GRANT tenant_owner TO ayatori WITH SET TRUE; CREATE DATABASE tenant_probe OWNER tenant_owner;",True,'fixture-ayatori-password','ayatori') + assert sql('dev',"SELECT to_regclass('public.probe') IS NULL")=='t' + wait_for(lambda: sql('p2','SELECT count(*) FROM probe')=='1') + # 相同 stanza 在两节点上使用同一隔离测试仓库;生产 SSH 传输另行验收。 + def backrest(name,*args):return run(['docker','exec',prefix+'-'+name,'pgbackrest','--config=/node/config/pgbackrest.conf','--stanza=prod',*args]) + backrest('p1','stanza-create');backrest('p1','check');backrest('p1','--type=full','backup') + initial_backup=json.loads(backrest('p1','--output=json','info').stdout)[0]['backup'][-1]['label'] + recovery_target=sql('p1','SELECT clock_timestamp()::text') + print('PASS: separate dev, TLS verify-full, non-superuser Ayatori native privileges, initial backup',flush=True) + proxycfg=w/'proxy.cfg' + # 文件父目录仍由调用者拥有,新增代理配置不触碰 PG 文件。 + proxycfg.write_text(env.get_template('haproxy.cfg.j2').render(pg_proxy_bind='127.0.2.10:26432',pg_proxy_members=[dict(name=n,address=values[n]['pg_address'],port=25432,api_port=28008) for n in ['p1','p2']])) + cname=prefix+'-proxy';run(['docker','run','-d','--name',cname,'--network','host','-v',f'{proxycfg}:/proxy.cfg:ro',IMAGE,'haproxy','-f','/proxy.cfg','-db']);containers.append(cname) + def proxy_sql():return run(['docker','exec',prefix+'-dev','psql','-X','-At','host=127.0.2.10 port=26432 user=ayatori dbname=postgres sslmode=verify-ca sslrootcert=/node/config/ca.crt password=fixture-ayatori-password','-c','SELECT pg_is_in_recovery()']).stdout.strip() + wait_for(lambda: proxy_sql()=='f') + began=time.monotonic();run(['docker','kill',prefix+'-p1']) + wait_for(lambda: sql('p2','SELECT NOT pg_is_in_recovery()')=='t') + wait_for(lambda: proxy_sql()=='f') + rto=time.monotonic()-began + sql('p2','INSERT INTO probe VALUES (2)');backrest('p2','check');backrest('p2','--type=full','backup') + assert sql('dev','SELECT 1')=='1' + print(f'PASS: automatic failover + proxy routing in {rto:.1f}s; dev unaffected; backup after failover',flush=True) + run(['docker','start',prefix+'-p1']) + wait_for(lambda: sql('p1','SELECT pg_is_in_recovery()')=='t') + wait_for(lambda: sql('p1','SELECT count(*) FROM probe')=='2') + assert sql('p2','SELECT NOT pg_is_in_recovery()')=='t' + print('PASS: old primary rejoins as replica without stealing leadership',flush=True) + backrest('p2','--pg1-path=/restore','--type=immediate','restore') + # 恢复出的副本不连接 DCS,不向生产/测试 stanza 继续归档。 + cname=prefix+'-restore';run(['docker','run','-d','--name',cname,'--network','host','--user','999:999','-v',f'{pg/"restore"}:/restore','-v',f'{pg/"p2"}:/node','-v',f'{pg/"repo"}:/repo',IMAGE,'postgres','-D','/restore','-c','listen_addresses=127.0.2.20','-p','27432','-c','unix_socket_directories=/tmp','-c','archive_mode=off','-c','hot_standby=on']);containers.append(cname) + def restored():return run(['docker','exec',cname,'psql','-X','-At','-h','/tmp','-p','27432','-U','postgres','postgres','-c','SELECT count(*) FROM probe']).stdout.strip() + wait_for(lambda: restored()=='2') + print('PASS: isolated restore contains both pre/post-failover rows',flush=True) + backrest('p2','--pg1-path=/pitr','--set='+initial_backup,'--type=time', + '--target='+recovery_target,'--target-action=promote','restore') + cname=prefix+'-pitr' + run(['docker','run','-d','--name',cname,'--network','host','--user','999:999', + '-v',f'{pg/"pitr"}:/pitr','-v',f'{pg/"p2"}:/node','-v',f'{pg/"repo"}:/repo',IMAGE, + 'postgres','-D','/pitr','-c','listen_addresses=127.0.2.21','-p','27433', + '-c','unix_socket_directories=/tmp','-c','archive_mode=off']);containers.append(cname) + def pitr_rows(): + return run(['docker','exec',cname,'psql','-X','-At','-h','/tmp','-p','27433', + '-U','postgres','postgres','-c','SELECT count(*) FROM probe']).stdout.strip() + wait_for(lambda: pitr_rows()=='1') + print('PASS: WAL point-in-time restore recovers pre-failover state from initial backup',flush=True) + for process in processes[1:]: + process.terminate(); process.wait(timeout=10) + def write_fenced(): + try: + sql('p2','INSERT INTO probe VALUES (3) ON CONFLICT DO NOTHING') + return False + except subprocess.CalledProcessError: + return True + wait_for(write_fenced) + print('PASS: DCS majority loss eventually prevents primary writes (failsafe disabled)',flush=True) + except Exception: + for container in containers: + result=subprocess.run(['docker','logs','--tail','15',container],capture_output=True,text=True) + print(container,result.stdout,result.stderr) + raise + finally: + for container in reversed(containers):subprocess.run(['docker','rm','-f',container],capture_output=True) + for p in processes: + p.terminate() + try:p.wait(timeout=10) + except subprocess.TimeoutExpired:p.kill();p.wait() + for h in handles:h.close() + subprocess.run(['docker','run','--rm','-v',f'{pg}:/fixture',IMAGE,'chown','-R',f'{os.getuid()}:{os.getgid()}','/fixture'],capture_output=True) + + +if __name__=='__main__':main() diff --git a/infrastructure/shared-postgresql/tests/live_failover.py b/infrastructure/shared-postgresql/tests/live_failover.py new file mode 100644 index 0000000..bc3aa2d --- /dev/null +++ b/infrastructure/shared-postgresql/tests/live_failover.py @@ -0,0 +1,112 @@ +#!/usr/bin/env python3 +"""显式演练新 PG:停止 laptop 主库服务,验证代理自动切换,再回切。需 --run。""" +import importlib.util +import json +import os +import shlex +from pathlib import Path +import subprocess +import sys +import time +import urllib.request + +ROOT = Path(__file__).resolve().parents[1] +spec = importlib.util.spec_from_file_location('pg_runner', ROOT / 'run.py') +runner = importlib.util.module_from_spec(spec) +spec.loader.exec_module(runner) + + +def command(args, **kw): + return subprocess.run(args, capture_output=True, text=True, timeout=35, **kw) + + +def wait_until(fn, description, timeout=100): + end = time.monotonic() + timeout + while time.monotonic() < end: + try: + if fn(): + print(description, flush=True) + return + except (OSError, ValueError, subprocess.TimeoutExpired): + pass + time.sleep(2) + raise RuntimeError(description + ' timeout') + + +def role(ip): + with urllib.request.urlopen('http://' + ip + ':8008/patroni', timeout=3) as r: + return json.load(r)['role'] + + +def main(): + if sys.argv[1:] != ['--run']: + raise SystemExit('需显式 --run;会停止生产 laptop PG 服务并在验收后回切。') + address = 'https://bao.ad.ddupan.top:8200' + token = runner.spiffe_token(address) + database = 'homelab_ha_probe_' + str(int(time.time())) + try: + request = urllib.request.Request(address + '/v1/kv/data/infra/postgresql/ayatori/prod', + headers={'X-Vault-Token': token}) + with urllib.request.urlopen(request, timeout=10) as r: + credentials = json.load(r)['data']['data'] + env = dict(os.environ, PGHOST='pg-prod.ad.ddupan.top', PGHOSTADDR='192.168.10.2', + PGPORT='5432', PGDATABASE='postgres', PGUSER=credentials['username'], + PGPASSWORD=credentials['password'], PGSSLMODE='verify-full', + PGSSLROOTCERT=str(ROOT / 'ayatori/ca.crt'), PGCONNECT_TIMEOUT='3') + + def sql(query, db='postgres'): + result = command(['/usr/lib/postgresql/18/bin/psql', '-X', '-At', '-v', 'ON_ERROR_STOP=1'], + input=query, env=dict(env, PGDATABASE=db)) + if result.returncode: + raise RuntimeError('SQL 验证失败: ' + result.stderr) + return result.stdout.strip() + + def via(ip): + try: + return sql("SELECT host(inet_server_addr()) || ':' || " + "(NOT pg_is_in_recovery() AND current_setting('transaction_read_only')='off')::text") == ip + ':true' + except RuntimeError: + return False + + def rows(ip, count): + args = ['sudo', '-n', '-u', 'pgprod', '/usr/lib/postgresql/18/bin/psql', '-X', '-At', + '-h', '/run/homelab-postgresql-prod', '-d', database, '-c', 'SELECT count(*) FROM probe'] + if ip != '192.168.10.127': + args = ['ssh', 'root@' + ip, shlex.join(args)] + return command(args).stdout.strip() == str(count) + + assert role('192.168.10.127') in ('master', 'primary') + assert role('10.60.0.20') == 'replica' + assert via('192.168.10.127') + sql('CREATE DATABASE ' + database) + print('探针数据库:', database, flush=True) + sql('CREATE TABLE probe(id integer PRIMARY KEY); INSERT INTO probe VALUES(1)', database) + wait_until(lambda: rows('10.60.0.20', 1), '初始数据已复制') + started = time.monotonic() + result = command(['sudo', '-n', 'systemctl', 'stop', 'homelab-postgresql-prod']) + if result.returncode: + raise RuntimeError('停止主库服务失败') + wait_until(lambda: via('10.60.0.20'), '代理已自动切到 standby') + sql('INSERT INTO probe VALUES(2)', database) + print('自动恢复写入秒数:', round(time.monotonic() - started, 1), flush=True) + command(['sudo', '-n', 'systemctl', 'start', 'homelab-postgresql-prod'], check=True) + wait_until(lambda: role('192.168.10.127') == 'replica' and rows('192.168.10.127', 2), + '旧主重新作为 replica 加入,数据一致') + result = command(['sudo', '-n', '-u', 'pgprod', '/opt/homelab-patroni/bin/patronictl', + '-c', '/etc/homelab-postgresql/prod/patroni.yml', 'switchover', + '--leader', 'pg-pve1', '--candidate', 'pg-laptop', '--force']) + if result.returncode: + raise RuntimeError('计划回切失败: ' + result.stderr) + wait_until(lambda: via('192.168.10.127'), '已计划回切 laptop') + sql('INSERT INTO probe VALUES(3)', database) + wait_until(lambda: role('10.60.0.20') == 'replica' and rows('10.60.0.20', 3), '回切后复制正常') + sql('DROP DATABASE ' + database) + print('PASS;专属探针数据库已删除', flush=True) + finally: + # 即使演练失败,也确保原主服务重新运行;Patroni 自行决定角色,不强行提升。 + command(['sudo', '-n', 'systemctl', 'start', 'homelab-postgresql-prod']) + runner.bao_request(address, 'auth/token/revoke-self', {}, token) + + +if __name__ == '__main__': + main() diff --git a/infrastructure/shared-postgresql/tests/live_restore.py b/infrastructure/shared-postgresql/tests/live_restore.py new file mode 100644 index 0000000..e81a81f --- /dev/null +++ b/infrastructure/shared-postgresql/tests/live_restore.py @@ -0,0 +1,63 @@ +#!/usr/bin/env python3 +"""将真实 pgBackRest 备份恢复到临时目录,仅用私有 Unix socket 启动并验证。""" +import os +from pathlib import Path +import shutil +import subprocess +import sys +import tempfile +import time + + +def run(args, **kwargs): + return subprocess.run(['sudo', '-n', '-u', 'pgprod', *args], + check=True, capture_output=True, text=True, timeout=240, **kwargs) + + +def main(): + if sys.argv[1:] != ['--run']: + raise SystemExit('需显式 --run;仅恢复到新的临时目录,不接触原 PGDATA。') + root = Path(tempfile.mkdtemp(prefix='homelab-pg-restore-', dir='/tmp')) + data = root / 'data' + started = False + try: + (root / 'pg_hba.conf').write_text('local all pgprod peer\n') + (root / 'test.conf').write_text( + f"data_directory='{data}'\nhba_file='{root}/pg_hba.conf'\n" + f"unix_socket_directories='{root}'\nunix_socket_permissions=0700\n" + "listen_addresses=''\nport=55432\nssl=off\narchive_mode=off\n" + "shared_buffers=32MB\nmax_connections=80\nmax_wal_senders=5\n" + "hot_standby=on\nprimary_conninfo=''\n") + subprocess.run(['sudo', '-n', 'chown', '-R', 'pgprod:pgprod', str(root)], check=True) + run(['pgbackrest', '--config=/etc/homelab-postgresql/prod/pgbackrest.conf', '--stanza=prod', + '--pg1-path=' + str(data), '--type=immediate', '--target-action=promote', + '--archive-mode=off', 'restore']) + print('真实仓库备份已恢复到临时目录', flush=True) + run(['/usr/lib/postgresql/18/bin/pg_ctl', '-D', str(data), '-l', str(root / 'server.log'), + '-o', '-c config_file=' + str(root / 'test.conf'), '-w', '-t', '60', 'start']) + started = True + # pg_ctl ready 可早于 WAL recovery 完成;等待恢复结束并可写。 + for _ in range(60): + result = run(['/usr/lib/postgresql/18/bin/psql', '-X', '-At', '-h', str(root), '-p', '55432', + '-d', 'postgres', '-c', + "SELECT NOT pg_is_in_recovery() AND NOT rolsuper AND rolcreatedb AND rolcreaterole " + "FROM pg_roles WHERE rolname='ayatori'"]) + if result.stdout.strip() == 't': + break + time.sleep(1) + else: + raise RuntimeError('恢复后 SQL/管理角色验证未通过') + print('PASS:恢复实例可写、Ayatori 管理角色属性正确;仅私有 Unix socket', flush=True) + except subprocess.CalledProcessError as e: + print(e.stderr, file=sys.stderr) + raise + finally: + # pg_ctl start 超时也可能已经创建 postmaster,清理前必须先确认停止。 + if started or (data / 'postmaster.pid').exists(): + run(['/usr/lib/postgresql/18/bin/pg_ctl', '-D', str(data), '-m', 'immediate', '-w', 'stop']) + subprocess.run(['sudo', '-n', 'chown', '-R', str(os.getuid()) + ':' + str(os.getgid()), str(root)], check=True) + shutil.rmtree(root) + + +if __name__ == '__main__': + main() diff --git a/infrastructure/shared-postgresql/tests/test_runner.py b/infrastructure/shared-postgresql/tests/test_runner.py new file mode 100644 index 0000000..27b4ef8 --- /dev/null +++ b/infrastructure/shared-postgresql/tests/test_runner.py @@ -0,0 +1,40 @@ +"""验证短期会话边界:失败不回退、子进程失败也撤销;不使用真实秘密。""" +import importlib.util +from pathlib import Path +import unittest +from unittest.mock import patch + +spec = importlib.util.spec_from_file_location('pg_runner', Path(__file__).parents[1] / 'run.py') +runner = importlib.util.module_from_spec(spec) +spec.loader.exec_module(runner) + + +class RunnerTests(unittest.TestCase): + def test_revoke_on_child_failure(self): + with patch.object(runner.sys, 'argv', ['run.py', '--spiffe', 'ansible', 'credentials.yml']), \ + patch.object(runner, 'spiffe_token', return_value='test-token'), \ + patch.object(runner, 'run_command', side_effect=RuntimeError('child failed')), \ + patch.object(runner, 'bao_request') as request: + with self.assertRaises(RuntimeError): + runner.main() + self.assertEqual(request.call_args.args[1:], ('auth/token/revoke-self', {}, 'test-token')) + + def test_no_fallback_after_login_failure(self): + with patch.object(runner.sys, 'argv', ['run.py', '--spiffe', 'ansible', 'credentials.yml']), \ + patch.object(runner, 'spiffe_token', side_effect=SystemExit('login failed')), \ + patch.object(runner, 'run_command') as command: + with self.assertRaises(SystemExit): + runner.main() + command.assert_not_called() + + def test_revoke_on_success_and_preserve_exit_code(self): + with patch.object(runner.sys, 'argv', ['run.py', '--spiffe', 'ansible', 'credentials.yml']), \ + patch.object(runner, 'spiffe_token', return_value='test-token'), \ + patch.object(runner, 'run_command', return_value=7), \ + patch.object(runner, 'bao_request') as request: + self.assertEqual(runner.main(), 7) + request.assert_called_once() + + +if __name__ == '__main__': + unittest.main() diff --git a/platform/observability/metrics/kustomization.yaml b/platform/observability/metrics/kustomization.yaml index c0a5970..2ad314a 100644 --- a/platform/observability/metrics/kustomization.yaml +++ b/platform/observability/metrics/kustomization.yaml @@ -21,6 +21,10 @@ resources: - rules/kubernetes-health.yaml - rules/host-health.yaml - rules/monitoring-delivery.yaml + - rules/shared-etcd.yaml + - scrapes/shared-etcd.yaml + - rules/shared-postgresql.yaml + - scrapes/shared-postgresql.yaml - scrapes/platform-controllers.yaml - rules/platform-controllers.yaml - scrapes/cnpg.yaml diff --git a/platform/observability/metrics/rules/shared-etcd.yaml b/platform/observability/metrics/rules/shared-etcd.yaml new file mode 100644 index 0000000..e00cea4 --- /dev/null +++ b/platform/observability/metrics/rules/shared-etcd.yaml @@ -0,0 +1,61 @@ +# 参考 Pigsty v4.5.0 etcd 指标与阈值,适配原生 up 和现有标签;无 Pigsty UI 依赖。 +apiVersion: operator.victoriametrics.com/v1beta1 +kind: VMRule +metadata: + name: shared-etcd + namespace: monitoring +spec: + groups: + - name: shared-etcd + rules: + - alert: SharedEtcdMemberUnavailable + expr: up{job="shared-etcd"} == 0 + for: 2m + labels: + severity: warning + service: shared-etcd + annotations: + summary: 共享 etcd 成员 {{ $labels.member }} 无法采集 + - alert: SharedEtcdMonitoringQuorumUnavailable + expr: (sum(up{job="shared-etcd"}) or vector(0)) < 2 + for: 2m + labels: + severity: critical + service: shared-etcd + annotations: + summary: 共享 etcd 不足两个成员可采集,请用 endpoint health 核实 quorum + - alert: SharedEtcdNoLeader + expr: etcd_server_has_leader{job="shared-etcd"} == 0 + for: 1m + labels: + severity: critical + service: shared-etcd + annotations: + summary: 共享 etcd 成员 {{ $labels.member }} 无法确认 leader + - alert: SharedEtcdQuotaHigh + expr: etcd_mvcc_db_total_size_in_bytes{job="shared-etcd"} / etcd_server_quota_backend_bytes{job="shared-etcd"} + > 0.8 + for: 10m + labels: + severity: warning + service: shared-etcd + annotations: + summary: 共享 etcd 后端容量超过 80% + - alert: SharedEtcdSlowFsync + expr: histogram_quantile(0.95, rate(etcd_disk_wal_fsync_duration_seconds_bucket{job="shared-etcd"}[5m])) + > 0.05 + for: 10m + labels: + severity: warning + service: shared-etcd + annotations: + summary: 共享 etcd {{ $labels.member }} WAL fsync p95 超过 50ms + - alert: SharedEtcdFrequentElections + expr: increase(etcd_server_leader_changes_seen_total{job="shared-etcd"}[15m]) + > 3 + for: 5m + labels: + severity: warning + service: shared-etcd + annotations: + summary: 共享 etcd 15 分钟内多次选举 diff --git a/platform/observability/metrics/rules/shared-postgresql.yaml b/platform/observability/metrics/rules/shared-postgresql.yaml new file mode 100644 index 0000000..4c748cc --- /dev/null +++ b/platform/observability/metrics/rules/shared-postgresql.yaml @@ -0,0 +1,40 @@ +# 使用 Patroni 原生指标及现有 Alertmanager;不新增监控栈。 +apiVersion: operator.victoriametrics.com/v1beta1 +kind: VMRule +metadata: + name: shared-postgresql + namespace: monitoring +spec: + groups: + - name: shared-postgresql + rules: + - alert: SharedPostgresMemberUnavailable + expr: up{job="shared-postgresql-patroni"} == 0 + for: 2m + labels: {severity: warning, service: shared-postgresql} + annotations: + summary: '共享 PG 成员 {{ $labels.member }} 无法采集' + - alert: SharedPostgresNoObservedPrimary + expr: (sum(patroni_primary{job="shared-postgresql-patroni"}) or vector(0)) == 0 + for: 1m + labels: {severity: critical, service: shared-postgresql} + annotations: + summary: 共享 PG 监控未观察到 primary,请检查实际角色和入口 + - alert: SharedPostgresMultiplePrimaries + expr: sum(patroni_primary{job="shared-postgresql-patroni"}) > 1 + for: 30s + labels: {severity: critical, service: shared-postgresql} + annotations: + summary: 共享 PG 监控观察到多个 primary,需立即核实 + - alert: SharedPostgresReplicaNotStreaming + expr: patroni_replica{job="shared-postgresql-patroni"} == 1 and patroni_postgres_streaming{job="shared-postgresql-patroni"} == 0 + for: 2m + labels: {severity: warning, service: shared-postgresql} + annotations: + summary: '共享 PG replica {{ $labels.member }} 没有流复制' + - alert: SharedPostgresProcessDown + expr: patroni_postgres_running{job="shared-postgresql-patroni"} == 0 + for: 1m + labels: {severity: critical, service: shared-postgresql} + annotations: + summary: '共享 PG 成员 {{ $labels.member }} 数据库进程未运行' diff --git a/platform/observability/metrics/scrapes/shared-etcd.yaml b/platform/observability/metrics/scrapes/shared-etcd.yaml new file mode 100644 index 0000000..f313e27 --- /dev/null +++ b/platform/observability/metrics/scrapes/shared-etcd.yaml @@ -0,0 +1,26 @@ +apiVersion: operator.victoriametrics.com/v1beta1 +kind: VMStaticScrape +metadata: + name: shared-etcd + namespace: monitoring +spec: + jobName: shared-etcd + targetEndpoints: + - targets: + - 192.168.10.127:2381 + labels: + cluster: homelab-etcd + member: etcd-laptop + interval: 30s + - targets: + - 10.60.0.20:2381 + labels: + cluster: homelab-etcd + member: etcd-pve1 + interval: 30s + - targets: + - 10.60.0.21:2381 + labels: + cluster: homelab-etcd + member: etcd-pve2 + interval: 30s diff --git a/platform/observability/metrics/scrapes/shared-postgresql.yaml b/platform/observability/metrics/scrapes/shared-postgresql.yaml new file mode 100644 index 0000000..c88569e --- /dev/null +++ b/platform/observability/metrics/scrapes/shared-postgresql.yaml @@ -0,0 +1,14 @@ +apiVersion: operator.victoriametrics.com/v1beta1 +kind: VMStaticScrape +metadata: + name: shared-postgresql + namespace: monitoring +spec: + jobName: shared-postgresql-patroni + targetEndpoints: + - targets: [192.168.10.127:8008] + labels: {cluster: homelab-pg-prod, member: pg-laptop} + interval: 15s + - targets: [10.60.0.20:8008] + labels: {cluster: homelab-pg-prod, member: pg-pve1} + interval: 15s