feat: 纳管共享 etcd 与 k3s 外 PostgreSQL 高可用及备份

2026-09-25 19:34:48 +00:00
parent acd4b55722
commit 3a2fe5fa0c
92 changed files with 4211 additions and 0 deletions
@@ -0,0 +1,17 @@
+---
+etcd_version: 3.7.2
+etcd_archive_checksum: sha256:3a3679bc51a4ee9d30bccea1da7cd4fe62c6fc1d2ca1255068d2c53bf3026135
+etcd_install_dir: /opt/homelab-etcd
+etcd_config_dir: /etc/homelab-etcd
+etcd_data_dir: /var/lib/homelab-etcd
+etcd_cluster_token: homelab-shared-etcd-v1
+etcd_client_port: 2379
+etcd_peer_port: 2380
+etcd_metrics_port: 2381
+etcd_quota_bytes: 268435456
+etcd_memory_high: 256M
+etcd_memory_max: 384M
+etcd_certificate_ttl: 1440h
+etcd_renew_before: +14d
+etcd_snapshot_dir: /var/backups/homelab-etcd
+etcd_snapshot_keep: 3
@@ -0,0 +1,93 @@
+---
+- name: 在成员本地生成私钥
+  community.crypto.openssl_privatekey:
+    path: "{{ etcd_config_dir }}/{{ etcd_cert.name }}.key"
+    type: ECC
+    curve: secp256r1
+    owner: root
+    group: "{{ etcd_cert.group }}"
+    mode: '0640'
+
+- name: 本地生成 CSR
+  community.crypto.openssl_csr:
+    path: "{{ etcd_config_dir }}/{{ etcd_cert.name }}.csr"
+    privatekey_path: "{{ etcd_config_dir }}/{{ etcd_cert.name }}.key"
+    common_name: "{{ etcd_cert.cn | default(omit, true) }}"
+    subject_alt_name: >-
+      {{ ['IP:' ~ etcd_address, 'DNS:' ~ inventory_hostname] if etcd_cert.name in ['server', 'peer']
+         else (['DNS:' ~ inventory_hostname] if etcd_cert.name == 'gateway' else []) }}
+    use_common_name_for_san: false
+    extended_key_usage: "{{ etcd_cert.eku }}"
+    key_usage: [digitalSignature]
+    mode: '0644'
+  register: etcd_csr_state
+
+- name: 检查已有证书
+  ansible.builtin.stat:
+    path: "{{ etcd_config_dir }}/{{ etcd_cert.name }}.crt"
+  register: etcd_cert_file
+
+- name: 检查续签窗口
+  community.crypto.x509_certificate_info:
+    path: "{{ etcd_config_dir }}/{{ etcd_cert.name }}.crt"
+    valid_at:
+      renewal: "{{ etcd_renew_before }}"
+  register: etcd_cert_info
+  when: etcd_cert_file.stat.exists
+
+- name: 通过中央 CA 签发需更新的证书
+  when: >-
+    not etcd_cert_file.stat.exists or etcd_csr_state is changed or
+    not (etcd_cert_info.valid_at.renewal | default(false))
+  block:
+    - name: 仅在确需签发时要求 Bao 凭据
+      ansible.builtin.assert:
+        that: etcd_bao_token | length > 0
+      no_log: true
+
+    - name: 读取 CSR 公共内容
+      ansible.builtin.slurp:
+        src: "{{ etcd_config_dir }}/{{ etcd_cert.name }}.csr"
+      register: etcd_csr
+
+    - name: 控制端提交 Bao 签名请求
+      ansible.builtin.uri:
+        url: "{{ etcd_bao_url }}/v1/{{ etcd_bao_pki_mount }}/sign/{{ etcd_cert.role }}"
+        method: POST
+        headers:
+          X-Vault-Token: "{{ etcd_bao_token }}"
+        body_format: json
+        body:
+          csr: "{{ etcd_csr.content | b64decode }}"
+          ttl: "{{ etcd_certificate_ttl }}"
+        status_code: 200
+      delegate_to: localhost
+      become: false
+      register: etcd_signed
+      no_log: true
+      when: not ansible_check_mode
+
+    - name: 保存签发的证书链
+      ansible.builtin.copy:
+        content: |
+          {{ etcd_signed.json.data.certificate }}
+          {{ etcd_signed.json.data.ca_chain | join('\n') }}
+        dest: "{{ etcd_config_dir }}/{{ etcd_cert.name }}.crt"
+        owner: root
+        group: "{{ etcd_cert.group }}"
+        mode: '0644'
+      when: not ansible_check_mode
+
+    - name: 保存中央 CA 信任链
+      ansible.builtin.copy:
+        content: |
+          {{ etcd_signed.json.data.ca_chain | join('\n') }}
+        dest: "{{ etcd_config_dir }}/ca.crt"
+        owner: root
+        group: homelab-etcd
+        mode: '0644'
+      when: not ansible_check_mode
+
+    - name: 标记证书已更新
+      ansible.builtin.set_fact:
+        etcd_certificates_changed: true
@@ -0,0 +1,3 @@
+---
+# 只加载 role defaults，供运维入口复用。
+[]
@@ -0,0 +1,134 @@
+---
+- name: 验证拓扑和签发配置
+  ansible.builtin.assert:
+    that:
+      - groups['etcd'] | length == 3
+      - groups['etcd'] | map('extract', hostvars, 'etcd_address') | unique | length == 3
+      - ansible_facts['architecture'] == 'x86_64'
+      - etcd_bao_url is match('^https://')
+      - etcd_archive_checksum is match('^sha256:[a-f0-9]{64}$')
+  no_log: true
+
+- name: 安装证书处理依赖
+  ansible.builtin.package:
+    name: [python3-cryptography, openssl]
+    state: present
+
+- name: 创建独立 etcd 组
+  ansible.builtin.group:
+    name: homelab-etcd
+    system: true
+
+- name: 创建独立 etcd 用户
+  ansible.builtin.user:
+    name: homelab-etcd
+    group: homelab-etcd
+    system: true
+    shell: /usr/sbin/nologin
+    create_home: false
+
+- name: 创建受管目录
+  ansible.builtin.file:
+    path: "{{ item.path }}"
+    state: directory
+    owner: "{{ item.owner }}"
+    group: homelab-etcd
+    mode: "{{ item.mode }}"
+  loop:
+    - {path: "{{ etcd_config_dir }}", owner: root, mode: '0750'}
+    - {path: "{{ etcd_data_dir }}", owner: homelab-etcd, mode: '0700'}
+    - {path: "{{ etcd_install_dir }}", owner: root, mode: '0755'}
+
+- name: 下载固定版本及校验归档
+  ansible.builtin.get_url:
+    url: >-
+      https://github.com/etcd-io/etcd/releases/download/v{{ etcd_version }}/etcd-v{{ etcd_version }}-linux-amd64.tar.gz
+    dest: "{{ etcd_install_dir }}/etcd-v{{ etcd_version }}.tar.gz"
+    checksum: "{{ etcd_archive_checksum }}"
+    mode: '0644'
+  register: etcd_download
+  retries: 3
+  delay: 5
+  until: etcd_download is succeeded
+
+- name: 展开固定版本
+  ansible.builtin.unarchive:
+    src: "{{ etcd_install_dir }}/etcd-v{{ etcd_version }}.tar.gz"
+    dest: "{{ etcd_install_dir }}"
+    remote_src: true
+    creates: "{{ etcd_install_dir }}/etcd-v{{ etcd_version }}-linux-amd64/etcd"
+
+- name: 安装版本链接
+  ansible.builtin.file:
+    src: "{{ etcd_install_dir }}/etcd-v{{ etcd_version }}-linux-amd64/{{ item }}"
+    dest: "{{ etcd_install_dir }}/{{ item }}"
+    state: link
+  loop: [etcd, etcdctl, etcdutl]
+  register: etcd_binary_links
+
+- name: 签发成员与管理员证书
+  ansible.builtin.include_tasks: certificate.yml
+  loop:
+    - {name: server, role: homelab-etcd-server, cn: "{{ inventory_hostname }}", eku: [serverAuth], group: homelab-etcd}
+    - {name: peer, role: homelab-etcd-peer, cn: homelab-etcd-peer, eku: [serverAuth, clientAuth], group: homelab-etcd}
+    - {name: gateway, role: homelab-etcd-gateway, cn: "", eku: [clientAuth], group: homelab-etcd}
+    - {name: admin, role: homelab-etcd-admin, cn: root, eku: [clientAuth], group: root}
+  loop_control:
+    loop_var: etcd_cert
+
+- name: 写入独立 etcd 配置
+  ansible.builtin.template:
+    src: etcd.yml.j2
+    dest: "{{ etcd_config_dir }}/etcd.yml"
+    owner: root
+    group: homelab-etcd
+    mode: '0640'
+  register: etcd_config_file
+
+- name: 写入独立 systemd unit
+  ansible.builtin.template:
+    src: homelab-etcd.service.j2
+    dest: /etc/systemd/system/homelab-etcd.service
+    mode: '0644'
+  register: etcd_unit
+
+# 跨失败重跑记录激活状态，防止上一轮写文件后中断导致漏掉必要重启。
+- name: 计算受管文件校验和
+  ansible.builtin.stat:
+    path: "{{ item }}"
+    checksum_algorithm: sha256
+  loop:
+    - "{{ etcd_config_dir }}/etcd.yml"
+    - "{{ etcd_config_dir }}/server.crt"
+    - "{{ etcd_config_dir }}/peer.crt"
+    - "{{ etcd_config_dir }}/gateway.crt"
+    - "{{ etcd_config_dir }}/ca.crt"
+    - /etc/systemd/system/homelab-etcd.service
+  register: etcd_managed_files
+
+- name: 计算期望激活指纹
+  ansible.builtin.set_fact:
+    etcd_config_fingerprint: >-
+      {{ ((etcd_managed_files.results | map(attribute='stat.checksum') | list | join(':'))
+          ~ ':' ~ etcd_version) | hash('sha256') }}
+  when: not ansible_check_mode
+
+- name: 检查已激活指纹
+  ansible.builtin.stat:
+    path: "{{ etcd_config_dir }}/activated.sha256"
+  register: etcd_activated_file
+
+- name: 读取已激活指纹
+  ansible.builtin.slurp:
+    src: "{{ etcd_config_dir }}/activated.sha256"
+  register: etcd_activated
+  when: etcd_activated_file.stat.exists
+
+- name: 判断是否需要滚动激活
+  ansible.builtin.set_fact:
+    etcd_config_changed: >-
+      {{ not etcd_activated_file.stat.exists or
+         (etcd_activated.content | default('') | b64decode | trim) != etcd_config_fingerprint | default('check-mode') }}
+
+- name: 管理本地快照任务
+  ansible.builtin.import_tasks: snapshot.yml
@@ -0,0 +1,26 @@
+---
+- name: 创建仅 root 可访问的快照目录
+  ansible.builtin.file:
+    path: "{{ etcd_snapshot_dir }}"
+    state: directory
+    owner: root
+    group: root
+    mode: '0700'
+
+- name: 写入快照脚本和 systemd 任务
+  ansible.builtin.template:
+    src: "{{ item.src }}"
+    dest: "{{ item.dest }}"
+    mode: "{{ item.mode }}"
+  loop:
+    - {src: snapshot.sh.j2, dest: "{{ etcd_install_dir }}/snapshot", mode: '0700'}
+    - {src: snapshot.service.j2, dest: /etc/systemd/system/homelab-etcd-snapshot.service, mode: '0644'}
+    - {src: snapshot.timer.j2, dest: /etc/systemd/system/homelab-etcd-snapshot.timer, mode: '0644'}
+  register: etcd_snapshot_units
+
+- name: 启用本地快照计划
+  ansible.builtin.systemd_service:
+    name: homelab-etcd-snapshot.timer
+    daemon_reload: "{{ etcd_snapshot_units is changed }}"
+    enabled: true
+    state: started
@@ -0,0 +1,33 @@
+# Ansible 管理；与 k3s、数据库生命周期独立。
+name: {{ inventory_hostname | to_json }}
+data-dir: {{ etcd_data_dir | to_json }}
+listen-client-urls: https://{{ etcd_address }}:{{ etcd_client_port }}
+advertise-client-urls: https://{{ etcd_address }}:{{ etcd_client_port }}
+listen-peer-urls: https://{{ etcd_address }}:{{ etcd_peer_port }}
+initial-advertise-peer-urls: https://{{ etcd_address }}:{{ etcd_peer_port }}
+initial-cluster: "{% for member in groups['etcd'] %}{{ member }}=https://{{ hostvars[member].etcd_address }}:{{ etcd_peer_port }}{{ ',' if not loop.last else '' }}{% endfor %}"
+initial-cluster-token: {{ etcd_cluster_token | to_json }}
+initial-cluster-state: new
+# 已存在的数据目录优先；成员替换必须走单独 runbook，不删除数据重建。
+client-transport-security:
+  cert-file: {{ etcd_config_dir }}/server.crt
+  key-file: {{ etcd_config_dir }}/server.key
+  client-cert-file: {{ etcd_config_dir }}/gateway.crt
+  client-key-file: {{ etcd_config_dir }}/gateway.key
+  trusted-ca-file: {{ etcd_config_dir }}/ca.crt
+  client-cert-auth: true
+peer-transport-security:
+  cert-file: {{ etcd_config_dir }}/peer.crt
+  key-file: {{ etcd_config_dir }}/peer.key
+  trusted-ca-file: {{ etcd_config_dir }}/ca.crt
+  client-cert-auth: true
+  allowed-cn: [homelab-etcd-peer]
+# 独立 metrics listener 仅提供指标/健康，不开放 KV API；只绑定受管内网地址。
+listen-metrics-urls: {{ etcd_metrics_urls | default("http://127.0.0.1:" ~ etcd_metrics_port) | to_json }}
+quota-backend-bytes: {{ etcd_quota_bytes }}
+auto-compaction-mode: periodic
+auto-compaction-retention: '1h'
+heartbeat-interval: 100
+election-timeout: 1000
+logger: zap
+log-level: info
@@ -0,0 +1,23 @@
+[Unit]
+Description=Homelab shared etcd
+Wants=network-online.target
+After=network-online.target
+
+[Service]
+User=homelab-etcd
+Group=homelab-etcd
+ExecStart={{ etcd_install_dir }}/etcd --config-file={{ etcd_config_dir }}/etcd.yml
+Restart=on-failure
+RestartSec=5
+TimeoutStopSec=60
+MemoryHigh={{ etcd_memory_high }}
+MemoryMax={{ etcd_memory_max }}
+UMask=0077
+NoNewPrivileges=true
+ProtectSystem=strict
+ProtectHome=true
+PrivateTmp=true
+ReadWritePaths={{ etcd_data_dir }}
+
+[Install]
+WantedBy=multi-user.target
@@ -0,0 +1,11 @@
+[Unit]
+Description=Snapshot homelab shared etcd
+After=homelab-etcd.service
+
+[Service]
+Type=oneshot
+ExecStart={{ etcd_install_dir }}/snapshot
+User=root
+UMask=0077
+TimeoutStartSec=300
+Nice=10
@@ -0,0 +1,20 @@
+#!/bin/bash
+set -euo pipefail
+umask 077
+export ETCDCTL_ENDPOINTS="https://{{ etcd_address }}:{{ etcd_client_port }}"
+export ETCDCTL_CACERT="{{ etcd_config_dir }}/ca.crt"
+export ETCDCTL_CERT="{{ etcd_config_dir }}/admin.crt"
+export ETCDCTL_KEY="{{ etcd_config_dir }}/admin.key"
+repo="{{ etcd_snapshot_dir }}"
+exec 9>"$repo/.lock"
+flock -n 9 || exit 0
+output="$repo/$(date -u +%Y%m%dT%H%M%SZ).db"
+trap 'rm -f "$output.partial" "$output.partial.part"' EXIT
+{{ etcd_install_dir }}/etcdctl snapshot save "$output.partial"
+{{ etcd_install_dir }}/etcdutl snapshot status "$output.partial" >/dev/null
+mv "$output.partial" "$output"
+# 只有新快照成功且验证可读才清理旧备份。目录仅由此任务管理。
+mapfile -t snapshots < <(find "$repo" -maxdepth 1 -type f -name '????????T??????Z.db' -printf '%f\n' | sort -r)
+for old in "${snapshots[@]:{{ etcd_snapshot_keep }}}"; do
+    rm -- "$repo/$old"
+done
@@ -0,0 +1,10 @@
+[Unit]
+Description=Daily homelab etcd snapshot
+
+[Timer]
+OnCalendar=*-*-* 03:20:00 UTC
+RandomizedDelaySec=300
+Persistent=true
+
+[Install]
+WantedBy=timers.target