Compare commits
32
Commits
7a62dde601
...
main
| Author | SHA1 | Date | |
|---|---|---|---|
 panxiao81
|
f5c8c09f96 | ||
|
panxiao81
|
7137426e8f | ||
|
panxiao81
|
834f654941 | ||
|
panxiao81
|
52322116f9
|
||
|
panxiao81
|
f6d12d673a | ||
|
panxiao81
|
67d4869c88
|
||
|
panxiao81
|
655ca567ba | ||
|
panxiao81
|
b75043cb45 | ||
|
panxiao81
|
3a2fe5fa0c
|
||
|
panxiao81
|
acd4b55722
|
||
|
panxiao81
|
235288189d | ||
|
panxiao81
|
15676a6a06
|
||
|
panxiao81
|
18ded1e4a8 | ||
|
panxiao81
|
ca0862fe5a
|
||
|
panxiao81
|
a729f01546 | ||
|
panxiao81
|
bbcd5d6aa5
|
||
|
panxiao81
|
c0c4ff521b | ||
|
panxiao81
|
8bc039b40f | ||
|
panxiao81
|
349aa77fb0
|
||
|
panxiao81
|
cae6acbfb0 | ||
|
panxiao81
|
4d80f8efd7
|
||
|
panxiao81
|
9009e3fa11 | ||
|
panxiao81
|
a252ee5602 | ||
|
panxiao81
|
dbf66590ac | ||
|
panxiao81
|
e8e7bd4b20 | ||
|
panxiao81
|
185e3edaba | ||
|
panxiao81
|
072936ce03 | ||
|
panxiao81
|
39aeac25ee | ||
|
panxiao81
|
a43b7d3c26 | ||
|
panxiao81
|
c06c6f7857 | ||
|
panxiao81
|
6a27adc18d | ||
|
panxiao81
|
29b3bdeeb1 |
@@ -0,0 +1,20 @@
|
||||
# Backstage
|
||||
|
||||
Backstage 作为 homelab 的只读开发者门户运行。应用源码、插件、测试和镜像构建归
|
||||
`panxiao81/backstage` 仓库管理；本目录只保存 Kubernetes/Flux 部署声明，并通过
|
||||
OCI digest 固定镜像。
|
||||
|
||||
## 外部前置
|
||||
|
||||
- OpenBao `kv/k8s/backstage` 必须包含以下与容器环境变量同名的字段：
|
||||
`BACKEND_SECRET`、`AUTH_SESSION_SECRET`、`AUTH_OIDC_CLIENT_ID`、
|
||||
`AUTH_OIDC_CLIENT_SECRET`、`POSTGRES_PASSWORD`、`GITEA_TOKEN`。
|
||||
- PostgreSQL 需要在共享集群中预先创建由 `backstage` 角色拥有的 `backstage`
|
||||
数据库；密码必须与 OpenBao 中的 `POSTGRES_PASSWORD` 一致。
|
||||
- Authelia OIDC 客户端回调地址为
|
||||
`https://backstage.ad.ddupan.top/api/auth/oidc/handler/frame`。
|
||||
- AD DNS 需要将 `backstage.ad.ddupan.top` 指向 Envoy Gateway
|
||||
`192.168.10.127`。
|
||||
|
||||
Flux 等待 ExternalSecret 和 Deployment 就绪；任何前置缺失都会使该
|
||||
Kustomization 保持 NotReady，而不会回退到明文 Secret。
|
||||
@@ -0,0 +1,85 @@
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: backstage
|
||||
namespace: backstage
|
||||
labels:
|
||||
app.kubernetes.io/name: backstage
|
||||
backstage.io/kubernetes-id: homelab-backstage
|
||||
spec:
|
||||
replicas: 1
|
||||
strategy:
|
||||
type: Recreate
|
||||
selector:
|
||||
matchLabels:
|
||||
app.kubernetes.io/name: backstage
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app.kubernetes.io/name: backstage
|
||||
backstage.io/kubernetes-id: homelab-backstage
|
||||
spec:
|
||||
serviceAccountName: backstage
|
||||
securityContext:
|
||||
fsGroup: 1000
|
||||
runAsNonRoot: true
|
||||
seccompProfile:
|
||||
type: RuntimeDefault
|
||||
containers:
|
||||
- name: backstage
|
||||
image: zot.ad.ddupan.top/panxiao81/backstage@sha256:e5a12550726f19a680bc7c40e2cc07cc624318f9279ee121814d293a006ef210
|
||||
imagePullPolicy: IfNotPresent
|
||||
env:
|
||||
- name: BACKSTAGE_BASE_URL
|
||||
value: https://backstage.ad.ddupan.top
|
||||
- name: POSTGRES_HOST
|
||||
value: shared-postgresql-rw.shared-db.svc.cluster.local
|
||||
- name: POSTGRES_PORT
|
||||
value: "5432"
|
||||
- name: POSTGRES_USER
|
||||
value: backstage
|
||||
- name: POSTGRES_DATABASE
|
||||
value: backstage
|
||||
- name: GITEA_HOST
|
||||
value: git.ddupan.top
|
||||
envFrom:
|
||||
- secretRef:
|
||||
name: backstage
|
||||
ports:
|
||||
- containerPort: 7007
|
||||
name: http
|
||||
protocol: TCP
|
||||
readinessProbe:
|
||||
httpGet:
|
||||
path: /.backstage/health/v1/readiness
|
||||
port: http
|
||||
initialDelaySeconds: 10
|
||||
periodSeconds: 10
|
||||
timeoutSeconds: 3
|
||||
livenessProbe:
|
||||
httpGet:
|
||||
path: /.backstage/health/v1/liveness
|
||||
port: http
|
||||
initialDelaySeconds: 30
|
||||
periodSeconds: 20
|
||||
timeoutSeconds: 3
|
||||
resources:
|
||||
requests:
|
||||
cpu: 100m
|
||||
memory: 512Mi
|
||||
limits:
|
||||
cpu: "1"
|
||||
memory: 1Gi
|
||||
securityContext:
|
||||
allowPrivilegeEscalation: false
|
||||
capabilities:
|
||||
drop: [ALL]
|
||||
readOnlyRootFilesystem: true
|
||||
runAsNonRoot: true
|
||||
runAsUser: 1000
|
||||
volumeMounts:
|
||||
- mountPath: /tmp
|
||||
name: tmp
|
||||
volumes:
|
||||
- emptyDir: {}
|
||||
name: tmp
|
||||
@@ -0,0 +1,16 @@
|
||||
apiVersion: external-secrets.io/v1
|
||||
kind: ExternalSecret
|
||||
metadata:
|
||||
name: backstage
|
||||
namespace: backstage
|
||||
spec:
|
||||
refreshInterval: 1h
|
||||
secretStoreRef:
|
||||
kind: ClusterSecretStore
|
||||
name: openbao
|
||||
target:
|
||||
creationPolicy: Owner
|
||||
name: backstage
|
||||
dataFrom:
|
||||
- extract:
|
||||
key: k8s/backstage
|
||||
@@ -0,0 +1,20 @@
|
||||
apiVersion: gateway.networking.k8s.io/v1
|
||||
kind: HTTPRoute
|
||||
metadata:
|
||||
name: backstage
|
||||
namespace: backstage
|
||||
spec:
|
||||
parentRefs:
|
||||
- name: eg
|
||||
namespace: envoy-gateway-system
|
||||
sectionName: https
|
||||
hostnames:
|
||||
- backstage.ad.ddupan.top
|
||||
rules:
|
||||
- matches:
|
||||
- path:
|
||||
type: PathPrefix
|
||||
value: /
|
||||
backendRefs:
|
||||
- name: backstage
|
||||
port: 7007
|
||||
@@ -0,0 +1,11 @@
|
||||
apiVersion: kustomize.config.k8s.io/v1beta1
|
||||
kind: Kustomization
|
||||
resources:
|
||||
- namespace.yaml
|
||||
- serviceaccount.yaml
|
||||
- rbac.yaml
|
||||
- external-secret.yaml
|
||||
- deployment.yaml
|
||||
- service.yaml
|
||||
- networkpolicy.yaml
|
||||
- httproute.yaml
|
||||
@@ -0,0 +1,7 @@
|
||||
apiVersion: v1
|
||||
kind: Namespace
|
||||
metadata:
|
||||
name: backstage
|
||||
labels:
|
||||
pod-security.kubernetes.io/enforce: restricted
|
||||
pod-security.kubernetes.io/enforce-version: v1.36
|
||||
@@ -0,0 +1,24 @@
|
||||
apiVersion: networking.k8s.io/v1
|
||||
kind: NetworkPolicy
|
||||
metadata:
|
||||
name: backstage-ingress
|
||||
namespace: backstage
|
||||
spec:
|
||||
podSelector:
|
||||
matchLabels:
|
||||
app.kubernetes.io/name: backstage
|
||||
policyTypes: [Ingress]
|
||||
ingress:
|
||||
- from:
|
||||
- namespaceSelector:
|
||||
matchLabels:
|
||||
kubernetes.io/metadata.name: envoy-gateway-system
|
||||
ports:
|
||||
- port: 7007
|
||||
protocol: TCP
|
||||
- from:
|
||||
- ipBlock:
|
||||
cidr: 192.168.10.127/32
|
||||
ports:
|
||||
- port: 7007
|
||||
protocol: TCP
|
||||
@@ -0,0 +1,51 @@
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: ClusterRole
|
||||
metadata:
|
||||
name: backstage-read-only
|
||||
rules:
|
||||
- apiGroups: [""]
|
||||
resources:
|
||||
- configmaps
|
||||
- limitranges
|
||||
- pods
|
||||
- pods/log
|
||||
- resourcequotas
|
||||
- services
|
||||
verbs: [get, list, watch]
|
||||
- apiGroups: [apps]
|
||||
resources:
|
||||
- daemonsets
|
||||
- deployments
|
||||
- replicasets
|
||||
- statefulsets
|
||||
verbs: [get, list, watch]
|
||||
- apiGroups: [autoscaling]
|
||||
resources:
|
||||
- horizontalpodautoscalers
|
||||
verbs: [get, list, watch]
|
||||
- apiGroups: [batch]
|
||||
resources:
|
||||
- cronjobs
|
||||
- jobs
|
||||
verbs: [get, list, watch]
|
||||
- apiGroups: [networking.k8s.io]
|
||||
resources:
|
||||
- ingresses
|
||||
verbs: [get, list, watch]
|
||||
- apiGroups: [metrics.k8s.io]
|
||||
resources:
|
||||
- pods
|
||||
verbs: [get, list]
|
||||
---
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: ClusterRoleBinding
|
||||
metadata:
|
||||
name: backstage-read-only
|
||||
roleRef:
|
||||
apiGroup: rbac.authorization.k8s.io
|
||||
kind: ClusterRole
|
||||
name: backstage-read-only
|
||||
subjects:
|
||||
- kind: ServiceAccount
|
||||
name: backstage
|
||||
namespace: backstage
|
||||
@@ -0,0 +1,15 @@
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: backstage
|
||||
namespace: backstage
|
||||
labels:
|
||||
backstage.io/kubernetes-id: homelab-backstage
|
||||
spec:
|
||||
selector:
|
||||
app.kubernetes.io/name: backstage
|
||||
ports:
|
||||
- name: http
|
||||
port: 7007
|
||||
protocol: TCP
|
||||
targetPort: http
|
||||
@@ -0,0 +1,6 @@
|
||||
apiVersion: v1
|
||||
kind: ServiceAccount
|
||||
metadata:
|
||||
name: backstage
|
||||
namespace: backstage
|
||||
automountServiceAccountToken: true
|
||||
@@ -0,0 +1,22 @@
|
||||
apiVersion: kustomize.toolkit.fluxcd.io/v1
|
||||
kind: Kustomization
|
||||
metadata:
|
||||
name: backstage
|
||||
namespace: flux-system
|
||||
spec:
|
||||
dependsOn:
|
||||
- name: envoy-gateway
|
||||
- name: external-secrets
|
||||
healthChecks:
|
||||
- apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
name: backstage
|
||||
namespace: backstage
|
||||
interval: 10m
|
||||
path: ./apps/backstage
|
||||
prune: true
|
||||
sourceRef:
|
||||
kind: GitRepository
|
||||
name: flux-system
|
||||
timeout: 5m
|
||||
wait: true
|
||||
@@ -7,6 +7,7 @@ resources:
|
||||
- apps/envoy-gateway.yaml
|
||||
- apps/external-secrets.yaml
|
||||
- apps/gitea.yaml
|
||||
- apps/backstage.yaml
|
||||
- apps/http-echo.yaml
|
||||
- apps/openebs.yaml
|
||||
- apps/nats.yaml
|
||||
|
||||
@@ -27,6 +27,9 @@ homelab_dns:
|
||||
- { zone: ad.ddupan.top, name: zot, type: A, values: [192.168.10.127] }
|
||||
- { zone: ad.ddupan.top, name: zot-push, type: A, values: [192.168.10.127] }
|
||||
|
||||
- { zone: ad.ddupan.top, name: pg-prod, type: A, values: [192.168.10.2] }
|
||||
- { zone: ad.ddupan.top, name: pg-dev, type: A, values: [192.168.10.127] }
|
||||
|
||||
split_horizon:
|
||||
# backends records the current adoption boundary. obj is deliberately not
|
||||
# emitted to CoreDNS yet, preserving the current pod resolver behaviour.
|
||||
|
||||
@@ -0,0 +1,6 @@
|
||||
.terraform/
|
||||
*.tfstate*
|
||||
*.tfplan
|
||||
*.tfvars
|
||||
!*.tfvars.example
|
||||
__pycache__/
|
||||
@@ -0,0 +1,163 @@
|
||||
# Homelab shared etcd
|
||||
|
||||
独立于 PostgreSQL 与 k3s 的三成员协调服务。Ansible 管主机、证书文件和 etcd 账号；
|
||||
独立 Terraform root 管现有 Bao PKI 下的签发角色与 policy，不管理 CA 或秘密值。
|
||||
旧 Ansible Vault 不在本次迁移范围。
|
||||
|
||||
状态：2026-09-25 已部署三成员 etcd，启用 Bao mTLS 和 RBAC；三个端点健康检查通过。
|
||||
PVE 两个 LXC 的 rootfs 已通过 `pct move-volume` 从 local-lvm 迁入 `pve-rg` SSD DRBD 池，
|
||||
保留容器与 etcd 数据。当前 PVE LXC 迁卷要求停止容器，维护入口按成员串行执行，回归健康后再处理下一个。
|
||||
|
||||
| 成员 | 承载 | 地址 | 资源 |
|
||||
| --- | --- | --- | --- |
|
||||
| etcd-laptop | laptop systemd | 192.168.10.127 | MemoryHigh 256 MiB / MemoryMax 384 MiB |
|
||||
| etcd-pve1 | pve1 新无特权 LXC 150 | 10.60.0.20 | 1 core / 1536 MiB / 8 GiB SSD DRBD |
|
||||
| etcd-pve2 | pve2 新无特权 LXC 151 | 10.60.0.21 | 1 core / 768 MiB / 8 GiB SSD DRBD |
|
||||
|
||||
资源限额是初始测试预算，不是已测容量。LXC 已增加 PG standby/备份的独立 HDD mp0 与内存上限，生产 PG standby 与备份仓库已启动。
|
||||
etcd 自身的 MemoryHigh/MemoryMax 仍为 256/384 MiB。labnet 依赖现有 VyOS 路由；切换 bare-metal 时复用服务角色，更换主机 inventory。
|
||||
|
||||
## 接入方式与认证
|
||||
|
||||
客户端使用三个 `https://地址:2379` endpoint，校验 Bao 中央 CA，出示独立客户端证书。
|
||||
成员间 2380 也开启 mTLS，额外限制 peer CN 为 `homelab-etcd-peer`。
|
||||
管理员证书 CN 为 `root`，只保存在受管成员 root 可读的文件中；对应 etcd root 用户不设置密码。
|
||||
服务进程只能读取自己的 server/peer 私钥，不能读取管理员私钥。
|
||||
|
||||
Patroni 走 etcd3 gateway，使用无 CN 的客户端证书及独立 username/password 完成 RBAC。
|
||||
实测带 CN 的证书会被 gateway 拒绝；管理员 CN=root 证书仅用于原生 etcdctl。
|
||||
成员另有无 CN 的 gateway 证书供内部 gateway→gRPC 连接及密码核对使用，不能将管理员证书复用到 HTTP API。首个消费者为
|
||||
`patroni-pg-prod`，只允许 `/homelab/patroni/pg-prod/`。其 Bao KV v2 路径为
|
||||
`kv/infra/etcd/consumers/patroni-pg-prod`，包含 username/password/prefix；不在本文复制实际值。
|
||||
配置 Patroni 时对应 `namespace: /homelab/patroni/` 与 `scope: pg-prod`。
|
||||
|
||||
第一次接入应由管理流程交付 CA、客户端证书及秘密引用，然后使用自己的身份对自己的 prefix
|
||||
执行一次 put/get/delete。跨 prefix 应被拒绝；不要把 root 证书复制给应用。
|
||||
生产消费者证书已签发，Patroni 已接入并验证主从自动切换。
|
||||
|
||||
## 安装与维护入口
|
||||
|
||||
需要 Ansible、community.crypto，以及目标机 Python/systemd。当前二进制固定为 etcd 3.7.2
|
||||
linux-amd64，并固定官方发布资产 SHA-256；版本升级必须单独评估兼容性、备份与 quorum。
|
||||
Terraform 使用已登录的 Vault 兼容 provider 身份；Ansible 读取控制端 `BAO_TOKEN` 环境变量，
|
||||
token 不下发目标机。`homelab-etcd-provisioner` policy 不自动绑定到现有身份，先由 Bao 管理流程授权。
|
||||
|
||||
```bash
|
||||
cd infrastructure/etcd/terraform
|
||||
terraform init
|
||||
terraform plan
|
||||
# 审阅计划后 apply；只新增此服务的 PKI roles/policies。
|
||||
|
||||
cd ../ansible
|
||||
ansible-galaxy collection install -r requirements.yml
|
||||
ansible-playbook lxc.yml --check
|
||||
ansible-playbook lxc.yml
|
||||
ansible-playbook site.yml
|
||||
ansible-playbook bootstrap-auth.yml -e etcd_bootstrap_auth=true
|
||||
ansible-playbook consumers.yml
|
||||
ansible-playbook verify.yml
|
||||
```
|
||||
|
||||
Terraform 使用现有 S3 tfstate bucket，独立 key `etcd/terraform.tfstate`，启用原生 lockfile；不复用 Bao 的 state。
|
||||
从仓库根目录运行 `python3 infrastructure/etcd/run.py terraform <子命令>` 或
|
||||
`python3 infrastructure/etcd/run.py ansible <playbook.yml>`，复用当前 Bao 会话并在内存中取得所需凭据。
|
||||
`lxc.yml` 是创建入口，不接管已有未知 VMID，也不自动调整运行中容器的网络与容量；配置漂移会报错。
|
||||
部署前核对模板与 `pve-rg` 存储可用性、IP 冲突、SSH host key、laptop sudo 和网络访问控制。
|
||||
数据库角色不会创建/删除这些 LXC 或 etcd 成员。
|
||||
|
||||
`site.yml` 管安装、CSR 本地生成、Bao 签发、配置和逐成员激活。全体健康检查通过才记录激活指纹；
|
||||
上次中断后重跑仍能识别尚未激活的磁盘配置。已有集群每个成员激活后检查健康，再处理下一个。
|
||||
首次建群先让三个服务启动，再检查 quorum。`bootstrap-auth.yml` 与日常收敛分开，认证已启用时不重建。
|
||||
首次认证启用前应限制客户端网络，不能把“已有中央 CA 签名”当作业务授权。
|
||||
|
||||
`consumers.yml` 首次生成 48 字符随机密码并以 KV v2 CAS=0 写入，之后只复用。
|
||||
元数据存在但数据被删除、Bao 403/超时、或 etcd 用户存在但秘密丢失时均失败，不自动换密码。
|
||||
并发 CAS 冲突会中止此次执行，重跑读取胜出的版本。既有额外角色或不同 prefix 权限也报错，
|
||||
不默默扩大权限。账号删除、prefix 迁移和密码轮换均不包含在普通收敛里。
|
||||
密码使用 stdin 送给 etcdctl，涉及秘密的任务 `no_log: true`；禁止用 `--diff` 打印未来消费者秘密文件。
|
||||
|
||||
证书有效期 60 天，`site.yml` 在剩余不足 14 天时续签；需要 Bao 的控制端身份。
|
||||
续签 IaC 已实现于 `terraform/renewal.tf`、`controller/` 和 `ansible/controller.yml`：
|
||||
独立 cert auth 挂载信任中央 CA，并限制 laptop 的 DNS SAN；用现有 peer 客户端证书登录，
|
||||
换取 10 分钟 token，仅允许签发 server/peer/admin/gateway 四类证书，不授予 KV 读取权限。
|
||||
控制端每日运行现有健康检查与串行部署，结束撤销 token；不保存长期 token，不依赖人工 OIDC 会话。
|
||||
控制程序安装为 root 管理的固定副本，以 panxiao81 身份复用现有 Ansible/SSH/sudo 环境。
|
||||
|
||||
**机器身份和 timer 已启用**（2026-09-25）：维护者恢复 OIDC 登录后，Terraform 新增三个
|
||||
续签资源，`controller.yml` 登录预检通过。首次实际运行三成员均 `changed=0`，服务 Result=success。
|
||||
Bao cert auth 需要非空 CN 作为 identity alias，不能使用无 CN 的 gateway 证书；因此登录使用
|
||||
同一主机已有 peer 证书，仍由 `allowed_dns_sans=etcd-laptop` 限制身份，未扩大到其他成员。
|
||||
手动触发 `systemctl start homelab-etcd-renew.service`，检查 journal 和
|
||||
`/var/lib/homelab-etcd-controller/last-success`；每日 timer 04:10 UTC 加随机延迟。
|
||||
本流程也续签自身登录所用 peer 证书；若超过有效期仍未修复，则需要管理身份重新签发。
|
||||
成员启动本身仅用本地证书，不实时依赖 Bao。
|
||||
|
||||
## 备份与故障处理
|
||||
|
||||
每个成员有每日一次的 `homelab-etcd-snapshot.timer`，快照保存在各自
|
||||
`/var/backups/homelab-etcd`，root-only，保留最近 3 个成功快照；snapshot status 验证成功才淘汰旧快照。
|
||||
首备可执行 `systemctl start homelab-etcd-snapshot.service`，用 journal 和 `etcdutl snapshot status`
|
||||
检查，不根据文件名推断备份成功。三处本地快照不等于异地备份。
|
||||
|
||||
quota 初始 256 MiB，按 1 小时保留进行自动 compaction；defrag 另择维护窗口逐成员做，
|
||||
不能三个成员同时执行。metrics 绑定 loopback 与成员内网地址的 2381 端口，独立 listener 不提供 KV API；
|
||||
与既有 LAN exporter 一样通过内网 HTTP 采集，禁止将端口映射到公网。
|
||||
现有 VictoriaMetrics 使用 `platform/observability/metrics/scrapes/shared-etcd.yaml` 采集，
|
||||
对应 VMRule 覆盖成员不可采集、少于两个可采集成员、无 leader、容量、WAL fsync 和频繁选举。
|
||||
采集失败不等于 quorum 丢失，需结合管理员 endpoint health 判断。
|
||||
备份年龄、证书到期与续签任务告警尚待补齐。既有 Alertmanager 已接入 Telegram，
|
||||
维护者已收到本次成员无 leader 与频繁选举通知；接收器配置由现有监控栈维护。
|
||||
|
||||
### DRBD I/O 故障导致成员根卷只读
|
||||
|
||||
`SharedEtcdNoLeader` 是单成员指标，不等于全体没有 leader。先检查三个 endpoint，
|
||||
并检查宿主机内核的 DRBD quorum、PingAck、I/O error 和 ext4 journal 日志。
|
||||
`findmnt` 显示 `rw,emergency_ro` 仍表示文件系统已因错误停止写入；DRBD 恢复 quorum 不会自动修复它。
|
||||
|
||||
2026-09-25 17:16 UTC 起，CT150 根卷发生 quorum 短暂丢失、journal I/O 错误和 emergency_ro，
|
||||
另外两个 etcd 成员保持健康。期间两个新 HDD 数据卷初始同步伴随多个 DRBD 资源 PingAck 超时。
|
||||
只对本项目两个新数据卷设置 `DrbdOptions/PeerDevice/c-max-rate=10240`（10 MiB/s），
|
||||
入口为 `../shared-postgresql/ansible/limit-resync.yml`，创建卷流程也复用该任务。
|
||||
限速后短期未见新增超时，尚不构成唯一根因证明；未调整全局协议、quorum 或超时。
|
||||
|
||||
恢复顺序：确认另外两个成员健康并保存快照，正常停止故障容器，确认根卷卸载、DRBD quorum
|
||||
和副本 UpToDate，再执行 `pct fsck 150 --device rootfs --force 1`。fsck 返回 1 表示已修复，
|
||||
PVE 包装命令仍会报非零；需再次检查返回 0 才启动容器。不能在已挂载卷上执行 fsck，不能直接强制 remount。
|
||||
本次修复后 CT150 启动，三个 endpoint 健康且 Raft term/index 一致；两个容器根卷与 mp0 均可写。
|
||||
频繁选举告警含 15 分钟历史窗口，应核对最新计数和健康状态，不通过关闭规则消除通知。
|
||||
|
||||
- 单成员故障：先确认其他两成员仍健康，修复原成员；普通部署不删数据目录。
|
||||
- 成员永久丢失：需单独执行 member remove/add 流程并同步 inventory，不用重新 init 覆盖。
|
||||
- 全集群丢失：先恢复 Bao/信任根与恢复凭据，隔离全部旧成员，用同一有效快照恢复新逻辑集群。
|
||||
开放客户端前逐个协调消费者状态。Patroni 必须核对真实主库、timeline、lease/DCS 状态；
|
||||
不因旧快照宣称某节点为主就允许它写入。完整跨服务灾难恢复演练尚未完成。
|
||||
|
||||
## 验证
|
||||
|
||||
```bash
|
||||
# 本机 loopback 三节点，临时测试 CA 与假的 Bao HTTP 服务，不接触线上秘密。
|
||||
ETCD_TEST_BIN=/path/to/etcd-v3.7.2-linux-amd64 python3 tests/integration.py
|
||||
cd terraform && terraform validate
|
||||
cd ../ansible && ansible-playbook --syntax-check site.yml
|
||||
```
|
||||
|
||||
集成测试验证实际模板启动、mTLS、认证初始化幂等、消费者幂等、CAS 首次创建、秘密缺失失败关闭、
|
||||
prefix 隔离、gateway 账号登录和单成员故障。假 Bao 不验证真实 PKI 签发或中央 policy 的权限，
|
||||
也不证明 PVE SSD DRBD 在业务负载下的延迟、systemd 内存预算、在线证书轮换和生产网络符合要求。
|
||||
|
||||
源码边界与 PostgreSQL 后续设计见 [研究记录](../shared-postgresql/RESEARCH.md)。
|
||||
|
||||
2026-09-25 验证结果：Terraform validate 通过；Ansible lint 零文件级问题；隔离三成员测试通过，
|
||||
认证初始化与消费者第二次执行均 `changed=0`，包含 gateway 写入、密码漂移拒绝、快照离线恢复、
|
||||
停止一成员后继续写入。测试低负载 RSS 约 36.6–40.4 MiB/成员，仅作开销参考。
|
||||
现场验证：Bao PKI roles/policies 已应用，三成员已签发证书并启动；认证初始化与
|
||||
`patroni-pg-prod` 账号创建成功，随机密码已保存 Bao，实际 gateway 登录验证通过。
|
||||
两台 LXC 的 rootfs 均为 `pve-rg`，运行状态正常；迁移后全部端点成功提交健康探测。
|
||||
现有监控已接入；自动续签已启用；备份/证书年龄告警尚待补齐。PostgreSQL 部署状态见其 README。
|
||||
|
||||
监控接入验收（2026-09-25）：三个 `up{job="shared-etcd"}` 均为 1，六条规则 health=ok，
|
||||
三端点健康检查通过。新增续签调度四项失败/成功路径测试通过，Terraform validate 与 Ansible lint 通过，
|
||||
隔离三成员测试再次通过。[IaC PR #159](https://git.ddupan.top/panxiao81/homelab-infra/pulls/159) 已合并。
|
||||
2026-09-25 合并后核实：Flux observability Ready，应用版本 `f6d12d6`，
|
||||
etcd/PG 的两份 VMRule 与两份 VMStaticScrape 均已纳入 inventory，状态 operational；
|
||||
三个 etcd 与两个 Patroni 抓取目标均 up=1，相关 11 条告警 health=ok、inactive。备份/证书年龄告警不包含在这六条规则中。
|
||||
@@ -0,0 +1,6 @@
|
||||
[defaults]
|
||||
inventory = inventory/hosts.yml
|
||||
roles_path = roles
|
||||
retry_files_enabled = False
|
||||
host_key_checking = True
|
||||
interpreter_python = auto_silent
|
||||
@@ -0,0 +1,60 @@
|
||||
---
|
||||
# 与常规 site 分离，防止重建/恢复时默默创建新的认证域。
|
||||
- name: 初始化共享 etcd 认证
|
||||
hosts: etcd[0]
|
||||
become: true
|
||||
gather_facts: false
|
||||
pre_tasks:
|
||||
- name: 加载共享默认参数
|
||||
ansible.builtin.import_role:
|
||||
name: shared_etcd
|
||||
tasks_from: context
|
||||
environment:
|
||||
ETCDCTL_ENDPOINTS: "https://{{ etcd_address }}:{{ etcd_client_port }}"
|
||||
ETCDCTL_CACERT: "{{ etcd_config_dir }}/ca.crt"
|
||||
ETCDCTL_CERT: "{{ etcd_config_dir }}/admin.crt"
|
||||
ETCDCTL_KEY: "{{ etcd_config_dir }}/admin.key"
|
||||
tasks:
|
||||
- name: 读取认证状态
|
||||
ansible.builtin.command:
|
||||
argv: ["{{ etcd_install_dir }}/etcdctl", --write-out=json, auth, status]
|
||||
changed_when: false
|
||||
register: etcd_auth_status
|
||||
check_mode: false
|
||||
|
||||
- name: 初始化管理员与认证
|
||||
when: not ((etcd_auth_status.stdout | from_json).enabled | default(false))
|
||||
block:
|
||||
- name: 要求显式初始化参数
|
||||
ansible.builtin.assert:
|
||||
that: etcd_bootstrap_auth | default(false) | bool
|
||||
fail_msg: 首次初始化需要 -e etcd_bootstrap_auth=true；常规运行不能重建认证。
|
||||
|
||||
- name: 读取现有用户
|
||||
ansible.builtin.command:
|
||||
argv: ["{{ etcd_install_dir }}/etcdctl", --write-out=json, user, list]
|
||||
changed_when: false
|
||||
register: etcd_users
|
||||
|
||||
- name: 创建仅证书认证的 root 用户
|
||||
ansible.builtin.command:
|
||||
argv: ["{{ etcd_install_dir }}/etcdctl", user, add, root, --no-password]
|
||||
when: "'root' not in ((etcd_users.stdout | from_json).users | default([], true))"
|
||||
changed_when: true
|
||||
|
||||
- name: 读取管理员角色
|
||||
ansible.builtin.command:
|
||||
argv: ["{{ etcd_install_dir }}/etcdctl", --write-out=json, user, get, root]
|
||||
changed_when: false
|
||||
register: etcd_root_roles
|
||||
|
||||
- name: 授予 root 管理角色
|
||||
ansible.builtin.command:
|
||||
argv: ["{{ etcd_install_dir }}/etcdctl", user, grant-role, root, root]
|
||||
when: "'root' not in ((etcd_root_roles.stdout | from_json).roles | default([], true))"
|
||||
changed_when: true
|
||||
|
||||
- name: 开启认证
|
||||
ansible.builtin.command:
|
||||
argv: ["{{ etcd_install_dir }}/etcdctl", auth, enable]
|
||||
changed_when: true
|
||||
@@ -0,0 +1,37 @@
|
||||
---
|
||||
- name: 收敛消费者账号及独立 prefix
|
||||
hosts: etcd[0]
|
||||
become: true
|
||||
gather_facts: false
|
||||
pre_tasks:
|
||||
- name: 加载共享默认参数
|
||||
ansible.builtin.import_role:
|
||||
name: shared_etcd
|
||||
tasks_from: context
|
||||
environment:
|
||||
ETCDCTL_ENDPOINTS: "https://{{ etcd_address }}:{{ etcd_client_port }}"
|
||||
ETCDCTL_CACERT: "{{ etcd_config_dir }}/ca.crt"
|
||||
ETCDCTL_CERT: "{{ etcd_config_dir }}/admin.crt"
|
||||
ETCDCTL_KEY: "{{ etcd_config_dir }}/admin.key"
|
||||
tasks:
|
||||
- name: 检查认证状态
|
||||
ansible.builtin.command:
|
||||
argv: ["{{ etcd_install_dir }}/etcdctl", --write-out=json, auth, status]
|
||||
register: etcd_auth_status
|
||||
changed_when: false
|
||||
check_mode: false
|
||||
|
||||
- name: 要求认证已启用
|
||||
ansible.builtin.assert:
|
||||
that:
|
||||
- ((etcd_auth_status.stdout | from_json).enabled | default(false))
|
||||
- etcd_consumers | map(attribute='name') | unique | length == etcd_consumers | length
|
||||
- etcd_bao_token | length > 0
|
||||
no_log: true
|
||||
|
||||
- name: 逐个收敛消费者
|
||||
ansible.builtin.include_tasks: tasks/consumer.yml
|
||||
loop: "{{ etcd_consumers }}"
|
||||
loop_control:
|
||||
loop_var: etcd_consumer
|
||||
label: "{{ etcd_consumer.name }}"
|
||||
@@ -0,0 +1,74 @@
|
||||
---
|
||||
# 仅在 Terraform 身份配置完成后部署；预检失败不启用定时器。
|
||||
- name: 安装独立于人工会话的续签调度
|
||||
hosts: etcd-laptop
|
||||
become: true
|
||||
gather_facts: false
|
||||
tasks:
|
||||
- name: 创建 root 管理的程序目录
|
||||
ansible.builtin.file:
|
||||
path: /opt/homelab-etcd-controller
|
||||
state: directory
|
||||
mode: '0755'
|
||||
- name: 安装证书登录与调度程序
|
||||
ansible.builtin.copy:
|
||||
src: "../controller/{{ item }}"
|
||||
dest: "/opt/homelab-etcd-controller/{{ item }}"
|
||||
owner: root
|
||||
group: root
|
||||
mode: '0755'
|
||||
loop: [login.py, renew.py]
|
||||
- name: 以机器身份预检（不输出 token）
|
||||
ansible.builtin.command:
|
||||
argv: [/usr/bin/python3, /opt/homelab-etcd-controller/login.py]
|
||||
changed_when: false
|
||||
no_log: true
|
||||
- name: 安装固定的 Ansible 配置副本
|
||||
ansible.builtin.copy:
|
||||
src: "{{ playbook_dir }}/"
|
||||
dest: /opt/homelab-etcd-controller/ansible/
|
||||
owner: root
|
||||
group: root
|
||||
mode: preserve
|
||||
- name: 创建调度状态目录
|
||||
ansible.builtin.file:
|
||||
path: /var/lib/homelab-etcd-controller
|
||||
state: directory
|
||||
owner: panxiao81
|
||||
group: panxiao81
|
||||
mode: '0700'
|
||||
- name: 安装续签 oneshot unit
|
||||
ansible.builtin.copy:
|
||||
dest: /etc/systemd/system/homelab-etcd-renew.service
|
||||
mode: '0644'
|
||||
content: |
|
||||
[Unit]
|
||||
Description=Renew shared etcd certificates through Bao machine identity
|
||||
After=network-online.target
|
||||
[Service]
|
||||
Type=oneshot
|
||||
User=panxiao81
|
||||
Environment=HOME=/home/panxiao81
|
||||
Environment=PATH=/home/panxiao81/.local/bin:/usr/local/bin:/usr/bin:/bin
|
||||
ExecStart=/usr/bin/python3 /opt/homelab-etcd-controller/renew.py
|
||||
TimeoutStartSec=30min
|
||||
UMask=0077
|
||||
- name: 安装每日续签 timer
|
||||
ansible.builtin.copy:
|
||||
dest: /etc/systemd/system/homelab-etcd-renew.timer
|
||||
mode: '0644'
|
||||
content: |
|
||||
[Unit]
|
||||
Description=Daily shared etcd certificate renewal check
|
||||
[Timer]
|
||||
OnCalendar=*-*-* 04:10:00 UTC
|
||||
RandomizedDelaySec=15min
|
||||
Persistent=true
|
||||
[Install]
|
||||
WantedBy=timers.target
|
||||
- name: 启用续签调度
|
||||
ansible.builtin.systemd_service:
|
||||
name: homelab-etcd-renew.timer
|
||||
daemon_reload: true
|
||||
enabled: true
|
||||
state: started
|
||||
@@ -0,0 +1,20 @@
|
||||
---
|
||||
etcd_bao_url: https://bao.ad.ddupan.top:8200
|
||||
etcd_bao_pki_mount: pki
|
||||
etcd_bao_kv_mount: kv
|
||||
etcd_bao_secret_base: infra/etcd/consumers
|
||||
# 只在控制端使用，不下发 Bao token。也不读取或复制历史 .vault_pass。
|
||||
etcd_bao_token: "{{ lookup('env', 'BAO_TOKEN') }}"
|
||||
etcd_consumers:
|
||||
- name: patroni-pg-prod
|
||||
prefix: /homelab/patroni/pg-prod/
|
||||
etcd_lxc_template: laptop:vztmpl/ubuntu-24.04-standard_24.04-2_amd64.tar.zst
|
||||
etcd_lxc_storage: pve-rg
|
||||
etcd_lxc_bridge: labnet
|
||||
etcd_lxc_gateway: 10.60.0.1
|
||||
etcd_lxc_memory: 512
|
||||
etcd_lxc_disk_gb: 8
|
||||
etcd_lxc_pubkey: "{{ lookup('file', '~/.ssh/id_ed25519.pub') }}"
|
||||
|
||||
# 与现有 LAN exporter 相同的内网采集边界；禁止映射到公网。
|
||||
etcd_metrics_urls: "http://127.0.0.1:2381,http://{{ etcd_address }}:2381"
|
||||
@@ -0,0 +1,33 @@
|
||||
---
|
||||
# 已部署成员；共置 PG standby/备份的承载预算，数据库数据用独立 HDD mp0。
|
||||
all:
|
||||
vars:
|
||||
ansible_user: root
|
||||
children:
|
||||
etcd:
|
||||
hosts:
|
||||
etcd-laptop:
|
||||
ansible_connection: local
|
||||
ansible_host: 192.168.10.127
|
||||
ansible_user: panxiao81
|
||||
etcd_address: 192.168.10.127
|
||||
etcd-pve1:
|
||||
ansible_host: 10.60.0.20
|
||||
etcd_address: 10.60.0.20
|
||||
etcd-pve2:
|
||||
ansible_host: 10.60.0.21
|
||||
etcd_address: 10.60.0.21
|
||||
etcd_pve:
|
||||
hosts:
|
||||
pve1:
|
||||
ansible_host: 192.168.10.4
|
||||
etcd_lxc_vmid: 150
|
||||
etcd_lxc_hostname: etcd-pve1
|
||||
etcd_lxc_address: 10.60.0.20/24
|
||||
etcd_lxc_memory: 1536
|
||||
pve2:
|
||||
ansible_host: 192.168.10.7
|
||||
etcd_lxc_vmid: 151
|
||||
etcd_lxc_hostname: etcd-pve2
|
||||
etcd_lxc_address: 10.60.0.21/24
|
||||
etcd_lxc_memory: 768
|
||||
@@ -0,0 +1,126 @@
|
||||
---
|
||||
# 只创建已声明且尚不存在的 LXC；不接管未知 VMID，不重启现有容器。
|
||||
- name: 创建独立 etcd LXC
|
||||
hosts: etcd_pve
|
||||
become: true
|
||||
gather_facts: false
|
||||
tasks:
|
||||
- name: 读取全局资源，避免 VMID 在其他节点已占用
|
||||
ansible.builtin.command:
|
||||
argv: [pvesh, get, /cluster/resources, --type, vm, --output-format, json]
|
||||
register: etcd_pve_resources
|
||||
changed_when: false
|
||||
check_mode: false
|
||||
|
||||
- name: 保存同号资源
|
||||
ansible.builtin.set_fact:
|
||||
etcd_lxc_existing: >-
|
||||
{{ etcd_pve_resources.stdout
|
||||
| from_json
|
||||
| selectattr('vmid', 'equalto', etcd_lxc_vmid)
|
||||
| list }}
|
||||
|
||||
- name: 拒绝接管未知资源
|
||||
ansible.builtin.assert:
|
||||
that:
|
||||
- >-
|
||||
etcd_lxc_existing | length == 0 or
|
||||
(etcd_lxc_existing[0].type == 'lxc' and etcd_lxc_existing[0].node == inventory_hostname
|
||||
and etcd_lxc_existing[0].name == etcd_lxc_hostname
|
||||
and 'shared-etcd' in (etcd_lxc_existing[0].tags | default('')))
|
||||
|
||||
- name: 新建无特权 LXC
|
||||
when: etcd_lxc_existing | length == 0 and not ansible_check_mode
|
||||
block:
|
||||
- name: 暂存 SSH 公钥
|
||||
ansible.builtin.copy:
|
||||
content: "{{ etcd_lxc_pubkey }}\n"
|
||||
dest: /run/shared-etcd-bootstrap.pub
|
||||
mode: '0600'
|
||||
- name: 创建声明的容器
|
||||
ansible.builtin.command:
|
||||
argv:
|
||||
- pct
|
||||
- create
|
||||
- "{{ etcd_lxc_vmid }}"
|
||||
- "{{ etcd_lxc_template }}"
|
||||
- --hostname
|
||||
- "{{ etcd_lxc_hostname }}"
|
||||
- --unprivileged
|
||||
- '1'
|
||||
- --cores
|
||||
- '1'
|
||||
- --memory
|
||||
- "{{ etcd_lxc_memory }}"
|
||||
- --swap
|
||||
- '0'
|
||||
- --rootfs
|
||||
- "{{ etcd_lxc_storage }}:{{ etcd_lxc_disk_gb }}"
|
||||
- --net0
|
||||
- "name=eth0,bridge={{ etcd_lxc_bridge }},ip={{ etcd_lxc_address }},gw={{ etcd_lxc_gateway }},type=veth"
|
||||
- --nameserver
|
||||
- 192.168.10.5
|
||||
- --searchdomain
|
||||
- ad.ddupan.top
|
||||
- --ssh-public-keys
|
||||
- /run/shared-etcd-bootstrap.pub
|
||||
- --onboot
|
||||
- '1'
|
||||
- --tags
|
||||
- ansible;shared-etcd
|
||||
changed_when: true
|
||||
always:
|
||||
- name: 删除暂存公钥
|
||||
ansible.builtin.file:
|
||||
path: /run/shared-etcd-bootstrap.pub
|
||||
state: absent
|
||||
|
||||
- name: 读取容器配置
|
||||
ansible.builtin.command:
|
||||
argv: [pct, config, "{{ etcd_lxc_vmid }}"]
|
||||
changed_when: false
|
||||
register: etcd_lxc_config
|
||||
when: not ansible_check_mode or etcd_lxc_existing | length > 0
|
||||
|
||||
- name: 配置漂移先报错，不直接改运行中的网络/资源
|
||||
ansible.builtin.assert:
|
||||
that:
|
||||
- >-
|
||||
('ip=' ~ etcd_lxc_address ~ ',') in etcd_lxc_config.stdout or
|
||||
('ip=' ~ etcd_lxc_address ~ '\n') in etcd_lxc_config.stdout
|
||||
- "('bridge=' ~ etcd_lxc_bridge ~ ',') in etcd_lxc_config.stdout"
|
||||
- "'unprivileged: 1' in etcd_lxc_config.stdout"
|
||||
- "('memory: ' ~ etcd_lxc_memory) in etcd_lxc_config.stdout"
|
||||
- "('rootfs: ' ~ etcd_lxc_storage ~ ':') in etcd_lxc_config.stdout"
|
||||
when: etcd_lxc_config is not skipped
|
||||
|
||||
- name: 读取容器运行状态
|
||||
ansible.builtin.command:
|
||||
argv: [pct, status, "{{ etcd_lxc_vmid }}"]
|
||||
changed_when: false
|
||||
register: etcd_lxc_status
|
||||
when: not ansible_check_mode
|
||||
|
||||
- name: 启动容器
|
||||
ansible.builtin.command:
|
||||
argv: [pct, start, "{{ etcd_lxc_vmid }}"]
|
||||
changed_when: true
|
||||
when: not ansible_check_mode and 'running' not in etcd_lxc_status.stdout
|
||||
|
||||
- name: 通过可信宿主机取得容器 SSH 公钥
|
||||
ansible.builtin.command:
|
||||
argv: [pct, exec, "{{ etcd_lxc_vmid }}", --, cat, /etc/ssh/ssh_host_ed25519_key.pub]
|
||||
register: etcd_lxc_hostkey
|
||||
changed_when: false
|
||||
retries: 12
|
||||
delay: 5
|
||||
until: etcd_lxc_hostkey.rc == 0
|
||||
when: not ansible_check_mode
|
||||
|
||||
- name: 保存经宿主机验证的 SSH host key
|
||||
ansible.builtin.known_hosts:
|
||||
name: "{{ etcd_lxc_address.split('/')[0] }}"
|
||||
key: "{{ etcd_lxc_address.split('/')[0] }} {{ etcd_lxc_hostkey.stdout }}"
|
||||
delegate_to: localhost
|
||||
become: false
|
||||
when: not ansible_check_mode
|
||||
@@ -0,0 +1,90 @@
|
||||
---
|
||||
# 首次部署承载调整，逐节点迁移；普通 lxc.yml 不自动移动磁盘。
|
||||
- name: 逐个将新 etcd 容器迁入声明的 SSD 池
|
||||
hosts: etcd_pve
|
||||
become: true
|
||||
gather_facts: false
|
||||
serial: 1
|
||||
any_errors_fatal: true
|
||||
vars:
|
||||
etcd_move_health_command:
|
||||
- /opt/homelab-etcd/etcdctl
|
||||
- --endpoints=https://192.168.10.127:2379,https://10.60.0.20:2379,https://10.60.0.21:2379
|
||||
- --cacert=/etc/homelab-etcd/ca.crt
|
||||
- --cert=/etc/homelab-etcd/admin.crt
|
||||
- --key=/etc/homelab-etcd/admin.key
|
||||
- endpoint
|
||||
- health
|
||||
tasks:
|
||||
- name: 核对容器配置
|
||||
ansible.builtin.command:
|
||||
argv: [pct, config, "{{ etcd_lxc_vmid }}"]
|
||||
register: etcd_move_config
|
||||
changed_when: false
|
||||
check_mode: false
|
||||
|
||||
- name: 限定本项目新建容器与允许的源池
|
||||
ansible.builtin.assert:
|
||||
that:
|
||||
- etcd_lxc_vmid in [150, 151]
|
||||
- "('hostname: ' ~ etcd_lxc_hostname) in etcd_move_config.stdout"
|
||||
- "'shared-etcd' in etcd_move_config.stdout"
|
||||
- "'rootfs: local-lvm:' in etcd_move_config.stdout or 'rootfs: pve-rg:' in etcd_move_config.stdout"
|
||||
- etcd_lxc_storage == 'pve-rg'
|
||||
|
||||
- name: 验证迁移前全部端点健康
|
||||
ansible.builtin.command:
|
||||
argv: "{{ etcd_move_health_command }}"
|
||||
delegate_to: localhost
|
||||
changed_when: false
|
||||
check_mode: false
|
||||
|
||||
- name: 迁移当前尚在本地池的根卷
|
||||
when: "'rootfs: local-lvm:' in etcd_move_config.stdout and not ansible_check_mode"
|
||||
block:
|
||||
- name: 创建已验证的集群快照
|
||||
ansible.builtin.command:
|
||||
argv: [systemctl, start, homelab-etcd-snapshot.service]
|
||||
delegate_to: localhost
|
||||
changed_when: true
|
||||
|
||||
- name: 正常关闭一个容器
|
||||
ansible.builtin.command:
|
||||
argv: [pct, shutdown, "{{ etcd_lxc_vmid }}", --timeout, '60']
|
||||
changed_when: true
|
||||
|
||||
- name: 复制成功后移除该新建容器的原卷
|
||||
ansible.builtin.command:
|
||||
argv:
|
||||
- pct
|
||||
- move-volume
|
||||
- "{{ etcd_lxc_vmid }}"
|
||||
- rootfs
|
||||
- "{{ etcd_lxc_storage }}"
|
||||
- --delete
|
||||
- '1'
|
||||
- --bwlimit
|
||||
- '32768'
|
||||
changed_when: true
|
||||
always:
|
||||
- name: 核对容器运行状态
|
||||
ansible.builtin.command:
|
||||
argv: [pct, status, "{{ etcd_lxc_vmid }}"]
|
||||
register: etcd_move_status
|
||||
changed_when: false
|
||||
- name: 重新启动容器
|
||||
ansible.builtin.command:
|
||||
argv: [pct, start, "{{ etcd_lxc_vmid }}"]
|
||||
when: "'running' not in etcd_move_status.stdout"
|
||||
changed_when: true
|
||||
|
||||
- name: 等待当前成员回归
|
||||
ansible.builtin.command:
|
||||
argv: "{{ etcd_move_health_command }}"
|
||||
delegate_to: localhost
|
||||
changed_when: false
|
||||
register: etcd_move_health
|
||||
retries: 18
|
||||
delay: 5
|
||||
until: etcd_move_health.rc == 0
|
||||
when: not ansible_check_mode
|
||||
@@ -0,0 +1,4 @@
|
||||
---
|
||||
collections:
|
||||
- name: community.crypto
|
||||
version: 3.2.1
|
||||
@@ -0,0 +1,17 @@
|
||||
---
|
||||
etcd_version: 3.7.2
|
||||
etcd_archive_checksum: sha256:3a3679bc51a4ee9d30bccea1da7cd4fe62c6fc1d2ca1255068d2c53bf3026135
|
||||
etcd_install_dir: /opt/homelab-etcd
|
||||
etcd_config_dir: /etc/homelab-etcd
|
||||
etcd_data_dir: /var/lib/homelab-etcd
|
||||
etcd_cluster_token: homelab-shared-etcd-v1
|
||||
etcd_client_port: 2379
|
||||
etcd_peer_port: 2380
|
||||
etcd_metrics_port: 2381
|
||||
etcd_quota_bytes: 268435456
|
||||
etcd_memory_high: 256M
|
||||
etcd_memory_max: 384M
|
||||
etcd_certificate_ttl: 1440h
|
||||
etcd_renew_before: +14d
|
||||
etcd_snapshot_dir: /var/backups/homelab-etcd
|
||||
etcd_snapshot_keep: 3
|
||||
@@ -0,0 +1,93 @@
|
||||
---
|
||||
- name: 在成员本地生成私钥
|
||||
community.crypto.openssl_privatekey:
|
||||
path: "{{ etcd_config_dir }}/{{ etcd_cert.name }}.key"
|
||||
type: ECC
|
||||
curve: secp256r1
|
||||
owner: root
|
||||
group: "{{ etcd_cert.group }}"
|
||||
mode: '0640'
|
||||
|
||||
- name: 本地生成 CSR
|
||||
community.crypto.openssl_csr:
|
||||
path: "{{ etcd_config_dir }}/{{ etcd_cert.name }}.csr"
|
||||
privatekey_path: "{{ etcd_config_dir }}/{{ etcd_cert.name }}.key"
|
||||
common_name: "{{ etcd_cert.cn | default(omit, true) }}"
|
||||
subject_alt_name: >-
|
||||
{{ ['IP:' ~ etcd_address, 'DNS:' ~ inventory_hostname] if etcd_cert.name in ['server', 'peer']
|
||||
else (['DNS:' ~ inventory_hostname] if etcd_cert.name == 'gateway' else []) }}
|
||||
use_common_name_for_san: false
|
||||
extended_key_usage: "{{ etcd_cert.eku }}"
|
||||
key_usage: [digitalSignature]
|
||||
mode: '0644'
|
||||
register: etcd_csr_state
|
||||
|
||||
- name: 检查已有证书
|
||||
ansible.builtin.stat:
|
||||
path: "{{ etcd_config_dir }}/{{ etcd_cert.name }}.crt"
|
||||
register: etcd_cert_file
|
||||
|
||||
- name: 检查续签窗口
|
||||
community.crypto.x509_certificate_info:
|
||||
path: "{{ etcd_config_dir }}/{{ etcd_cert.name }}.crt"
|
||||
valid_at:
|
||||
renewal: "{{ etcd_renew_before }}"
|
||||
register: etcd_cert_info
|
||||
when: etcd_cert_file.stat.exists
|
||||
|
||||
- name: 通过中央 CA 签发需更新的证书
|
||||
when: >-
|
||||
not etcd_cert_file.stat.exists or etcd_csr_state is changed or
|
||||
not (etcd_cert_info.valid_at.renewal | default(false))
|
||||
block:
|
||||
- name: 仅在确需签发时要求 Bao 凭据
|
||||
ansible.builtin.assert:
|
||||
that: etcd_bao_token | length > 0
|
||||
no_log: true
|
||||
|
||||
- name: 读取 CSR 公共内容
|
||||
ansible.builtin.slurp:
|
||||
src: "{{ etcd_config_dir }}/{{ etcd_cert.name }}.csr"
|
||||
register: etcd_csr
|
||||
|
||||
- name: 控制端提交 Bao 签名请求
|
||||
ansible.builtin.uri:
|
||||
url: "{{ etcd_bao_url }}/v1/{{ etcd_bao_pki_mount }}/sign/{{ etcd_cert.role }}"
|
||||
method: POST
|
||||
headers:
|
||||
X-Vault-Token: "{{ etcd_bao_token }}"
|
||||
body_format: json
|
||||
body:
|
||||
csr: "{{ etcd_csr.content | b64decode }}"
|
||||
ttl: "{{ etcd_certificate_ttl }}"
|
||||
status_code: 200
|
||||
delegate_to: localhost
|
||||
become: false
|
||||
register: etcd_signed
|
||||
no_log: true
|
||||
when: not ansible_check_mode
|
||||
|
||||
- name: 保存签发的证书链
|
||||
ansible.builtin.copy:
|
||||
content: |
|
||||
{{ etcd_signed.json.data.certificate }}
|
||||
{{ etcd_signed.json.data.ca_chain | join('\n') }}
|
||||
dest: "{{ etcd_config_dir }}/{{ etcd_cert.name }}.crt"
|
||||
owner: root
|
||||
group: "{{ etcd_cert.group }}"
|
||||
mode: '0644'
|
||||
when: not ansible_check_mode
|
||||
|
||||
- name: 保存中央 CA 信任链
|
||||
ansible.builtin.copy:
|
||||
content: |
|
||||
{{ etcd_signed.json.data.ca_chain | join('\n') }}
|
||||
dest: "{{ etcd_config_dir }}/ca.crt"
|
||||
owner: root
|
||||
group: homelab-etcd
|
||||
mode: '0644'
|
||||
when: not ansible_check_mode
|
||||
|
||||
- name: 标记证书已更新
|
||||
ansible.builtin.set_fact:
|
||||
etcd_certificates_changed: true
|
||||
@@ -0,0 +1,3 @@
|
||||
---
|
||||
# 只加载 role defaults，供运维入口复用。
|
||||
[]
|
||||
@@ -0,0 +1,134 @@
|
||||
---
|
||||
- name: 验证拓扑和签发配置
|
||||
ansible.builtin.assert:
|
||||
that:
|
||||
- groups['etcd'] | length == 3
|
||||
- groups['etcd'] | map('extract', hostvars, 'etcd_address') | unique | length == 3
|
||||
- ansible_facts['architecture'] == 'x86_64'
|
||||
- etcd_bao_url is match('^https://')
|
||||
- etcd_archive_checksum is match('^sha256:[a-f0-9]{64}$')
|
||||
no_log: true
|
||||
|
||||
- name: 安装证书处理依赖
|
||||
ansible.builtin.package:
|
||||
name: [python3-cryptography, openssl]
|
||||
state: present
|
||||
|
||||
- name: 创建独立 etcd 组
|
||||
ansible.builtin.group:
|
||||
name: homelab-etcd
|
||||
system: true
|
||||
|
||||
- name: 创建独立 etcd 用户
|
||||
ansible.builtin.user:
|
||||
name: homelab-etcd
|
||||
group: homelab-etcd
|
||||
system: true
|
||||
shell: /usr/sbin/nologin
|
||||
create_home: false
|
||||
|
||||
- name: 创建受管目录
|
||||
ansible.builtin.file:
|
||||
path: "{{ item.path }}"
|
||||
state: directory
|
||||
owner: "{{ item.owner }}"
|
||||
group: homelab-etcd
|
||||
mode: "{{ item.mode }}"
|
||||
loop:
|
||||
- {path: "{{ etcd_config_dir }}", owner: root, mode: '0750'}
|
||||
- {path: "{{ etcd_data_dir }}", owner: homelab-etcd, mode: '0700'}
|
||||
- {path: "{{ etcd_install_dir }}", owner: root, mode: '0755'}
|
||||
|
||||
- name: 下载固定版本及校验归档
|
||||
ansible.builtin.get_url:
|
||||
url: >-
|
||||
https://github.com/etcd-io/etcd/releases/download/v{{ etcd_version }}/etcd-v{{ etcd_version }}-linux-amd64.tar.gz
|
||||
dest: "{{ etcd_install_dir }}/etcd-v{{ etcd_version }}.tar.gz"
|
||||
checksum: "{{ etcd_archive_checksum }}"
|
||||
mode: '0644'
|
||||
register: etcd_download
|
||||
retries: 3
|
||||
delay: 5
|
||||
until: etcd_download is succeeded
|
||||
|
||||
- name: 展开固定版本
|
||||
ansible.builtin.unarchive:
|
||||
src: "{{ etcd_install_dir }}/etcd-v{{ etcd_version }}.tar.gz"
|
||||
dest: "{{ etcd_install_dir }}"
|
||||
remote_src: true
|
||||
creates: "{{ etcd_install_dir }}/etcd-v{{ etcd_version }}-linux-amd64/etcd"
|
||||
|
||||
- name: 安装版本链接
|
||||
ansible.builtin.file:
|
||||
src: "{{ etcd_install_dir }}/etcd-v{{ etcd_version }}-linux-amd64/{{ item }}"
|
||||
dest: "{{ etcd_install_dir }}/{{ item }}"
|
||||
state: link
|
||||
loop: [etcd, etcdctl, etcdutl]
|
||||
register: etcd_binary_links
|
||||
|
||||
- name: 签发成员与管理员证书
|
||||
ansible.builtin.include_tasks: certificate.yml
|
||||
loop:
|
||||
- {name: server, role: homelab-etcd-server, cn: "{{ inventory_hostname }}", eku: [serverAuth], group: homelab-etcd}
|
||||
- {name: peer, role: homelab-etcd-peer, cn: homelab-etcd-peer, eku: [serverAuth, clientAuth], group: homelab-etcd}
|
||||
- {name: gateway, role: homelab-etcd-gateway, cn: "", eku: [clientAuth], group: homelab-etcd}
|
||||
- {name: admin, role: homelab-etcd-admin, cn: root, eku: [clientAuth], group: root}
|
||||
loop_control:
|
||||
loop_var: etcd_cert
|
||||
|
||||
- name: 写入独立 etcd 配置
|
||||
ansible.builtin.template:
|
||||
src: etcd.yml.j2
|
||||
dest: "{{ etcd_config_dir }}/etcd.yml"
|
||||
owner: root
|
||||
group: homelab-etcd
|
||||
mode: '0640'
|
||||
register: etcd_config_file
|
||||
|
||||
- name: 写入独立 systemd unit
|
||||
ansible.builtin.template:
|
||||
src: homelab-etcd.service.j2
|
||||
dest: /etc/systemd/system/homelab-etcd.service
|
||||
mode: '0644'
|
||||
register: etcd_unit
|
||||
|
||||
# 跨失败重跑记录激活状态，防止上一轮写文件后中断导致漏掉必要重启。
|
||||
- name: 计算受管文件校验和
|
||||
ansible.builtin.stat:
|
||||
path: "{{ item }}"
|
||||
checksum_algorithm: sha256
|
||||
loop:
|
||||
- "{{ etcd_config_dir }}/etcd.yml"
|
||||
- "{{ etcd_config_dir }}/server.crt"
|
||||
- "{{ etcd_config_dir }}/peer.crt"
|
||||
- "{{ etcd_config_dir }}/gateway.crt"
|
||||
- "{{ etcd_config_dir }}/ca.crt"
|
||||
- /etc/systemd/system/homelab-etcd.service
|
||||
register: etcd_managed_files
|
||||
|
||||
- name: 计算期望激活指纹
|
||||
ansible.builtin.set_fact:
|
||||
etcd_config_fingerprint: >-
|
||||
{{ ((etcd_managed_files.results | map(attribute='stat.checksum') | list | join(':'))
|
||||
~ ':' ~ etcd_version) | hash('sha256') }}
|
||||
when: not ansible_check_mode
|
||||
|
||||
- name: 检查已激活指纹
|
||||
ansible.builtin.stat:
|
||||
path: "{{ etcd_config_dir }}/activated.sha256"
|
||||
register: etcd_activated_file
|
||||
|
||||
- name: 读取已激活指纹
|
||||
ansible.builtin.slurp:
|
||||
src: "{{ etcd_config_dir }}/activated.sha256"
|
||||
register: etcd_activated
|
||||
when: etcd_activated_file.stat.exists
|
||||
|
||||
- name: 判断是否需要滚动激活
|
||||
ansible.builtin.set_fact:
|
||||
etcd_config_changed: >-
|
||||
{{ not etcd_activated_file.stat.exists or
|
||||
(etcd_activated.content | default('') | b64decode | trim) != etcd_config_fingerprint | default('check-mode') }}
|
||||
|
||||
- name: 管理本地快照任务
|
||||
ansible.builtin.import_tasks: snapshot.yml
|
||||
@@ -0,0 +1,26 @@
|
||||
---
|
||||
- name: 创建仅 root 可访问的快照目录
|
||||
ansible.builtin.file:
|
||||
path: "{{ etcd_snapshot_dir }}"
|
||||
state: directory
|
||||
owner: root
|
||||
group: root
|
||||
mode: '0700'
|
||||
|
||||
- name: 写入快照脚本和 systemd 任务
|
||||
ansible.builtin.template:
|
||||
src: "{{ item.src }}"
|
||||
dest: "{{ item.dest }}"
|
||||
mode: "{{ item.mode }}"
|
||||
loop:
|
||||
- {src: snapshot.sh.j2, dest: "{{ etcd_install_dir }}/snapshot", mode: '0700'}
|
||||
- {src: snapshot.service.j2, dest: /etc/systemd/system/homelab-etcd-snapshot.service, mode: '0644'}
|
||||
- {src: snapshot.timer.j2, dest: /etc/systemd/system/homelab-etcd-snapshot.timer, mode: '0644'}
|
||||
register: etcd_snapshot_units
|
||||
|
||||
- name: 启用本地快照计划
|
||||
ansible.builtin.systemd_service:
|
||||
name: homelab-etcd-snapshot.timer
|
||||
daemon_reload: "{{ etcd_snapshot_units is changed }}"
|
||||
enabled: true
|
||||
state: started
|
||||
@@ -0,0 +1,33 @@
|
||||
# Ansible 管理；与 k3s、数据库生命周期独立。
|
||||
name: {{ inventory_hostname | to_json }}
|
||||
data-dir: {{ etcd_data_dir | to_json }}
|
||||
listen-client-urls: https://{{ etcd_address }}:{{ etcd_client_port }}
|
||||
advertise-client-urls: https://{{ etcd_address }}:{{ etcd_client_port }}
|
||||
listen-peer-urls: https://{{ etcd_address }}:{{ etcd_peer_port }}
|
||||
initial-advertise-peer-urls: https://{{ etcd_address }}:{{ etcd_peer_port }}
|
||||
initial-cluster: "{% for member in groups['etcd'] %}{{ member }}=https://{{ hostvars[member].etcd_address }}:{{ etcd_peer_port }}{{ ',' if not loop.last else '' }}{% endfor %}"
|
||||
initial-cluster-token: {{ etcd_cluster_token | to_json }}
|
||||
initial-cluster-state: new
|
||||
# 已存在的数据目录优先；成员替换必须走单独 runbook，不删除数据重建。
|
||||
client-transport-security:
|
||||
cert-file: {{ etcd_config_dir }}/server.crt
|
||||
key-file: {{ etcd_config_dir }}/server.key
|
||||
client-cert-file: {{ etcd_config_dir }}/gateway.crt
|
||||
client-key-file: {{ etcd_config_dir }}/gateway.key
|
||||
trusted-ca-file: {{ etcd_config_dir }}/ca.crt
|
||||
client-cert-auth: true
|
||||
peer-transport-security:
|
||||
cert-file: {{ etcd_config_dir }}/peer.crt
|
||||
key-file: {{ etcd_config_dir }}/peer.key
|
||||
trusted-ca-file: {{ etcd_config_dir }}/ca.crt
|
||||
client-cert-auth: true
|
||||
allowed-cn: [homelab-etcd-peer]
|
||||
# 独立 metrics listener 仅提供指标/健康，不开放 KV API；只绑定受管内网地址。
|
||||
listen-metrics-urls: {{ etcd_metrics_urls | default("http://127.0.0.1:" ~ etcd_metrics_port) | to_json }}
|
||||
quota-backend-bytes: {{ etcd_quota_bytes }}
|
||||
auto-compaction-mode: periodic
|
||||
auto-compaction-retention: '1h'
|
||||
heartbeat-interval: 100
|
||||
election-timeout: 1000
|
||||
logger: zap
|
||||
log-level: info
|
||||
@@ -0,0 +1,23 @@
|
||||
[Unit]
|
||||
Description=Homelab shared etcd
|
||||
Wants=network-online.target
|
||||
After=network-online.target
|
||||
|
||||
[Service]
|
||||
User=homelab-etcd
|
||||
Group=homelab-etcd
|
||||
ExecStart={{ etcd_install_dir }}/etcd --config-file={{ etcd_config_dir }}/etcd.yml
|
||||
Restart=on-failure
|
||||
RestartSec=5
|
||||
TimeoutStopSec=60
|
||||
MemoryHigh={{ etcd_memory_high }}
|
||||
MemoryMax={{ etcd_memory_max }}
|
||||
UMask=0077
|
||||
NoNewPrivileges=true
|
||||
ProtectSystem=strict
|
||||
ProtectHome=true
|
||||
PrivateTmp=true
|
||||
ReadWritePaths={{ etcd_data_dir }}
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
@@ -0,0 +1,11 @@
|
||||
[Unit]
|
||||
Description=Snapshot homelab shared etcd
|
||||
After=homelab-etcd.service
|
||||
|
||||
[Service]
|
||||
Type=oneshot
|
||||
ExecStart={{ etcd_install_dir }}/snapshot
|
||||
User=root
|
||||
UMask=0077
|
||||
TimeoutStartSec=300
|
||||
Nice=10
|
||||
@@ -0,0 +1,20 @@
|
||||
#!/bin/bash
|
||||
set -euo pipefail
|
||||
umask 077
|
||||
export ETCDCTL_ENDPOINTS="https://{{ etcd_address }}:{{ etcd_client_port }}"
|
||||
export ETCDCTL_CACERT="{{ etcd_config_dir }}/ca.crt"
|
||||
export ETCDCTL_CERT="{{ etcd_config_dir }}/admin.crt"
|
||||
export ETCDCTL_KEY="{{ etcd_config_dir }}/admin.key"
|
||||
repo="{{ etcd_snapshot_dir }}"
|
||||
exec 9>"$repo/.lock"
|
||||
flock -n 9 || exit 0
|
||||
output="$repo/$(date -u +%Y%m%dT%H%M%SZ).db"
|
||||
trap 'rm -f "$output.partial" "$output.partial.part"' EXIT
|
||||
{{ etcd_install_dir }}/etcdctl snapshot save "$output.partial"
|
||||
{{ etcd_install_dir }}/etcdutl snapshot status "$output.partial" >/dev/null
|
||||
mv "$output.partial" "$output"
|
||||
# 只有新快照成功且验证可读才清理旧备份。目录仅由此任务管理。
|
||||
mapfile -t snapshots < <(find "$repo" -maxdepth 1 -type f -name '????????T??????Z.db' -printf '%f\n' | sort -r)
|
||||
for old in "${snapshots[@]:{{ etcd_snapshot_keep }}}"; do
|
||||
rm -- "$repo/$old"
|
||||
done
|
||||
@@ -0,0 +1,10 @@
|
||||
[Unit]
|
||||
Description=Daily homelab etcd snapshot
|
||||
|
||||
[Timer]
|
||||
OnCalendar=*-*-* 03:20:00 UTC
|
||||
RandomizedDelaySec=300
|
||||
Persistent=true
|
||||
|
||||
[Install]
|
||||
WantedBy=timers.target
|
||||
@@ -0,0 +1,66 @@
|
||||
---
|
||||
# 先配置全体，再滚动启动：首次集群形成不能在第一个节点等待 quorum。
|
||||
- name: 配置 shared etcd 成员
|
||||
hosts: etcd
|
||||
become: true
|
||||
roles:
|
||||
- shared_etcd
|
||||
|
||||
- name: 滚动启动并验证 shared etcd
|
||||
hosts: etcd
|
||||
become: true
|
||||
serial: 1
|
||||
pre_tasks:
|
||||
- name: 加载共享默认参数
|
||||
ansible.builtin.import_role:
|
||||
name: shared_etcd
|
||||
tasks_from: context
|
||||
tasks:
|
||||
- name: 启动已配置的成员
|
||||
ansible.builtin.systemd_service:
|
||||
name: homelab-etcd
|
||||
enabled: true
|
||||
daemon_reload: true
|
||||
state: "{{ 'restarted' if etcd_config_changed | bool else 'started' }}"
|
||||
- name: 等待本机客户端端口
|
||||
ansible.builtin.wait_for:
|
||||
host: "{{ etcd_address }}"
|
||||
port: "{{ etcd_client_port }}"
|
||||
timeout: 60
|
||||
when: not ansible_check_mode
|
||||
|
||||
- name: 已有集群每次激活后等待本成员恢复 quorum 通信
|
||||
ansible.builtin.command:
|
||||
argv:
|
||||
- "{{ etcd_install_dir }}/etcdctl"
|
||||
- --endpoints=https://{{ etcd_address }}:{{ etcd_client_port }}
|
||||
- --cacert={{ etcd_config_dir }}/ca.crt
|
||||
- --cert={{ etcd_config_dir }}/admin.crt
|
||||
- --key={{ etcd_config_dir }}/admin.key
|
||||
- endpoint
|
||||
- health
|
||||
changed_when: false
|
||||
register: etcd_member_health
|
||||
retries: 12
|
||||
delay: 5
|
||||
until: etcd_member_health.rc == 0
|
||||
when: etcd_activated_file.stat.exists and not ansible_check_mode
|
||||
|
||||
- name: 核对全部成员
|
||||
ansible.builtin.import_playbook: verify.yml
|
||||
|
||||
- name: 记录已成功激活的配置
|
||||
hosts: etcd
|
||||
become: true
|
||||
pre_tasks:
|
||||
- name: 加载共享默认参数
|
||||
ansible.builtin.import_role:
|
||||
name: shared_etcd
|
||||
tasks_from: context
|
||||
tasks:
|
||||
- name: 写入激活指纹（全体健康检查通过后）
|
||||
ansible.builtin.copy:
|
||||
content: "{{ etcd_config_fingerprint }}\n"
|
||||
dest: "{{ etcd_config_dir }}/activated.sha256"
|
||||
mode: '0644'
|
||||
when: not ansible_check_mode
|
||||
@@ -0,0 +1,170 @@
|
||||
---
|
||||
- name: 验证消费者范围
|
||||
ansible.builtin.assert:
|
||||
that:
|
||||
- etcd_consumer.name is match('^[a-z][a-z0-9-]+$')
|
||||
- etcd_consumer.name != 'root'
|
||||
- etcd_consumer.prefix is match('^/homelab/[a-zA-Z0-9/_-]+/$')
|
||||
- etcd_consumer.prefix | length > 10
|
||||
|
||||
- name: 读取已有用户和角色
|
||||
ansible.builtin.command:
|
||||
argv: ["{{ etcd_install_dir }}/etcdctl", --write-out=json, "{{ item }}", list]
|
||||
loop: [user, role]
|
||||
register: etcd_identities
|
||||
changed_when: false
|
||||
check_mode: false
|
||||
|
||||
# 404 data 也可能代表被删除/销毁的旧秘密；只有 metadata 不存在才允许生成。
|
||||
- name: 控制端读取 Bao 秘密元数据
|
||||
ansible.builtin.uri:
|
||||
url: "{{ etcd_bao_url }}/v1/{{ etcd_bao_kv_mount }}/metadata/{{ etcd_bao_secret_base }}/{{ etcd_consumer.name }}"
|
||||
headers:
|
||||
X-Vault-Token: "{{ etcd_bao_token }}"
|
||||
status_code: [200, 404]
|
||||
delegate_to: localhost
|
||||
become: false
|
||||
no_log: true
|
||||
register: etcd_secret_metadata
|
||||
check_mode: false
|
||||
|
||||
- name: 禁止已有用户丢失秘密后自动换密码
|
||||
ansible.builtin.assert:
|
||||
that: >-
|
||||
etcd_secret_metadata.status == 200 or
|
||||
etcd_consumer.name not in ((etcd_identities.results[0].stdout | from_json).users | default([], true))
|
||||
fail_msg: etcd 用户已存在但 Bao 秘密缺失；需恢复原秘密或执行显式轮换。
|
||||
|
||||
- name: 首次创建随机秘密且禁止覆盖已有版本
|
||||
ansible.builtin.uri:
|
||||
url: "{{ etcd_bao_url }}/v1/{{ etcd_bao_kv_mount }}/data/{{ etcd_bao_secret_base }}/{{ etcd_consumer.name }}"
|
||||
method: POST
|
||||
headers:
|
||||
X-Vault-Token: "{{ etcd_bao_token }}"
|
||||
body_format: json
|
||||
body:
|
||||
options: {cas: 0}
|
||||
data:
|
||||
username: "{{ etcd_consumer.name }}"
|
||||
password: "{{ lookup('ansible.builtin.password', '/dev/null', length=48, chars=['ascii_letters', 'digits']) }}"
|
||||
prefix: "{{ etcd_consumer.prefix }}"
|
||||
status_code: 200
|
||||
delegate_to: localhost
|
||||
become: false
|
||||
no_log: true
|
||||
changed_when: true
|
||||
when:
|
||||
- etcd_secret_metadata.status == 404
|
||||
- not ansible_check_mode
|
||||
|
||||
- name: 控制端读取既有秘密
|
||||
ansible.builtin.uri:
|
||||
url: "{{ etcd_bao_url }}/v1/{{ etcd_bao_kv_mount }}/data/{{ etcd_bao_secret_base }}/{{ etcd_consumer.name }}"
|
||||
headers:
|
||||
X-Vault-Token: "{{ etcd_bao_token }}"
|
||||
status_code: 200
|
||||
delegate_to: localhost
|
||||
become: false
|
||||
register: etcd_consumer_secret
|
||||
no_log: true
|
||||
when: etcd_secret_metadata.status == 200 or not ansible_check_mode
|
||||
check_mode: false
|
||||
|
||||
- name: 核对已保存秘密归属
|
||||
ansible.builtin.assert:
|
||||
that:
|
||||
- etcd_consumer_secret.json.data.data.username == etcd_consumer.name
|
||||
- etcd_consumer_secret.json.data.data.prefix == etcd_consumer.prefix
|
||||
- etcd_consumer_secret.json.data.data.password | length >= 32
|
||||
no_log: true
|
||||
when: etcd_secret_metadata.status == 200 or not ansible_check_mode
|
||||
|
||||
- name: 新建消费者用户（密码只通过 stdin 传递）
|
||||
ansible.builtin.command:
|
||||
argv: ["{{ etcd_install_dir }}/etcdctl", user, add, "{{ etcd_consumer.name }}", --interactive=false]
|
||||
stdin: "{{ etcd_consumer_secret.json.data.data.password }}"
|
||||
no_log: true
|
||||
changed_when: true
|
||||
when:
|
||||
- etcd_consumer.name not in ((etcd_identities.results[0].stdout | from_json).users | default([], true))
|
||||
- not ansible_check_mode
|
||||
|
||||
- name: 新建消费者角色
|
||||
ansible.builtin.command:
|
||||
argv: ["{{ etcd_install_dir }}/etcdctl", role, add, "{{ etcd_consumer.name }}"]
|
||||
changed_when: true
|
||||
when: etcd_consumer.name not in ((etcd_identities.results[1].stdout | from_json).roles | default([], true))
|
||||
|
||||
- name: 读取角色权限
|
||||
ansible.builtin.command:
|
||||
argv: ["{{ etcd_install_dir }}/etcdctl", --write-out=json, role, get, "{{ etcd_consumer.name }}"]
|
||||
register: etcd_role_state
|
||||
changed_when: false
|
||||
when: >-
|
||||
not ansible_check_mode or
|
||||
etcd_consumer.name in ((etcd_identities.results[1].stdout | from_json).roles | default([], true))
|
||||
|
||||
- name: 拒绝不符合声明的既有权限（不自动扩大或删除）
|
||||
ansible.builtin.assert:
|
||||
that:
|
||||
- >-
|
||||
((etcd_role_state.stdout | from_json).perm | default([], true)) in
|
||||
[[], [{'permType': 2, 'key': etcd_consumer.prefix | b64encode,
|
||||
'range_end': (etcd_consumer.prefix[:-1] ~ '0') | b64encode}]]
|
||||
fail_msg: 既有角色权限与声明不同，请显式审查权限迁移。
|
||||
when: etcd_role_state is not skipped
|
||||
|
||||
- name: 授予唯一 prefix 读写权限
|
||||
ansible.builtin.command:
|
||||
argv:
|
||||
- "{{ etcd_install_dir }}/etcdctl"
|
||||
- role
|
||||
- grant-permission
|
||||
- "{{ etcd_consumer.name }}"
|
||||
- readwrite
|
||||
- "{{ etcd_consumer.prefix }}"
|
||||
- --prefix=true
|
||||
changed_when: true
|
||||
when:
|
||||
- etcd_role_state is not skipped
|
||||
- ((etcd_role_state.stdout | from_json).perm | default([], true)) | length == 0
|
||||
|
||||
- name: 读取用户角色
|
||||
ansible.builtin.command:
|
||||
argv: ["{{ etcd_install_dir }}/etcdctl", --write-out=json, user, get, "{{ etcd_consumer.name }}"]
|
||||
register: etcd_user_state
|
||||
changed_when: false
|
||||
when: >-
|
||||
not ansible_check_mode or
|
||||
etcd_consumer.name in ((etcd_identities.results[0].stdout | from_json).users | default([], true))
|
||||
|
||||
- name: 拒绝消费者已有额外角色
|
||||
ansible.builtin.assert:
|
||||
that: >-
|
||||
((etcd_user_state.stdout | from_json).roles | default([], true))
|
||||
| difference([etcd_consumer.name]) | length == 0
|
||||
when: etcd_user_state is not skipped
|
||||
|
||||
- name: 绑定消费者角色
|
||||
ansible.builtin.command:
|
||||
argv: ["{{ etcd_install_dir }}/etcdctl", user, grant-role, "{{ etcd_consumer.name }}", "{{ etcd_consumer.name }}"]
|
||||
changed_when: true
|
||||
when:
|
||||
- etcd_user_state is not skipped
|
||||
- etcd_consumer.name not in ((etcd_user_state.stdout | from_json).roles | default([], true))
|
||||
|
||||
# 使用 gateway 的真实密码登录来核对 Bao 与 etcd 一致性，不能只看用户已存在。
|
||||
- name: 验证消费者密码可经 gateway 登录
|
||||
ansible.builtin.uri:
|
||||
url: "https://{{ etcd_address }}:{{ etcd_client_port }}/v3/auth/authenticate"
|
||||
method: POST
|
||||
client_cert: "{{ etcd_config_dir }}/gateway.crt"
|
||||
client_key: "{{ etcd_config_dir }}/gateway.key"
|
||||
ca_path: "{{ etcd_config_dir }}/ca.crt"
|
||||
body_format: json
|
||||
body:
|
||||
name: "{{ etcd_consumer.name }}"
|
||||
password: "{{ etcd_consumer_secret.json.data.data.password }}"
|
||||
status_code: 200
|
||||
no_log: true
|
||||
when: not ansible_check_mode
|
||||
@@ -0,0 +1,29 @@
|
||||
---
|
||||
- name: 验证 shared etcd 全部端点
|
||||
hosts: etcd[0]
|
||||
become: true
|
||||
gather_facts: false
|
||||
pre_tasks:
|
||||
- name: 加载共享默认参数
|
||||
ansible.builtin.import_role:
|
||||
name: shared_etcd
|
||||
tasks_from: context
|
||||
tasks:
|
||||
- name: 通过管理员 mTLS 检查所有端点健康
|
||||
ansible.builtin.command:
|
||||
argv:
|
||||
- "{{ etcd_install_dir }}/etcdctl"
|
||||
- >-
|
||||
--endpoints={{ groups['etcd'] | map('extract', hostvars, 'etcd_address')
|
||||
| map('regex_replace', '^(.*)$', 'https://\1:' ~ etcd_client_port) | join(',') }}
|
||||
- --cacert={{ etcd_config_dir }}/ca.crt
|
||||
- --cert={{ etcd_config_dir }}/admin.crt
|
||||
- --key={{ etcd_config_dir }}/admin.key
|
||||
- endpoint
|
||||
- health
|
||||
changed_when: false
|
||||
register: etcd_health
|
||||
retries: 12
|
||||
delay: 5
|
||||
until: etcd_health.rc == 0
|
||||
when: not ansible_check_mode
|
||||
@@ -0,0 +1,30 @@
|
||||
#!/usr/bin/python3
|
||||
"""仅由受控调用者捕获 stdout；不得手动运行以免在终端输出短期 token。"""
|
||||
import json
|
||||
import ssl
|
||||
import sys
|
||||
import urllib.request
|
||||
|
||||
|
||||
def main():
|
||||
context = ssl.create_default_context()
|
||||
context.load_cert_chain('/etc/homelab-etcd/peer.crt', '/etc/homelab-etcd/peer.key')
|
||||
request = urllib.request.Request(
|
||||
'https://bao.ad.ddupan.top:8200/v1/auth/homelab-etcd-renewal/login',
|
||||
data=json.dumps({'name': 'etcd-laptop'}).encode(),
|
||||
headers={'Content-Type': 'application/json'}, method='POST',
|
||||
)
|
||||
with urllib.request.urlopen(request, context=context, timeout=30) as response:
|
||||
token = json.load(response)['auth']['client_token']
|
||||
if not isinstance(token, str) or not token:
|
||||
raise ValueError('empty token')
|
||||
sys.stdout.write(token)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
try:
|
||||
main()
|
||||
except Exception:
|
||||
# Bao 响应和异常对象可能带敏感内容，不写入 journal。
|
||||
sys.stderr.write('Bao certificate login failed\n')
|
||||
sys.exit(1)
|
||||
@@ -0,0 +1,51 @@
|
||||
#!/usr/bin/python3
|
||||
"""每日一次，机器证书换短期 token；使用现有串行 Ansible 收敛，不常驻 agent。"""
|
||||
import fcntl
|
||||
import os
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import sys
|
||||
import urllib.request
|
||||
|
||||
ROOT = Path('/opt/homelab-etcd-controller')
|
||||
|
||||
|
||||
def main():
|
||||
with open('/var/lib/homelab-etcd-controller/renew.lock', 'a') as lock:
|
||||
try:
|
||||
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
||||
except BlockingIOError:
|
||||
return 0
|
||||
login = subprocess.run(
|
||||
['sudo', '-n', '/usr/bin/python3', str(ROOT / 'login.py')],
|
||||
capture_output=True, text=True, timeout=40,
|
||||
)
|
||||
if login.returncode or not login.stdout.strip():
|
||||
print('Bao 机器证书登录失败；保留现有证书与运行中的 etcd。', file=sys.stderr)
|
||||
return 1
|
||||
token = login.stdout.strip()
|
||||
env = dict(os.environ, BAO_TOKEN=token)
|
||||
try:
|
||||
for play in ['verify.yml', 'site.yml']:
|
||||
result = subprocess.run(
|
||||
['/home/panxiao81/.local/bin/ansible-playbook', play],
|
||||
cwd=ROOT / 'ansible', env=env, timeout=750,
|
||||
)
|
||||
if result.returncode:
|
||||
return result.returncode
|
||||
Path('/var/lib/homelab-etcd-controller/last-success').touch()
|
||||
return 0
|
||||
finally:
|
||||
request = urllib.request.Request(
|
||||
'https://bao.ad.ddupan.top:8200/v1/auth/token/revoke-self',
|
||||
data=b'{}', headers={'X-Vault-Token': token}, method='POST',
|
||||
)
|
||||
try:
|
||||
with urllib.request.urlopen(request, timeout=15):
|
||||
pass
|
||||
except Exception:
|
||||
print('短期 token 撤销未确认，将由 TTL 自动失效。', file=sys.stderr)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,43 @@
|
||||
#!/usr/bin/env python3
|
||||
"""从当前 Bao 登录会话向子进程传递凭据，不写临时秘密文件或输出秘密。"""
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
ROOT = Path(__file__).resolve().parent
|
||||
|
||||
|
||||
def main():
|
||||
if len(sys.argv) < 3 or sys.argv[1] not in ('terraform', 'ansible'):
|
||||
raise SystemExit('用法：python3 run.py terraform <args> | ansible <playbook> [args]')
|
||||
env = dict(os.environ)
|
||||
env.setdefault('BAO_ADDR', 'https://bao.ad.ddupan.top:8200')
|
||||
token = env.get('BAO_TOKEN') or env.get('VAULT_TOKEN')
|
||||
if not token:
|
||||
token = Path('~/.vault-token').expanduser().read_text().strip()
|
||||
env['BAO_TOKEN'] = env['VAULT_TOKEN'] = token
|
||||
if sys.argv[1] == 'terraform':
|
||||
response = subprocess.run(
|
||||
['bao', 'kv', 'get', '-format=json', 'kv/k8s/seaweedfs-s3'],
|
||||
env=env, capture_output=True, text=True,
|
||||
)
|
||||
if response.returncode:
|
||||
raise SystemExit('读取 tfstate 受限身份失败；请检查 Bao 登录和授权。')
|
||||
config = json.loads(response.stdout)['data']['data']['seaweedfs_s3_config']
|
||||
config = json.loads(config) if isinstance(config, str) else config
|
||||
identities = [i for i in config['identities'] if i['name'] == 'terraform']
|
||||
if len(identities) != 1 or len(identities[0]['credentials']) != 1:
|
||||
raise SystemExit('tfstate 身份不唯一，拒绝猜测凭据。')
|
||||
credential = identities[0]['credentials'][0]
|
||||
env['AWS_ACCESS_KEY_ID'] = credential['accessKey']
|
||||
env['AWS_SECRET_ACCESS_KEY'] = credential['secretKey']
|
||||
command, cwd = ['terraform', *sys.argv[2:]], ROOT / 'terraform'
|
||||
else:
|
||||
command, cwd = ['ansible-playbook', *sys.argv[2:]], ROOT / 'ansible'
|
||||
raise SystemExit(subprocess.run(command, cwd=cwd, env=env).returncode)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
+22
@@ -0,0 +1,22 @@
|
||||
# This file is maintained automatically by "terraform init".
|
||||
# Manual edits may be lost in future updates.
|
||||
|
||||
provider "registry.terraform.io/hashicorp/vault" {
|
||||
version = "4.8.0"
|
||||
constraints = "~> 4.0"
|
||||
hashes = [
|
||||
"h1:aHqgWQhDBMeZO9iUKwJYMlh4q+xNMUlMIcjRbF4d02Y=",
|
||||
"zh:269ab13433f67684012ae7e15876532b0312f5d0d2002a9cf9febb1279ce5ea6",
|
||||
"zh:4babc95bf0c40eb85005db1dc2ca403c46be4a71dd3e409db3711a56f7a5ca0e",
|
||||
"zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3",
|
||||
"zh:86e27c1c625ecc24446a11eeffc3ac319b36c2b4e51251db8579256a0dbcf136",
|
||||
"zh:a32f31da94824009e26b077374440b52098aecb93c92ff55dc3d31dd37c4ea25",
|
||||
"zh:be0a18c6c0425518bab4fbffd82078b82036a88503b5d76064de551c9f646cbf",
|
||||
"zh:be5a77fdfd36863ebeec79cd12b1d13322ffad6821d157a0b279789fa06b5937",
|
||||
"zh:be8317d142a3caad74c7d936039ae27076a1b2b8312ef5208e2871a5f525977c",
|
||||
"zh:c94a84895a3d9954b80e983eed4603330a5cdbbd8eef5b3c99278c2d1402ef3c",
|
||||
"zh:de1fb712784dd8415f011ca5346a34f87fab6046c730557615247e511dbc7d98",
|
||||
"zh:e3eafae7da550f86cae395d6660b2a0e93ec8d2b0e0e5ef982ec762e961fc952",
|
||||
"zh:ff35fb1ab6add288f0f368981e56f780b50405accd1937131cba1137999c8d83",
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,15 @@
|
||||
# 与既有服务复用受限 tfstate 身份，使用独立对象与原生锁；不复用 Bao 的 state。
|
||||
terraform {
|
||||
backend "s3" {
|
||||
bucket = "tfstate"
|
||||
key = "etcd/terraform.tfstate"
|
||||
endpoints = { s3 = "https://s3.ad.ddupan.top" }
|
||||
region = "us-east-1"
|
||||
use_path_style = true
|
||||
skip_credentials_validation = true
|
||||
skip_metadata_api_check = true
|
||||
skip_region_validation = true
|
||||
skip_requesting_account_id = true
|
||||
use_lockfile = true
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,93 @@
|
||||
terraform {
|
||||
required_version = ">= 1.10"
|
||||
required_providers {
|
||||
vault = {
|
||||
source = "hashicorp/vault"
|
||||
version = "~> 4.0"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
provider "vault" {
|
||||
address = var.bao_address
|
||||
}
|
||||
|
||||
variable "bao_address" {
|
||||
type = string
|
||||
default = "https://bao.ad.ddupan.top:8200"
|
||||
}
|
||||
variable "pki_mount" {
|
||||
type = string
|
||||
default = "pki"
|
||||
}
|
||||
variable "kv_mount" {
|
||||
type = string
|
||||
default = "kv"
|
||||
}
|
||||
variable "member_names" {
|
||||
type = list(string)
|
||||
default = ["etcd-laptop", "etcd-pve1", "etcd-pve2"]
|
||||
}
|
||||
variable "consumer_names" {
|
||||
type = set(string)
|
||||
default = ["patroni-pg-prod"]
|
||||
}
|
||||
|
||||
# 仅管理既有 PKI 下的新 role/policy，不纳管 CA 私钥、mount 或秘密值。
|
||||
locals {
|
||||
certificate_roles = {
|
||||
server = { names = var.member_names, server = true, client = false, ips = true }
|
||||
peer = { names = concat(var.member_names, ["homelab-etcd-peer"]), server = true, client = true, ips = true }
|
||||
admin = { names = ["root"], server = false, client = true, ips = false }
|
||||
gateway = { names = var.member_names, server = false, client = true, ips = false }
|
||||
client = { names = [for name in var.consumer_names : "etcd-${name}"], server = false, client = true, ips = false }
|
||||
}
|
||||
}
|
||||
resource "vault_pki_secret_backend_role" "etcd" {
|
||||
for_each = local.certificate_roles
|
||||
backend = var.pki_mount
|
||||
name = "homelab-etcd-${each.key}"
|
||||
allowed_domains = each.value.names
|
||||
allow_bare_domains = true
|
||||
allow_subdomains = false
|
||||
allow_glob_domains = false
|
||||
allow_any_name = false
|
||||
allow_localhost = false
|
||||
allow_wildcard_certificates = false
|
||||
allow_ip_sans = each.value.ips
|
||||
server_flag = each.value.server
|
||||
client_flag = each.value.client
|
||||
key_type = "ec"
|
||||
key_bits = 256
|
||||
ttl = 5184000
|
||||
max_ttl = 5184000
|
||||
require_cn = !contains(["gateway", "client"], each.key)
|
||||
use_csr_common_name = true
|
||||
use_csr_sans = true
|
||||
}
|
||||
|
||||
resource "vault_policy" "etcd_provisioner" {
|
||||
name = "homelab-etcd-provisioner"
|
||||
policy = <<-EOT
|
||||
path "${var.pki_mount}/sign/homelab-etcd-*" {
|
||||
capabilities = ["update"]
|
||||
}
|
||||
path "${var.kv_mount}/metadata/infra/etcd/consumers/*" {
|
||||
capabilities = ["read"]
|
||||
}
|
||||
path "${var.kv_mount}/data/infra/etcd/consumers/*" {
|
||||
capabilities = ["create", "read", "update"]
|
||||
}
|
||||
EOT
|
||||
}
|
||||
|
||||
# 身份绑定沿用既有控制端认证方式，创建 policy 不自动授权任何身份。
|
||||
resource "vault_policy" "etcd_consumer" {
|
||||
for_each = var.consumer_names
|
||||
name = "homelab-etcd-${each.key}"
|
||||
policy = <<-EOT
|
||||
path "${var.kv_mount}/data/infra/etcd/consumers/${each.key}" {
|
||||
capabilities = ["read"]
|
||||
}
|
||||
EOT
|
||||
}
|
||||
@@ -0,0 +1,28 @@
|
||||
# 独立挂载：不接管全局认证，不保存长期 token，也不授权 KV 消费者秘密读取。
|
||||
data "vault_generic_secret" "etcd_ca" {
|
||||
path = "${var.pki_mount}/cert/ca"
|
||||
}
|
||||
|
||||
resource "vault_auth_backend" "etcd_renewal" {
|
||||
type = "cert"
|
||||
path = "homelab-etcd-renewal"
|
||||
}
|
||||
|
||||
resource "vault_policy" "etcd_renewal" {
|
||||
name = "homelab-etcd-renewal"
|
||||
policy = join("\n", concat([
|
||||
for kind in ["server", "peer", "admin", "gateway"] :
|
||||
"path \"${var.pki_mount}/sign/homelab-etcd-${kind}\" { capabilities = [\"update\"] }"
|
||||
], ["path \"auth/token/revoke-self\" { capabilities = [\"update\"] }"]))
|
||||
}
|
||||
|
||||
resource "vault_cert_auth_backend_role" "etcd_renewal" {
|
||||
backend = vault_auth_backend.etcd_renewal.path
|
||||
name = "etcd-laptop"
|
||||
certificate = data.vault_generic_secret.etcd_ca.data["certificate"]
|
||||
allowed_dns_sans = ["etcd-laptop"]
|
||||
token_policies = [vault_policy.etcd_renewal.name]
|
||||
token_no_default_policy = true
|
||||
token_ttl = 600
|
||||
token_max_ttl = 900
|
||||
}
|
||||
@@ -0,0 +1,233 @@
|
||||
#!/usr/bin/env python3
|
||||
"""临时三成员 mTLS/RBAC 集成测试；仅绑定 loopback，不访问生产 Bao。"""
|
||||
import base64
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import re
|
||||
import shutil
|
||||
import ssl
|
||||
import subprocess
|
||||
import tempfile
|
||||
import threading
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
||||
|
||||
import jinja2
|
||||
import yaml
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
BIN = Path(os.environ.get('ETCD_TEST_BIN', '/tmp/etcd-v3.7.2-linux-amd64'))
|
||||
|
||||
|
||||
def run(argv, **kwargs):
|
||||
return subprocess.run([str(x) for x in argv], capture_output=True, text=True, check=True, **kwargs)
|
||||
|
||||
|
||||
class FakeBao(BaseHTTPRequestHandler):
|
||||
records = {}
|
||||
versions = {}
|
||||
writes = 0
|
||||
deny = False
|
||||
|
||||
def log_message(self, *_):
|
||||
pass
|
||||
|
||||
def respond(self, status, body):
|
||||
self.send_response(status)
|
||||
self.send_header('Content-Type', 'application/json')
|
||||
self.end_headers()
|
||||
self.wfile.write(json.dumps(body).encode())
|
||||
|
||||
def do_GET(self):
|
||||
if self.deny:
|
||||
return self.respond(403, {'errors': ['permission denied']})
|
||||
name = self.path.rsplit('/', 1)[-1]
|
||||
if '/metadata/' in self.path and name in self.versions:
|
||||
return self.respond(200, {'data': {'current_version': self.versions[name]}})
|
||||
if '/data/' in self.path and name in self.records:
|
||||
return self.respond(200, {'data': {'data': self.records[name]}})
|
||||
self.respond(404, {'errors': []})
|
||||
|
||||
def do_POST(self):
|
||||
body = json.loads(self.rfile.read(int(self.headers['Content-Length'])))
|
||||
name = self.path.rsplit('/', 1)[-1]
|
||||
if body['options']['cas'] != 0 or name in self.versions:
|
||||
return self.respond(400, {'errors': ['CAS mismatch']})
|
||||
self.records[name] = body['data']
|
||||
self.versions[name] = 1
|
||||
type(self).writes += 1
|
||||
self.respond(200, {'data': {'version': 1}})
|
||||
|
||||
|
||||
def main():
|
||||
for key in ('HTTP_PROXY', 'HTTPS_PROXY', 'ALL_PROXY', 'http_proxy', 'https_proxy', 'all_proxy'):
|
||||
os.environ.pop(key, None)
|
||||
os.environ['NO_PROXY'] = '*'
|
||||
processes = []
|
||||
handles = []
|
||||
server = None
|
||||
with tempfile.TemporaryDirectory(prefix='shared-etcd-test-') as work:
|
||||
w = Path(work)
|
||||
w.chmod(0o700)
|
||||
try:
|
||||
run(['openssl', 'req', '-x509', '-newkey', 'rsa:2048', '-nodes', '-keyout', w/'ca.key',
|
||||
'-out', w/'ca.crt', '-days', '1', '-subj', '/CN=isolated-test-ca'])
|
||||
|
||||
def cert(name, cn, eku, ip=None):
|
||||
run(['openssl', 'req', '-new', '-newkey', 'rsa:2048', '-nodes', '-keyout', w/f'{name}.key',
|
||||
'-out', w/f'{name}.csr', '-subj', f'/CN={cn}' if cn else '/'])
|
||||
ext = w/f'{name}.ext'
|
||||
ext.write_text(f'extendedKeyUsage={eku}\n' + (f'subjectAltName=IP:{ip},IP:127.0.0.1\n' if ip else ''))
|
||||
run(['openssl', 'x509', '-req', '-in', w/f'{name}.csr', '-CA', w/'ca.crt',
|
||||
'-CAkey', w/'ca.key', '-CAcreateserial', '-out', w/f'{name}.crt', '-days', '1', '-extfile', ext])
|
||||
|
||||
cert('admin', 'root', 'clientAuth')
|
||||
cert('client', '', 'clientAuth')
|
||||
cert('gateway', '', 'clientAuth')
|
||||
env = jinja2.Environment(loader=jinja2.FileSystemLoader(ROOT/'ansible/roles/shared_etcd/templates'), undefined=jinja2.StrictUndefined)
|
||||
env.filters['to_json'] = json.dumps
|
||||
defaults = yaml.safe_load((ROOT/'ansible/roles/shared_etcd/defaults/main.yml').read_text())
|
||||
names = ['test1', 'test2', 'test3']
|
||||
hosts = {name: {'etcd_address': f'127.0.0.{i+2}'} for i, name in enumerate(names)}
|
||||
for name in names:
|
||||
d = w/name
|
||||
d.mkdir()
|
||||
for kind, cn, eku in [('server', name, 'serverAuth'), ('peer', 'homelab-etcd-peer', 'serverAuth,clientAuth')]:
|
||||
cert(f'{name}-{kind}', cn, eku, hosts[name]['etcd_address'])
|
||||
for suffix in ['crt', 'key']:
|
||||
shutil.copy(w/f'{name}-{kind}.{suffix}', d/f'{kind}.{suffix}')
|
||||
shutil.copy(w/'ca.crt', d/'ca.crt')
|
||||
for suffix in ['crt', 'key']:
|
||||
shutil.copy(w/f'gateway.{suffix}', d/f'gateway.{suffix}')
|
||||
values = dict(defaults, inventory_hostname=name, groups={'etcd': names}, hostvars=hosts,
|
||||
etcd_address=hosts[name]['etcd_address'], etcd_config_dir=str(d), etcd_data_dir=str(d/'data'),
|
||||
etcd_client_port=22379, etcd_peer_port=22380, etcd_metrics_port=0)
|
||||
config = env.get_template('etcd.yml.j2').render(**values)
|
||||
# 三成员共享进程命名空间，metrics 用独立 loopback IP。
|
||||
config = config.replace('http://127.0.0.1:0', f"http://{hosts[name]['etcd_address']}:22381")
|
||||
(d/'config.yml').write_text(config)
|
||||
handle = (d/'etcd.log').open('w')
|
||||
handles.append(handle)
|
||||
processes.append(subprocess.Popen([str(BIN/'etcd'), '--config-file='+str(d/'config.yml')], stdout=handle, stderr=handle))
|
||||
|
||||
base_env = dict(os.environ, ETCDCTL_ENDPOINTS='https://127.0.0.2:22379', ETCDCTL_CACERT=str(w/'ca.crt'),
|
||||
ETCDCTL_CERT=str(w/'admin.crt'), ETCDCTL_KEY=str(w/'admin.key'), ETCDCTL_DIAL_TIMEOUT='2s', ETCDCTL_COMMAND_TIMEOUT='3s')
|
||||
|
||||
def ctl(*args, input=None, env=None):
|
||||
return run([BIN/'etcdctl', *args], input=input, env=env or base_env)
|
||||
|
||||
for attempt in range(5):
|
||||
try:
|
||||
ctl('endpoint', 'health')
|
||||
break
|
||||
except subprocess.CalledProcessError:
|
||||
if any(p.poll() is not None for p in processes):
|
||||
|
||||
for name in names:
|
||||
print((w/name/'etcd.log').read_text()[-4000:])
|
||||
raise RuntimeError('test etcd exited during startup')
|
||||
time.sleep(0.2)
|
||||
else:
|
||||
|
||||
print((w/'test1'/'etcd.log').read_text()[-5000:])
|
||||
raise RuntimeError('test quorum did not form')
|
||||
server = ThreadingHTTPServer(('127.0.0.1', 0), FakeBao)
|
||||
threading.Thread(target=server.serve_forever, daemon=True).start()
|
||||
inventory = w/'hosts.yml'
|
||||
inventory.write_text(yaml.safe_dump({'all': {'children': {'etcd': {'hosts': {'test1': {'ansible_connection': 'local'}}}}}}))
|
||||
extra = {
|
||||
'ansible_become': False, 'etcd_address': '127.0.0.2', 'etcd_client_port': 22379,
|
||||
'etcd_install_dir': str(BIN), 'etcd_config_dir': str(w), 'etcd_bootstrap_auth': True,
|
||||
'etcd_bao_url': f'http://127.0.0.1:{server.server_port}', 'etcd_bao_token': 'isolated-test-token',
|
||||
'etcd_bao_kv_mount': 'kv', 'etcd_bao_secret_base': 'infra/etcd/consumers',
|
||||
'etcd_consumers': [{'name': 'patroni-pg-prod', 'prefix': '/homelab/patroni/pg-prod/'}],
|
||||
}
|
||||
(w/'extra.json').write_text(json.dumps(extra))
|
||||
ansible_env = dict(os.environ, ANSIBLE_LOCAL_TEMP=str(w/'ansible-tmp'), ANSIBLE_NOCOLOR='1',
|
||||
ANSIBLE_ROLES_PATH=str(ROOT/'ansible/roles'))
|
||||
|
||||
def play(name, fail=False):
|
||||
r = subprocess.run(['ansible-playbook', '-i', str(inventory), str(ROOT/'ansible'/name), '-e', '@'+str(w/'extra.json')],
|
||||
capture_output=True, text=True, env=ansible_env)
|
||||
(w/(name+'.log')).write_text(r.stdout+r.stderr)
|
||||
if fail:
|
||||
assert r.returncode != 0, 'expected a fail-closed playbook error'
|
||||
elif r.returncode:
|
||||
# Tasks use no_log for secret data; retain useful task/line diagnostic.
|
||||
print(r.stdout[-5000:]);print(r.stderr[-2000:])
|
||||
raise RuntimeError(name+' failed')
|
||||
return r.stdout
|
||||
|
||||
play('bootstrap-auth.yml')
|
||||
assert re.search(r'changed=0\s', play('bootstrap-auth.yml'))
|
||||
play('consumers.yml')
|
||||
secret = FakeBao.records['patroni-pg-prod']['password']
|
||||
assert len(secret) == 48
|
||||
assert re.search(r'changed=0\s', play('consumers.yml'))
|
||||
assert FakeBao.writes == 1
|
||||
role = json.loads(ctl('--write-out=json', 'role', 'get', 'patroni-pg-prod').stdout)
|
||||
assert role['perm'][0]['key'] == base64.b64encode(b'/homelab/patroni/pg-prod/').decode()
|
||||
client_env = dict(base_env, ETCDCTL_CERT=str(w/'client.crt'), ETCDCTL_KEY=str(w/'client.key'),
|
||||
ETCDCTL_USER='patroni-pg-prod', ETCDCTL_PASSWORD=secret)
|
||||
ctl('put', '/homelab/patroni/pg-prod/test', 'ok', env=client_env)
|
||||
try:
|
||||
ctl('put', '/homelab/other/test', 'denied', env=client_env)
|
||||
raise AssertionError('cross-prefix write succeeded')
|
||||
except subprocess.CalledProcessError:
|
||||
pass
|
||||
# 模拟 Patroni 的 gateway 协议，证明 mTLS + username/password 可组合使用。
|
||||
ctx = ssl.create_default_context(cafile=str(w/'ca.crt'))
|
||||
ctx.load_cert_chain(w/'client.crt', w/'client.key')
|
||||
body = json.dumps({'name':'patroni-pg-prod', 'password':secret}).encode()
|
||||
req = urllib.request.Request('https://127.0.0.2:22379/v3/auth/authenticate', data=body, headers={'Content-Type':'application/json'})
|
||||
with urllib.request.urlopen(req, context=ctx) as response:
|
||||
token = json.load(response)['token']
|
||||
gateway_body = json.dumps({'key': base64.b64encode(b'/homelab/patroni/pg-prod/gateway').decode(),
|
||||
'value': base64.b64encode(b'ok').decode()}).encode()
|
||||
req = urllib.request.Request('https://127.0.0.2:22379/v3/kv/put', data=gateway_body,
|
||||
headers={'Content-Type': 'application/json', 'Authorization': token})
|
||||
with urllib.request.urlopen(req, context=ctx) as response:
|
||||
assert response.status == 200
|
||||
# 对已有用户，秘密值被意外覆盖也必须失败，而不是悄悄改 etcd 密码。
|
||||
FakeBao.records['patroni-pg-prod']['password'] = 'x' * 48
|
||||
play('consumers.yml', fail=True)
|
||||
FakeBao.records['patroni-pg-prod']['password'] = secret
|
||||
FakeBao.deny = True
|
||||
play('consumers.yml', fail=True)
|
||||
FakeBao.deny = False
|
||||
saved = FakeBao.records.pop('patroni-pg-prod')
|
||||
play('consumers.yml', fail=True) # metadata 存在、data 已删除
|
||||
FakeBao.versions.clear()
|
||||
play('consumers.yml', fail=True) # 用户存在但 metadata 丢失
|
||||
assert FakeBao.writes == 1
|
||||
FakeBao.records['patroni-pg-prod'] = saved
|
||||
FakeBao.versions['patroni-pg-prod'] = 1
|
||||
ctl('snapshot', 'save', str(w/'snapshot.db'))
|
||||
snapshot = json.loads(run([BIN/'etcdutl', '--write-out=json', 'snapshot', 'status', w/'snapshot.db']).stdout)
|
||||
assert snapshot['totalKey'] > 0
|
||||
run([BIN/'etcdutl', 'snapshot', 'restore', w/'snapshot.db', '--data-dir='+str(w/'restored')])
|
||||
rss = []
|
||||
for process in processes:
|
||||
match = re.search(r'^VmRSS:\s+(\d+)', Path(f'/proc/{process.pid}/status').read_text(), re.M)
|
||||
rss.append(round(int(match[1]) / 1024, 1))
|
||||
processes[2].terminate();processes[2].wait(timeout=10)
|
||||
ctl('put', '/homelab/patroni/pg-prod/after-member-loss', 'ok', env=client_env)
|
||||
print('PASS: three-member mTLS, auth bootstrap/idempotence, Bao create-once/fail-closed, prefix isolation, gateway auth, snapshot/restore, one-member loss')
|
||||
print('Idle test member RSS MiB (not a production capacity result):', rss)
|
||||
finally:
|
||||
if server:
|
||||
server.shutdown();server.server_close()
|
||||
for process in processes:
|
||||
if process.poll() is None:
|
||||
process.terminate()
|
||||
try: process.wait(timeout=10)
|
||||
except subprocess.TimeoutExpired: process.kill();process.wait()
|
||||
for handle in handles:
|
||||
handle.close()
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,71 @@
|
||||
"""续签调度失败关闭测试；不会连接真实 Bao 或运行 Ansible。"""
|
||||
import importlib.util
|
||||
import subprocess
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
from unittest.mock import patch, Mock
|
||||
|
||||
spec = importlib.util.spec_from_file_location('renew', Path(__file__).parents[1] / 'controller/renew.py')
|
||||
renew = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(renew)
|
||||
|
||||
|
||||
class RenewalTests(unittest.TestCase):
|
||||
def execute(self, results):
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
lock = Path(directory) / 'lock'
|
||||
success = Path(directory) / 'last-success'
|
||||
original_open = open
|
||||
response = Mock()
|
||||
response.__enter__ = Mock(return_value=response)
|
||||
response.__exit__ = Mock(return_value=False)
|
||||
with patch('builtins.open', side_effect=lambda *_: original_open(lock, 'a')), \
|
||||
patch.object(renew.subprocess, 'run', side_effect=results) as run, \
|
||||
patch.object(renew.urllib.request, 'urlopen', return_value=response) as revoke, \
|
||||
patch.object(renew, 'Path', return_value=success):
|
||||
rc = renew.main()
|
||||
return rc, run.call_args_list, revoke.call_count, success.exists()
|
||||
|
||||
def test_login_failure_never_changes_members(self):
|
||||
rc, calls, revokes, success = self.execute([subprocess.CompletedProcess([], 1, '', '')])
|
||||
self.assertEqual(rc, 1)
|
||||
self.assertEqual(len(calls), 1)
|
||||
self.assertEqual(revokes, 0)
|
||||
self.assertFalse(success)
|
||||
|
||||
def test_unhealthy_cluster_never_runs_site(self):
|
||||
rc, calls, revokes, success = self.execute([
|
||||
subprocess.CompletedProcess([], 0, 'test-only-token'),
|
||||
subprocess.CompletedProcess([], 2),
|
||||
])
|
||||
self.assertEqual(rc, 2)
|
||||
self.assertEqual(calls[1].args[0][-1], 'verify.yml')
|
||||
self.assertEqual(len(calls), 2)
|
||||
self.assertEqual(revokes, 1)
|
||||
self.assertFalse(success)
|
||||
|
||||
def test_success_checks_then_converges_and_revokes(self):
|
||||
rc, calls, revokes, success = self.execute([
|
||||
subprocess.CompletedProcess([], 0, 'test-only-token'),
|
||||
subprocess.CompletedProcess([], 0),
|
||||
subprocess.CompletedProcess([], 0),
|
||||
])
|
||||
self.assertEqual(rc, 0)
|
||||
self.assertEqual([c.args[0][-1] for c in calls[1:]], ['verify.yml', 'site.yml'])
|
||||
self.assertEqual(revokes, 1)
|
||||
self.assertTrue(success)
|
||||
|
||||
def test_site_failure_is_not_success(self):
|
||||
rc, calls, revokes, success = self.execute([
|
||||
subprocess.CompletedProcess([], 0, 'test-only-token'),
|
||||
subprocess.CompletedProcess([], 0),
|
||||
subprocess.CompletedProcess([], 2),
|
||||
])
|
||||
self.assertEqual(rc, 2)
|
||||
self.assertEqual(revokes, 1)
|
||||
self.assertFalse(success)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
@@ -0,0 +1,57 @@
|
||||
# OpenBao 监控运维
|
||||
|
||||
受鉴权 `/v1/sys/metrics` 通过 vmagent 内的 Bao Agent 使用 Kubernetes 身份采集。
|
||||
`metrics` policy 仅允许读取该路径；policy 和 role 由 `terraform/monitoring.tf` 管理。
|
||||
|
||||
Bao VM 的 Ubuntu `prometheus-node-exporter` 软件包监听 `192.168.10.8:9100`，
|
||||
由 `ansible/roles/openbao_monitoring` 管理。仅供内网监控，不配置公共入口；
|
||||
现场 UFW 未启用，故当前边界是 LAN 地址绑定，并非逐来源访问控制。
|
||||
标准主机指标复用 `job=node-exporter` 的主机告警，额外标记 `node=bao1`。
|
||||
|
||||
首次部署前需要 bootstrap 已创建 `/etc/openbao/snapshot.token` 和快照目录。
|
||||
只更新监控与快照运行文件（不会重启 Bao 或轮换 token）：
|
||||
|
||||
```bash
|
||||
cd ansible
|
||||
ansible-playbook monitor-openbao.yml --check
|
||||
ansible-playbook monitor-openbao.yml
|
||||
# 在 bao1 上执行一次真实快照，建立成功基线：
|
||||
sudo systemctl start openbao-snapshot.service
|
||||
```
|
||||
|
||||
快照脚本在 `/var/lib/prometheus/node-exporter/` 原子更新两个 `.prom` 文件：
|
||||
|
||||
- `openbao_snapshot_result.prom`：最近一次运行结果（0/1）及完成时间。
|
||||
- `openbao_snapshot_success.prom`：最后一次成功生成完整本地快照的时间。
|
||||
|
||||
失败保留上次成功时间，先完成快照再删除超出保留数量的文件。
|
||||
指标文件 root 写、exporter 只读；不包含 token、路径标签或快照内容。
|
||||
首次成功前成功指标缺失，不用既有文件 mtime 冒充已验证成功。
|
||||
Bao sealed 或 token 失效时，独立 exporter 仍能报告失败及超时。
|
||||
|
||||
`PrometheusRule/openbao` 包含：采集不可用（3 分钟）、内部健康异常（3 分钟）、
|
||||
健康 gauge 缺失（5 分钟）、快照失败（5 分钟）、快照超过 36 小时或无成功记录
|
||||
（持续 15 分钟）、快照监控缺失/损坏（5 分钟）。内部健康规则按当前**单节点**设计，
|
||||
以后增加 standby 必须先调整 active 判定。快照指标仅证明本地任务成功，不能证明异地备份或可恢复性。
|
||||
|
||||
检查与恢复入口：
|
||||
|
||||
```bash
|
||||
sudo systemctl status openbao-snapshot.timer openbao-snapshot.service prometheus-node-exporter
|
||||
sudo journalctl -u openbao-snapshot.service -n 50 --no-pager
|
||||
curl -fsS http://192.168.10.8:9100/metrics | grep '^openbao_snapshot_'
|
||||
```
|
||||
|
||||
token 续期失败按[维护 runbook](https://git.ddupan.top/panxiao81/homelab-wiki/src/branch/main/guides/openbao-monitoring-maintenance.md)
|
||||
受控轮换；不要通过反复重启 Bao 排障。回滚 exporter 或快照脚本无需重启 Bao。
|
||||
回滚时同步撤回相应 ServiceMonitor/规则，否则缺指标告警会继续触发。
|
||||
|
||||
本地验证：
|
||||
|
||||
```bash
|
||||
python3 -m unittest discover -s infrastructure/openbao/ansible/tests -v
|
||||
# 需 PyYAML、Jinja2 和 promtool；已用 promtool 3.5.0 验证。
|
||||
PATH=/path/to/promtool:$PATH bash platform/observability/metrics/tests/check-rules.sh
|
||||
```
|
||||
|
||||
textfile 的原子发布方式遵循 [node_exporter 文档](https://github.com/prometheus/node_exporter#textfile-collector)。
|
||||
@@ -0,0 +1,13 @@
|
||||
---
|
||||
# 只部署 exporter、快照脚本和 timer，不改 Bao 配置、不重启 Bao、不创建 token。
|
||||
# 前提：bootstrap 已安装有效的 /etc/openbao/snapshot.token。
|
||||
- name: 补齐 OpenBao 主机和快照监控
|
||||
hosts: openbao
|
||||
become: true
|
||||
roles:
|
||||
- openbao_monitoring
|
||||
tasks:
|
||||
- name: 更新快照运行文件
|
||||
ansible.builtin.include_role:
|
||||
name: openbao_bootstrap
|
||||
tasks_from: snapshot-runtime
|
||||
@@ -47,3 +47,6 @@ openbao_transit_mount_path: "transit/"
|
||||
openbao_manage_firewall: false
|
||||
openbao_allowed_cidrs:
|
||||
- "100.64.0.0/10"
|
||||
|
||||
# 30 秒采集间隔下保留五分钟，指标端点继续要求鉴权。
|
||||
openbao_prometheus_retention_time: "5m"
|
||||
|
||||
@@ -25,3 +25,9 @@ seal "transit" {
|
||||
mount_path = "{{ openbao_transit_mount_path }}"
|
||||
}
|
||||
{% endif %}
|
||||
|
||||
# 显式固定 Prometheus 行为；应用此配置会经 handler 重启，须人工解封。
|
||||
telemetry {
|
||||
prometheus_retention_time = "{{ openbao_prometheus_retention_time }}"
|
||||
disable_hostname = true
|
||||
}
|
||||
|
||||
@@ -96,3 +96,8 @@ openbao_snapshot_oncalendar: "*-*-* 02:00:00"
|
||||
# 3. terraform apply # mounts, roles, policies
|
||||
# 4. ansible-playbook bootstrap-openbao.yml # CA material, OIDC secret, snapshots
|
||||
openbao_config_managed_by_terraform: true
|
||||
|
||||
# 仅在维护窗口显式启用，用于替换已失效的快照 token；不读取/打印 token。
|
||||
openbao_snapshot_rotate_token: false
|
||||
|
||||
openbao_snapshot_metrics_dir: /var/lib/prometheus/node-exporter
|
||||
|
||||
@@ -0,0 +1,34 @@
|
||||
---
|
||||
- name: Ensure the snapshot output directory exists
|
||||
ansible.builtin.file:
|
||||
path: "{{ openbao_snapshot_dir }}"
|
||||
state: directory
|
||||
owner: root
|
||||
group: root
|
||||
mode: "0700"
|
||||
|
||||
- name: Install the snapshot script
|
||||
ansible.builtin.template:
|
||||
src: bao-snapshot.sh.j2
|
||||
dest: /usr/local/bin/bao-snapshot.sh
|
||||
owner: root
|
||||
group: root
|
||||
mode: "0755"
|
||||
|
||||
- name: Install the snapshot systemd service + timer
|
||||
ansible.builtin.template:
|
||||
src: "{{ item }}.j2"
|
||||
dest: "/etc/systemd/system/{{ item }}"
|
||||
owner: root
|
||||
group: root
|
||||
mode: "0644"
|
||||
loop:
|
||||
- openbao-snapshot.service
|
||||
- openbao-snapshot.timer
|
||||
|
||||
- name: Enable and start the snapshot timer
|
||||
ansible.builtin.systemd:
|
||||
name: openbao-snapshot.timer
|
||||
state: started
|
||||
enabled: true
|
||||
daemon_reload: true
|
||||
@@ -17,11 +17,11 @@
|
||||
path: /etc/openbao/snapshot.token
|
||||
register: snap_tok_stat
|
||||
|
||||
- name: Create a periodic snapshot token (once)
|
||||
- name: Create or explicitly rotate the periodic snapshot token
|
||||
ansible.builtin.command: "bao token create -policy=snapshot -period=768h -orphan -field=token"
|
||||
environment: "{{ openbao_cli_env }}"
|
||||
register: snap_tok_new
|
||||
when: not snap_tok_stat.stat.exists
|
||||
when: (not snap_tok_stat.stat.exists) or (openbao_snapshot_rotate_token | bool)
|
||||
changed_when: snap_tok_new.rc == 0
|
||||
no_log: true
|
||||
|
||||
@@ -32,39 +32,12 @@
|
||||
owner: root
|
||||
group: root
|
||||
mode: "0600"
|
||||
when: not snap_tok_stat.stat.exists
|
||||
when: (not snap_tok_stat.stat.exists) or (openbao_snapshot_rotate_token | bool)
|
||||
no_log: true
|
||||
|
||||
- name: Ensure the snapshot output directory exists
|
||||
ansible.builtin.file:
|
||||
path: "{{ openbao_snapshot_dir }}"
|
||||
state: directory
|
||||
owner: root
|
||||
group: root
|
||||
mode: "0700"
|
||||
- name: 安装独立的主机与快照指标 exporter
|
||||
ansible.builtin.include_role:
|
||||
name: openbao_monitoring
|
||||
|
||||
- name: Install the snapshot script
|
||||
ansible.builtin.template:
|
||||
src: bao-snapshot.sh.j2
|
||||
dest: /usr/local/bin/bao-snapshot.sh
|
||||
owner: root
|
||||
group: root
|
||||
mode: "0755"
|
||||
|
||||
- name: Install the snapshot systemd service + timer
|
||||
ansible.builtin.template:
|
||||
src: "{{ item }}.j2"
|
||||
dest: "/etc/systemd/system/{{ item }}"
|
||||
owner: root
|
||||
group: root
|
||||
mode: "0644"
|
||||
loop:
|
||||
- openbao-snapshot.service
|
||||
- openbao-snapshot.timer
|
||||
|
||||
- name: Enable and start the snapshot timer
|
||||
ansible.builtin.systemd:
|
||||
name: openbao-snapshot.timer
|
||||
state: started
|
||||
enabled: true
|
||||
daemon_reload: true
|
||||
- name: 部署快照脚本和定时器
|
||||
ansible.builtin.import_tasks: snapshot-runtime.yml
|
||||
|
||||
+44
-6
@@ -1,20 +1,58 @@
|
||||
#!/usr/bin/env bash
|
||||
# {{ ansible_managed }}
|
||||
# Take a Raft snapshot and prune old ones. Ship {{ openbao_snapshot_dir }} off-box
|
||||
# separately (rsync/restic/scp) — a snapshot on the same host is not a backup.
|
||||
# 本机快照不等于异地备份。结果通过独立 node_exporter 上报。
|
||||
set -euo pipefail
|
||||
umask 077
|
||||
|
||||
dir="{{ openbao_snapshot_dir }}"
|
||||
metrics_dir="{{ openbao_snapshot_metrics_dir }}"
|
||||
# systemd oneshot 不会并发；flock 同时阻止手动执行与 timer 竞争。
|
||||
exec 9>"${dir}/.snapshot.lock"
|
||||
flock -n 9 || exit 0
|
||||
partial=""
|
||||
metrics_tmp=""
|
||||
finish() {
|
||||
rc=$?
|
||||
trap - EXIT
|
||||
[ -z "$partial" ] || rm -f -- "$partial"
|
||||
# success 文件只在实际快照完成后更新；失败不能抹掉上次成功时间。
|
||||
metrics_tmp="$(mktemp "${metrics_dir}/.openbao-result.XXXXXX")" || exit 1
|
||||
{
|
||||
echo '# HELP openbao_snapshot_last_run_success Whether the last snapshot run succeeded.'
|
||||
echo '# TYPE openbao_snapshot_last_run_success gauge'
|
||||
if [ "$rc" -eq 0 ]; then echo 'openbao_snapshot_last_run_success 1'; else echo 'openbao_snapshot_last_run_success 0'; fi
|
||||
echo '# HELP openbao_snapshot_last_run_timestamp_seconds Completion time of the last snapshot attempt.'
|
||||
echo '# TYPE openbao_snapshot_last_run_timestamp_seconds gauge'
|
||||
echo "openbao_snapshot_last_run_timestamp_seconds $(date +%s)"
|
||||
} > "$metrics_tmp"
|
||||
chmod 644 "$metrics_tmp"
|
||||
mv -f -- "$metrics_tmp" "${metrics_dir}/openbao_snapshot_result.prom"
|
||||
exit "$rc"
|
||||
}
|
||||
trap finish EXIT
|
||||
|
||||
export BAO_ADDR="{{ openbao_addr }}"
|
||||
export BAO_CACERT="{{ openbao_tls_dir }}/cert.pem"
|
||||
BAO_TOKEN="$(cat /etc/openbao/snapshot.token)"
|
||||
export BAO_TOKEN
|
||||
bao token renew >/dev/null
|
||||
|
||||
dir="{{ openbao_snapshot_dir }}"
|
||||
stamp="$(date +%Y%m%d-%H%M%S)"
|
||||
out="${dir}/openbao-${stamp}.snap"
|
||||
partial="${out}.partial"
|
||||
bao operator raft snapshot save "$partial"
|
||||
chmod 600 "$partial"
|
||||
mv -- "$partial" "$out"
|
||||
partial=""
|
||||
|
||||
bao operator raft snapshot save "${out}"
|
||||
chmod 600 "${out}"
|
||||
metrics_tmp="$(mktemp "${metrics_dir}/.openbao-success.XXXXXX")"
|
||||
{
|
||||
echo '# HELP openbao_snapshot_last_success_timestamp_seconds Completion time of the last successful local Raft snapshot.'
|
||||
echo '# TYPE openbao_snapshot_last_success_timestamp_seconds gauge'
|
||||
echo "openbao_snapshot_last_success_timestamp_seconds $(date +%s)"
|
||||
} > "$metrics_tmp"
|
||||
chmod 644 "$metrics_tmp"
|
||||
mv -f -- "$metrics_tmp" "${metrics_dir}/openbao_snapshot_success.prom"
|
||||
|
||||
# Retention: keep the newest {{ openbao_snapshot_keep }}.
|
||||
# 只有产生完整快照后才做保留清理。
|
||||
ls -1t "${dir}"/openbao-*.snap 2>/dev/null | tail -n +{{ openbao_snapshot_keep + 1 }} | xargs -r rm -f
|
||||
|
||||
@@ -0,0 +1,3 @@
|
||||
---
|
||||
openbao_metrics_listen_address: "{{ ansible_host }}:9100"
|
||||
openbao_snapshot_metrics_dir: /var/lib/prometheus/node-exporter
|
||||
@@ -0,0 +1,6 @@
|
||||
---
|
||||
- name: restart openbao node exporter
|
||||
ansible.builtin.systemd:
|
||||
name: prometheus-node-exporter
|
||||
state: restarted
|
||||
when: not ansible_check_mode
|
||||
@@ -0,0 +1,35 @@
|
||||
---
|
||||
- name: 安装 Ubuntu node_exporter 软件包
|
||||
ansible.builtin.apt:
|
||||
name: prometheus-node-exporter
|
||||
state: present
|
||||
update_cache: true
|
||||
cache_valid_time: 3600
|
||||
install_recommends: false
|
||||
policy_rc_d: 101
|
||||
|
||||
- name: 创建只允许 root 写入的 textfile 目录
|
||||
ansible.builtin.file:
|
||||
path: "{{ openbao_snapshot_metrics_dir }}"
|
||||
state: directory
|
||||
owner: root
|
||||
group: root
|
||||
mode: "0755"
|
||||
|
||||
- name: 将 exporter 绑定到内网地址并启用 textfile
|
||||
ansible.builtin.copy:
|
||||
dest: /etc/default/prometheus-node-exporter
|
||||
owner: root
|
||||
group: root
|
||||
mode: "0644"
|
||||
content: |
|
||||
# Ansible managed; no Bao credentials needed.
|
||||
ARGS="--web.listen-address={{ openbao_metrics_listen_address }} --collector.textfile.directory={{ openbao_snapshot_metrics_dir }}"
|
||||
notify: restart openbao node exporter
|
||||
|
||||
- name: 启用主机指标服务
|
||||
ansible.builtin.systemd:
|
||||
name: prometheus-node-exporter
|
||||
enabled: true
|
||||
state: started
|
||||
when: not ansible_check_mode
|
||||
@@ -0,0 +1,81 @@
|
||||
"""用假 bao 验证失败不会推进成功时间或删除已有快照。"""
|
||||
import os
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import tempfile
|
||||
import unittest
|
||||
from jinja2 import Template
|
||||
|
||||
TEMPLATE = Path(__file__).resolve().parents[1] / 'roles/openbao_bootstrap/templates/bao-snapshot.sh.j2'
|
||||
|
||||
|
||||
class SnapshotTest(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.tmp = tempfile.TemporaryDirectory()
|
||||
self.addCleanup(self.tmp.cleanup)
|
||||
self.root = Path(self.tmp.name)
|
||||
self.snap = self.root / 'snapshots'
|
||||
self.metrics = self.root / 'metrics'
|
||||
self.snap.mkdir()
|
||||
self.metrics.mkdir()
|
||||
self.token = self.root / 'token'
|
||||
self.token.write_text('test-only')
|
||||
self.script = self.root / 'snapshot.sh'
|
||||
self.script.write_text(Template(TEMPLATE.read_text()).render(
|
||||
ansible_managed='test', openbao_snapshot_dir=str(self.snap),
|
||||
openbao_snapshot_metrics_dir=str(self.metrics), openbao_addr='https://invalid',
|
||||
openbao_tls_dir='/unused', openbao_snapshot_keep=2,
|
||||
).replace('/etc/openbao/snapshot.token', str(self.token)))
|
||||
bao = self.root / 'bao'
|
||||
bao.write_text('''#!/usr/bin/env bash
|
||||
if [ "$1" = token ]; then
|
||||
[ "${FAIL_AT:-}" != renew ]; exit $?
|
||||
fi
|
||||
printf snapshot > "$5"
|
||||
[ "${FAIL_AT:-}" != save ]
|
||||
''')
|
||||
bao.chmod(0o755)
|
||||
for n in range(3):
|
||||
p = self.snap / f'openbao-old{n}.snap'
|
||||
p.write_text('old snapshot')
|
||||
os.utime(p, (100+n, 100+n))
|
||||
self.success = self.metrics / 'openbao_snapshot_success.prom'
|
||||
self.success.write_text('openbao_snapshot_last_success_timestamp_seconds 123\n')
|
||||
|
||||
def run_snapshot(self, failure=''):
|
||||
return subprocess.run(['bash', str(self.script)], env=dict(os.environ,
|
||||
PATH=str(self.root)+':'+os.environ['PATH'], FAIL_AT=failure), capture_output=True)
|
||||
|
||||
def test_renew_failure_preserves_success_and_snapshots(self):
|
||||
self.assertNotEqual(self.run_snapshot('renew').returncode, 0)
|
||||
self.check_failure()
|
||||
|
||||
def test_partial_snapshot_is_removed(self):
|
||||
self.assertNotEqual(self.run_snapshot('save').returncode, 0)
|
||||
self.check_failure()
|
||||
self.assertEqual(list(self.snap.glob('*.partial')), [])
|
||||
|
||||
def test_missing_token_is_reported(self):
|
||||
self.token.unlink()
|
||||
self.assertNotEqual(self.run_snapshot().returncode, 0)
|
||||
self.check_failure()
|
||||
|
||||
def check_failure(self):
|
||||
self.assertEqual(self.success.read_text(), 'openbao_snapshot_last_success_timestamp_seconds 123\n')
|
||||
self.assertIn('openbao_snapshot_last_run_success 0', (self.metrics/'openbao_snapshot_result.prom').read_text())
|
||||
self.assertEqual(len(list(self.snap.glob('*.snap'))), 3)
|
||||
|
||||
def test_success_retention_and_permissions(self):
|
||||
p = self.run_snapshot()
|
||||
self.assertEqual(p.returncode, 0, p.stderr)
|
||||
self.assertNotIn('seconds 123\n', self.success.read_text())
|
||||
self.assertIn('openbao_snapshot_last_run_success 1', (self.metrics/'openbao_snapshot_result.prom').read_text())
|
||||
snapshots = list(self.snap.glob('*.snap'))
|
||||
self.assertEqual(len(snapshots), 2)
|
||||
new = next(p for p in snapshots if 'old' not in p.name)
|
||||
self.assertEqual(new.stat().st_mode & 0o777, 0o600)
|
||||
self.assertEqual(self.success.stat().st_mode & 0o777, 0o644)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
@@ -68,3 +68,14 @@ import {
|
||||
to = vault_pki_secret_backend_config_urls.this
|
||||
id = "pki/config/urls"
|
||||
}
|
||||
|
||||
# 本次维护预检先通过同名 API 配置验证，后续 Terraform 纳入既有主 state。
|
||||
import {
|
||||
to = vault_policy.metrics
|
||||
id = "metrics"
|
||||
}
|
||||
|
||||
import {
|
||||
to = vault_kubernetes_auth_backend_role.metrics
|
||||
id = "auth/kubernetes/role/metrics"
|
||||
}
|
||||
|
||||
@@ -0,0 +1,15 @@
|
||||
# 指标只读身份；vmagent 的 Bao Agent sidecar 使用 Pod SA 自动登录与续期。
|
||||
resource "vault_policy" "metrics" {
|
||||
name = "metrics"
|
||||
policy = file("${path.module}/policies/metrics.hcl")
|
||||
}
|
||||
|
||||
resource "vault_kubernetes_auth_backend_role" "metrics" {
|
||||
backend = "kubernetes"
|
||||
role_name = "metrics"
|
||||
bound_service_account_names = ["vmagent-main"]
|
||||
bound_service_account_namespaces = ["monitoring"]
|
||||
token_policies = [vault_policy.metrics.name]
|
||||
token_ttl = 900
|
||||
token_max_ttl = 3600
|
||||
}
|
||||
@@ -0,0 +1,4 @@
|
||||
# 不含秘密读取或管理权限；默认 policy 只用于自身 token 续期等通用能力。
|
||||
path "sys/metrics" {
|
||||
capabilities = ["read"]
|
||||
}
|
||||
@@ -0,0 +1,8 @@
|
||||
.terraform/
|
||||
*.tfstate*
|
||||
*.tfplan
|
||||
*.tfvars
|
||||
__pycache__/
|
||||
|
||||
# 此文件是从 Bao 读取凭据的 playbook，不包含秘密值。
|
||||
!ansible/credentials.yml
|
||||
@@ -0,0 +1,116 @@
|
||||
# k3s 外共享 PostgreSQL
|
||||
|
||||
面向 homelab 的 PG 18 + Patroni + pgBackRest，复用现有 Bao、shared etcd、VyOS 和监控栈。
|
||||
开发与生产共用 laptop，但使用独立 OS 用户、数据集、端口与配置。没有部署 Pigsty 全栈。
|
||||
|
||||
**2026-09-25 已上线新生产主从、独立开发实例、稳定入口与本地备份。**
|
||||
现有 CNPG 和应用数据未迁移。Ayatori 按维护者决定沿用独立控制面规划，本轮只备齐接入材料，
|
||||
未向现有 k3s 部署 Ayatori 或 Instance CR，也不能宣称 Instance Ready。
|
||||
|
||||
## 入口与拓扑
|
||||
|
||||
| 承载 | 角色 | 数据目录/入口 | 初始预算 |
|
||||
| --- | --- | --- | --- |
|
||||
| laptop | prod 正常主库 / Patroni | `/var/lib/homelab-postgresql/prod`，192.168.10.127:5432 | ZFS quota 32 GiB；PG MemoryMax 1 GiB |
|
||||
| pve1 LXC150 | prod standby，与 etcd 共置 | 同 prod 目录，10.60.0.20:5432 | HDD mp0 16 GiB；容器 1536 MiB |
|
||||
| laptop | dev 独立实例 | `/var/lib/homelab-postgresql/dev`，pg-dev.ad.ddupan.top:5433 | ZFS quota 8 GiB；MemoryMax 384 MiB |
|
||||
| pve2 LXC151 | pgBackRest 仓库，与 etcd 共置 | `/var/lib/homelab-pgbackrest` | HDD mp0 32 GiB；容器 768 MiB |
|
||||
| VyOS | 生产稳定 TCP 入口 | pg-prod.ad.ddupan.top:5432 → 192.168.10.2:5432 | 独立 HAProxy service，MemoryMax 64 MiB |
|
||||
|
||||
两个 LXC 的 rootfs 保持 `pve-rg` SSD DRBD，数据盘为 `pve-rg-hdd`。
|
||||
laptop ZFS 使用 recordsize=16K、lz4、atime=off。预算不是容量或 SLA 承诺，standby 的 16 GiB
|
||||
小于主库 quota，扩容应以较小容量为约束。PG prod/dev 分别为 `pgprod`/`pgdev`，配置和 socket 隔离。
|
||||
初始 prod shared_buffers=128 MiB / max_connections=80；dev 为 32 MiB / 30。
|
||||
现场版本为 PostgreSQL 18.6、Patroni 4.1.5、pgBackRest 2.59.1。
|
||||
|
||||
客户端必须使用自己的业务凭据及 `sslmode=verify-full`。本轮仅建立 Ayatori 非 superuser
|
||||
管理账号（CREATEDB/CREATEROLE），未交付普通应用账号。公开中央 CA 位于 `ayatori/ca.crt`。
|
||||
例如，在已获管理身份的受控终端用交互密码提示验证，不在命令行填写密码：
|
||||
|
||||
```bash
|
||||
psql 'host=pg-prod.ad.ddupan.top hostaddr=192.168.10.2 port=5432 dbname=postgres user=ayatori sslmode=verify-full sslrootcert=ayatori/ca.crt' -W
|
||||
```
|
||||
|
||||
生产采用异步复制，自动切换可能损失未复制的已提交事务；最大候选落后阈值 1 MiB 不是精确 RPO。
|
||||
Patroni REST 绑定内网 8008，写操作有独立认证。HAProxy 校验 `/primary`，摘除旧主时关闭旧会话；
|
||||
角色变换与代理收敛期间可能出现短暂连接错误或只读响应，应用应使用适当重试。
|
||||
没有硬件 watchdog/fencing，尚未验收进程冻结、网络分区或整站故障，不能把服务停止演练当作覆盖所有故障。
|
||||
|
||||
## IaC 与秘密
|
||||
|
||||
本目录 Terraform 独立 state 为 `shared-postgresql/terraform.tfstate`，使用现有 S3 `terraform`
|
||||
身份；`run.py` 在内存读取 `kv/k8s/seaweedfs-s3` 的 AK/SK。首次 PKI/JWT role/policy 管理由有效
|
||||
Bao 管理会话 plan/apply，业务部署改用受限 SPIFFE role `homelab-postgresql`。
|
||||
|
||||
```bash
|
||||
python3 run.py terraform plan
|
||||
PG_BAO_SPIFFE_ROLE=homelab-postgresql python3 run.py --spiffe ansible credentials.yml
|
||||
```
|
||||
|
||||
`--spiffe` 从本机 Workload API 取得 JWT-SVID，交换短期 Bao token，执行后撤销；失败不回退到
|
||||
旧 CLI token，不写 `~/.vault-token`。专用 role 绑定 `spiffe://ddupan.top/dev/panxiao81`，只允许
|
||||
所需签发路径、实例 KV 与 Ayatori 交付路径，既有 etcd 消费者只读；没有 PKI/auth 管理或 KV 删除权限。
|
||||
Terraform provider 直接使用包装器的 token（skip_child_token），不另外创建子 token。
|
||||
|
||||
原始实例秘密位于 `kv/infra/postgresql/{prod,dev}`；`initialize-secrets.yml` 仅在元数据明确不存在、
|
||||
数据目录尚未初始化时 CAS=0 创建。403、软删除、密码缺失或漂移均不能触发隐式重置。
|
||||
Ayatori 单独使用 `kv/infra/postgresql/ayatori/{prod,dev}`，仅有 username/password，由
|
||||
`ayatori-credentials.yml` CAS 创建及回读核对。未来 ESO 只读这两个路径，不读取完整实例秘密。
|
||||
涉及秘密的任务 no_log，禁止对凭据 play 使用 diff。旧 Ansible Vault 未迁移。
|
||||
|
||||
## 部署与维护入口
|
||||
|
||||
1. `storage.yml` / `pve-storage.yml` 准备专属卷和预算；`preflight.yml` 拒绝未知数据或错误挂载。
|
||||
2. `packages.yml` 预装软件，阻止 apt 自动初始化默认 main；只刷新 Ubuntu 与本项目 PGDG 源，
|
||||
仍校验签名，不接管 laptop 无关第三方仓库故障。
|
||||
3. Terraform 准备 PKI 与身份，`initialize-secrets.yml -e pg_allow_initialize=true` 首次创建密码。
|
||||
4. `site.yml` 写入实例配置和证书；不初始化、不自动重启。`backup.yml` 建立专属仓库与 SSH 传输。
|
||||
5. `bootstrap.yml -e pg_allow_initialize=true` 显式初始化空目录，先 laptop 后 standby；未知数据不接管。
|
||||
6. `bootstrap-backup.yml` 验证 WAL 与首备成功后启用每日 timer；`proxy.yml` 要求唯一 primary 才发布入口。
|
||||
7. DNS 由 `infrastructure/dns/records.yml` 与 Samba `provision-dc.yml --tags dns` 管理。
|
||||
8. `controller.yml` 安装 PG 证书每日续签调度，预检通过才启用。
|
||||
9. Ayatori 独立控制面就绪后按 [接入说明](ayatori/README.md) 注册；CNPG 迁移另行按应用验收。
|
||||
|
||||
普通 `site.yml` 不代表所有配置已在线激活；尚未实现通用参数变更的 rolling restart。
|
||||
专用 `renew-certificates.yml` 只处理证书，60 天有效期、剩余不足 14 天续签；逐主机检查服务，
|
||||
需要时签发并向 Patroni/PostgreSQL 主进程发送 HUP，最后检查 SQL。`homelab-postgresql-renew.timer`
|
||||
每日 04:30 UTC 加随机延迟；日志由同名 service journal 查询。程序与 Ansible 副本位于
|
||||
`/opt/homelab-postgresql-controller`，由 root 管理，以本机开发身份取短期 token。
|
||||
续签依赖现有 SPIRE（在 k3s）和 Bao；数据库与 etcd 的正常运行仅用本地证书，不实时依赖 k3s。
|
||||
|
||||
## 备份与恢复
|
||||
|
||||
仓库主机发起备份，发现实际 primary；每天 03:40 UTC 加随机延迟执行 full，保留 3 个 full，
|
||||
本地连续归档 WAL。WAL archive_timeout=300s，未设置超限丢弃 WAL 的选项。
|
||||
生产两节点各持有独立 SSH key，仓库和数据库账号相互授权，固定 host key，关闭 SSH 自动追加 host keys（UpdateHostKeys=no），不允许 PTY/转发；
|
||||
专用账号仍能运行所需远端命令。密钥在目标文件系统原地生成，避免 ext4 属性复制到 ZFS 失败。
|
||||
|
||||
查看仓库 `homelab-postgresql-backup.timer/service` 与
|
||||
`pgbackrest --config=/etc/homelab-pgbackrest/repository.conf --stanza=prod info/check`。
|
||||
首次实际 full 的 DB 22.7 MB、仓库备份集约 2.7 MB，仅代表当前空实例，不是未来业务容量估算。
|
||||
独立的 dev 实例当前不备份。异地备份按维护者决定后置；本地 PVE 仓库不是异地或离线副本。
|
||||
|
||||
`tests/live_restore.py --run` 从真实仓库恢复到新临时目录，用私有 Unix socket 验证恢复实例可写与
|
||||
Ayatori 角色，随后停库清理；不覆盖生产 PGDATA。全站恢复及接管旧 CNPG 不由此脚本自动执行。
|
||||
|
||||
## 监控与验收
|
||||
|
||||
现有 VictoriaMetrics/Alertmanager 已采集生产两节点 Patroni 原生指标，规则位于
|
||||
`platform/observability/metrics/{scrapes,rules}/shared-postgresql.yaml`。覆盖成员不可采集、无可见主库、
|
||||
多主、replica 未流复制及 PG 进程停止。没有部署另一套监控栈；数据库 SQL 级 exporter、dev 运行告警、
|
||||
备份年龄与证书到期告警尚待补齐，不能宣称监控完整。
|
||||
|
||||
2026-09-25 已验证：
|
||||
|
||||
- 真实 prod 主从、dev 初始化；Ayatori 管理账号 verify-full TLS/权限校验；生产稳定入口 SQL 写入。
|
||||
- 配置第二次执行两主机 `changed=0`；备份配置三主机与代理重跑均 `changed=0`；实例秘密及 Ayatori 独立凭据重复执行不变更。
|
||||
- 真实 SSH WAL 归档、首个 full、每日 timer；真实备份恢复到隔离实例并验证可写。
|
||||
- `tests/live_failover.py --run` 停止主库服务，自动提升/入口写入、旧主重加入、计划回切后复制、探针清理全部通过；
|
||||
完整通过的一次恢复写入约 9.2 秒（首次观察 15.1 秒），是低负载演练结果，不是 SLA。
|
||||
- 隔离 Docker 测试使用实际模板，另覆盖时间点恢复、DCS 多数派丢失后拒绝写入及管理权限合同。
|
||||
- Terraform 最终 plan 无变更、Ansible lint、短期会话撤销测试与 Ayatori Kustomize 渲染通过。
|
||||
- PG 续签 systemd 首次运行 Result=success，生产两主机证书检查与 SQL 验收均 `changed=0`；尚未强制演练证书临期轮换。
|
||||
|
||||
新 HDD 卷同步限制为各 10 MiB/s，入口 `limit-resync.yml`；同步已完成，副本 UpToDate。
|
||||
此前 CT150 根卷 DRBD quorum 抖动导致 emergency_ro，已离线修复；详见 [etcd runbook](../etcd/README.md)。
|
||||
限速后未再观察到同类故障，长期稳定性与唯一根因尚未确认。
|
||||
@@ -0,0 +1,150 @@
|
||||
# Shared PostgreSQL：Pigsty 裁剪研究
|
||||
|
||||
日期：2026-09-25。状态：源码研究与方案建议，尚未部署、压测或决定迁移。维护者已确定 etcd 按全 homelab 共享基础设施设计，其余实现细节仍为建议。
|
||||
|
||||
研究基线：Pigsty `v4.5.0`，commit `dab5dba333a070d96fde1f9feb41761148f2be8c`，浅克隆位于 `/tmp/pigsty-source-review`。下文相对源码路径均指此版本。保留上游 Apache-2.0 许可及适用的版权声明。
|
||||
|
||||
## 结论
|
||||
|
||||
可以裁剪，优先保留 PostgreSQL + Patroni + etcd + pgBackRest，复用 pg_exporter 和经过筛选的规则。无需带入仓库服务器、PgBouncer、Supabase、Grafana/VictoriaMetrics 全栈、门户和 MinIO。
|
||||
|
||||
但这不是一份关闭功能开关的 inventory 就能完成的工作。主要改造点是：实例与宿主机解耦、移除对宿主机的全局接管、重做初始化权限、补齐可验证的备份和切换流程。建议维护独立的精简 Ansible 实现，明确记录取自上游的文件和基线；不保留无用组件及其配置兼容负担。
|
||||
|
||||
现阶段没有实测 RSS、HDD fsync 延迟和故障恢复时间，不能宣称任何 HA 方案已经满足资源预算。
|
||||
|
||||
## 已知需求与现状
|
||||
|
||||
- 数据库独立于 k3s；生产自动故障转移；开发不要求 HA。
|
||||
- laptop ZFS SSD 为正常生产 primary 所在地；PVE HDD LXC 承担 standby。
|
||||
- 开发与生产共用物理机器，但使用独立 PGDATA、端口、操作系统账号和资源约束。
|
||||
- IaC 管宿主资源、实例、HA、备份、监控；Ayatori 管业务数据库、账号和凭据；Backstage 管门户。
|
||||
- 异地备份暂缓，首期仍包含本地备份与恢复验证。
|
||||
- 本会话先前只读检查：现有 CNPG 为 PostgreSQL 18.3，数据库合计约 185 MiB，PGDATA 约 748 MiB，其中 WAL 约 561 MiB；活跃 10 GiB PVC 的文件系统使用量约 795 MiB。这不是备份压缩后的体积，也不是新的持续监控结果。
|
||||
- Gitea 数据库约 43 MiB、NetBox 约 29 MiB；NetBox 可能退役。Git 仓库、LFS、附件等不包含在数据库体积里。
|
||||
|
||||
## 源码发现与处理
|
||||
|
||||
| 范围 | 源码依据 | 结论与处理 |
|
||||
| --- | --- | --- |
|
||||
| 最小入口 | `slim.yml` | 已跳过 INFRA、监控栈和仓库服务，但仍执行 NODE、HAProxy、ETCD、PGSQL；不能直接当共享宿主部署入口。 |
|
||||
| 生命周期 | `pgsql.yml`、`roles/pgsql/tasks/main.yml` | 初始化与日常收敛不是同一件事。拆成 provision/configure/upgrade/recover，禁止普通配置更新重跑 bootstrap。 |
|
||||
| 同机多实例 | `roles/pgsql/tasks/install.yml`、`config.yml`；全仓检索 `pg_instances` | `pg_instances` 仅见注释/元数据，没有实例循环实现。硬编码 `/pg`、Patroni/PG service、pgBackRest/exporter 配置与全局环境文件；改端口或 inventory alias 不够。 |
|
||||
| 宿主机接管 | `roles/pgsql/tasks/install.yml:58` 起、`roles/node/tasks/main.yml` | Debian 清理逻辑会停止默认 PG、删除发行版 systemd unit，包含立即停库与 kill 回退；NODE 管 DNS、软件源、调优等。删除这些逻辑，仅管理自己声明的资源。 |
|
||||
| 包依赖 | `roles/node_id/vars/d12.aarch64.yml` 等发行版映射 | `pgsql-common` 仍包含 PgBouncer、vip-manager、backup exporter；`pgsql-main` 也含多种扩展/语言包。关闭服务不等于减少安装，必须改为明确的包清单并固定版本策略。 |
|
||||
| 调优 | `roles/node_id/tasks/main.yml`、`roles/pgsql/templates/tiny.yml:14` 起 | `tiny` 默认仍为 250 connections，内存按宿主计算，work_mem 下限 16 MiB。共享 laptop/LXC 应按实例预算计算，不能把整机资源分配给每个实例。 |
|
||||
| HA 替换 | `roles/pgsql/tasks/main.yml`、Patroni 模板与恢复脚本 | 禁用 Patroni 并不会得到另一套完整的初始化/HA 实现；换 pg_auto_failover 是重新接生命周期和恢复流程，不能按少一个组件直接判定更便宜。 |
|
||||
| 路由 | `roles/pgsql/templates/service.cfg` | 已有 Patroni HTTP 角色检查、PostgreSQL 直连目标与摘除旧会话的逻辑，可抽取接既有 HAProxy。自动选主必须同时解决稳定入口与客户端重连。 |
|
||||
| 初始化权限 | `roles/pgsql/tasks/patroni.yml:103` 起、`pg-init-roles.sql`、`pg-init-template.sql` | `pg_provision: false` 只跳过业务 provisioning，不跳过 bootstrap SQL。默认角色和 template1 修改仍会发生；替换为最小初始化，不能直接给 Ayatori 默认 superuser DBA。 |
|
||||
| 备份 | `roles/pgsql/tasks/pgbackrest.yml`、`roles/pgsql/templates/pgbackrest.conf` | stanza/首备错误被忽略；全局 initial.done；模板仅渲染 repo1；按整机 CPU 计算并行度。需要实例化路径、明确失败门禁、保守并行度。 |
|
||||
| 调度 | `roles/pgsql/defaults/main.yml:25`、`files/postgres/pg-backup` | 默认 pg_crontab 为空；备份脚本虽然检查 primary，但硬编码 `/pg` 与默认配置。部署成功不代表周期备份存在。 |
|
||||
| 监控接入 | `roles/pg_monitor/tasks/register_victoria.yml` 等 | 上游注册到自己的 infra 目录。只取 exporter 部分，改为本仓现有 VMStaticScrape/VMRule 接入方式，不部署新的监控服务。 |
|
||||
|
||||
上游固定版本入口：[slim.yml](https://github.com/pgsty/pigsty/blob/dab5dba333a070d96fde1f9feb41761148f2be8c/slim.yml)、[pgsql tasks](https://github.com/pgsty/pigsty/tree/dab5dba333a070d96fde1f9feb41761148f2be8c/roles/pgsql/tasks)、[监控规则](https://github.com/pgsty/pigsty/blob/dab5dba333a070d96fde1f9feb41761148f2be8c/files/victoria/rules/pgsql.yml)。
|
||||
|
||||
## 建议拓扑
|
||||
|
||||
| 故障域 | 首期部署 |
|
||||
| --- | --- |
|
||||
| laptop | prod primary + Patroni；dev 单实例；etcd 成员 A；exporter。prod/dev 分别使用 ZFS dataset。 |
|
||||
| PVE 主机 A 的 LXC | prod standby + Patroni；etcd 成员 B；exporter。 |
|
||||
| PVE 主机 B 的 LXC | etcd 成员 C；本地 pgBackRest 仓库可与此共置，不额外引入对象存储服务。 |
|
||||
| 既有入口 | HAProxy 按 Patroni `/primary` 健康检查转发生产写连接，开发使用独立入口。 |
|
||||
| 既有监控 | vmagent 抓取；现有 vmalert/Alertmanager 评估与发送告警。 |
|
||||
|
||||
首期只配一个 PG standby；第二个 PVE LXC 可以只承担投票成员和备份仓库。两个 PVE LXC 必须跨实际物理主机，否则不能算两个故障域。etcd 不复用 k3s 的 DCS。
|
||||
|
||||
## Shared etcd 的管理边界
|
||||
|
||||
维护者于本轮明确：etcd 做成全 homelab 共享服务。原方案将 etcd 列入数据库部署角色；调整后 etcd 拥有独立的 inventory、部署入口、升级、快照和恢复流程，PostgreSQL 是第一个消费者。建议代码归属 `infrastructure/etcd/`，数据库只声明端点、客户端凭据和自身的 DCS prefix。物理共置方式可沿用上表，不因此增加节点。
|
||||
|
||||
- 为每个消费者分配独立账号与 key prefix，并用 etcd RBAC 约束读写范围。Patroni 示例：`namespace: /homelab/patroni/`、`scope: pg-prod`，只授权 `/homelab/patroni/pg-prod/`。路径命名本身不是权限隔离。参考 [etcd RBAC](https://etcd.io/docs/v3.6/op-guide/authentication/rbac/)。
|
||||
- 客户端配置三个直接可达的 TLS endpoints；etcd 维护不依赖生产 PostgreSQL、Ayatori 或 k3s 的可用性。etcd root 仅用于基础设施管理，不交给消费者。
|
||||
- 维护者明确 mTLS 证书使用 OpenBao 中央 CA 签发，删除 Pigsty 自建 CA 的部署依赖。建议分别定义 etcd server、peer、consumer 签发角色，限制名称、SAN、用途和签发权限；peer 需要 serverAuth/clientAuth，消费者凭据按服务独立。中央 CA 签名本身不等于获准加入 peer 网络，必须限制 peer 身份及网络入口，必要时使用中央 CA 下的专用中间 CA。证书私钥在目标机生成并以 CSR 签发；IaC 管角色和续签流程，秘密不进入 Git/state。
|
||||
- 已签发证书与信任链保存在本地，etcd 正常启动不要求即时连接 Bao；提前续签、校验证书并按成员轮换，告警覆盖过期风险。Bao 不迁入此 etcd，也不依赖 PostgreSQL；灾难恢复沿用先恢复信任根的边界。Bao 暂时不可用时，现存有效证书仍可完成 TLS 验证。
|
||||
- Patroni etcd3 使用 gRPC gateway，不能依赖客户端证书 CN 映射来完成 RBAC 身份认证；隔离实测还要求 gateway 客户端证书省略 CN；配置独立 username/password，并保留 TLS 校验。凭据由现有秘密管理提供。Pigsty tiny 模板已包含 username/password，但默认密码可退化为集群名，必须替换。参考 [Patroni etcd 配置](https://patroni.readthedocs.io/en/latest/ENVIRONMENT.html#etcd)。
|
||||
- 维护者确定 Patroni etcd 密码随机生成并存入 Bao。实施时首次创建使用 KV v2 CAS 防止覆盖已有值，之后收敛复用同一 secret；读取失败不能当成不存在而重置密码。显式轮换需同步 etcd 账号与 Patroni 配置。Ansible 在执行时读取、以 no_log 和受限文件权限下发，不新增 Ansible Vault 副本。共享 etcd 实施阶段已生成并写入该线上 secret，gateway 登录验证通过；Patroni 尚未部署。
|
||||
- 仓库配置检查显示 Ansible Vault 尚未整体退出：OpenBao、Samba AD、Proxmox 的 ansible.cfg 仍引用根目录 `.vault_pass`；OpenBao/Samba AD 仍有 vault.yml 路径，OpenBao README 仍将 OIDC client secret 归入该文件。未解密这些文件，也未查询 Bao 中是否存在对应副本。既有秘密迁移另行盘点；Bao 自身 bootstrap/恢复材料须保留独立恢复途径，不能只存在需要它们才能恢复的 Bao 中。
|
||||
- 共享的是小规模配置、服务协调和选主能力。key 权限不隔离磁盘、写入负载、compaction 或集群故障；新增消费者须控制写入量、value 大小与历史保留，监控全局容量和延迟。暂不迁移 k3s 的内部 datastore。
|
||||
- 快照、compaction、defrag 和成员维护统一由共享 etcd IaC 管理。卸载 PostgreSQL只清理其授权范围内的资源，不删除 etcd 成员或恢复整个集群。
|
||||
- etcd 全集群恢复会回退所有消费者状态，不能拿整集群 snapshot 当单个应用的回滚。恢复 runbook 要协调 Patroni 的实际数据库角色、lease 与 DCS 状态，并为其他消费者保留恢复步骤。
|
||||
|
||||
这项决策进一步支持保留 Patroni：etcd 的运行与维护成本由多个使用者共享，无需为每个数据库集群再建一套 DCS。
|
||||
|
||||
PVE 改为 bare-metal 目前只是维护者表达的后续倾向，不构成本轮迁移决定。etcd/PG 角色面向普通 Linux systemd 主机，LXC 创建及宿主资源配置单独管理，以便未来更换承载方式时复用。
|
||||
|
||||
三个 etcd 成员不是三个额外的大 VM，可与上述角色共置。etcd 数据很小，但对磁盘延迟敏感；必须观察 PVE HDD 在备份/数据库负载下的 fsync 和选举稳定性，低数据量不自动代表低延迟。
|
||||
|
||||
开发默认不使用 Patroni/etcd，不做 standby，进一步减少常驻资源。若另建 laptop 容器，能保留更多上游单机单实例代码，但会引入容器生命周期、网络和挂载管理；因此首选直接实例化 systemd service，容器为已有成熟运行时可复用时的备选。
|
||||
|
||||
生产 PG、Patroni 和 exporter 数量分别为 2、2、2；开发再加一个 PG 和 exporter；etcd 共 3 个成员。pgBackRest 可通过 timer/SSH 工作，不必常驻对象存储服务器。这是角色数量，不是 RSS 估算。
|
||||
|
||||
## 自动切换的边界
|
||||
|
||||
建议先采用异步复制，使常态 SSD 写延迟不被 HDD 提交路径限制。故障切换可能丢失尚未复制的已提交事务。`maximum_lag_on_failover` 是候选资格阈值，不能宣传成精确 RPO；官方还计入最近 TTL 窗口的 WAL。见 [Patroni replication modes](https://patroni.readthedocs.io/en/latest/replication_modes.html)。
|
||||
|
||||
自动化包含：故障检测、候选提升、入口切换、旧主恢复后 rewind/rejoin。不自动抢回 laptop 主角色，避免反复抖动；正常时优先让 laptop 承担 primary，故障后先稳定运行于 HDD。是否需要 laptop 恢复后经过稳定窗口自动回切，须另定义策略，不能用循环抢主代替。
|
||||
|
||||
必须区分“进程死亡”和“进程暂停/内核卡死”。上游 watchdog 默认 off，DCS 多数派本身不构成对卡死旧主的物理隔离。LXC 不应随意透传宿主 `/dev/watchdog`，否则可能重启整台 PVE。首期可评估软件降级与入口隔离的 best-effort 边界，但不能宣称所有故障下零双主；暂停 Patroni、网络单向中断和旧连接的行为必须进入演练。必要时再增加受控的宿主级 fencing，而不是先部署一套重型编排。见 [Patroni watchdog](https://patroni.readthedocs.io/en/latest/watchdog.html)。
|
||||
|
||||
上游 tiny 开启 DCS failsafe：现有 primary 在满足与所有已知 PG 成员通信等条件时可继续服务，但没有 DCS quorum 不能把它当作通用选主替代。见 [DCS failsafe](https://patroni.readthedocs.io/en/latest/dcs_failsafe_mode.html)。
|
||||
|
||||
既有 VyOS/HAProxy 仍是端到端可用性的依赖；复用它不等于数据库入口已经消除单点。要验证现有 VyOS 故障恢复及角色感知检查，不能沿用只检查 TCP 可连接的逻辑。
|
||||
|
||||
## 备份首期设计
|
||||
|
||||
- pgBackRest 备份仓库放在 laptop 之外的 PVE HDD，使用普通文件系统仓库与 SSH/TLS 访问，不需要 MinIO。两台生产 PG 都能归档到同一 stanza/repository。
|
||||
- 以每日 full、保留最近 3 个成功 full 及恢复所需 WAL 为初始本地策略；这不等于严格覆盖任意时刻之前 72 小时。若要求完整 72 小时窗口，应增加锚点备份及相应 WAL。异地策略另行决定。
|
||||
- 周期任务跟随实际 primary 或由仓库端发现 primary；切换之后必须继续备份，不能把 inventory 中初始 primary 当作永久主库。
|
||||
- 备份并行度先从 1 开始，设置任务超时、磁盘配额与成功时间告警；初始备份失败即判部署验收失败。
|
||||
- 连续 WAL 归档也应明确 `archive_timeout`，低写入量下不能把“配置 archive_command”当作 WAL 已及时落到仓库。初始候选为 5 分钟，结合生成量与恢复目标实测调整。
|
||||
- 上游固定 `archive-push-queue-max=4GiB`。超过限制时 pgBackRest 可丢弃归档并破坏该区间 PITR，不能照抄后声称持续可恢复；先明确可用性与保留 WAL 的取舍，并补积压、归档失败和磁盘告警。见 [pgBackRest 配置](https://pgbackrest.org/configuration.html#section-archive/option-archive-push-queue-max)。
|
||||
- 备份文件存在不算验收：隔离恢复到指定时间点并执行一致性检查；避免恢复出来的开发实例向生产仓库归档。
|
||||
|
||||
## exporter 与规则：可以裁得很小
|
||||
|
||||
对固定版本 `files/victoria/rules/pgsql.yml` 做 YAML 解析：共 **402 条 recording rules、16 条 alerts**。排除 4 条 PgBouncer alerts 和依赖整机综合压力的 `PostgresPressureHigh`，剩余 **11 条 alerts 仅依赖 8 条 recording rules**（按表达式中的 recording metric 递归追踪）。没有必要搬完整 402 条。
|
||||
|
||||
需要保留的 recording metrics：`pg:cls:partition`、`pg:db:age`、`pg:db:conn_limit`、`pg:db:conn_usage`、`pg:db:ixact_backends`、`pg:db:num_backends`、`pg:ins:lag_bytes`、`pg:ins:lag_seconds`。这只是依赖分析，不代表这些告警已通过运行验证。
|
||||
|
||||
仍需修正：
|
||||
|
||||
- 使用 Pigsty 的 `pg_exporter` 指标体系，不能直接替换为另一款 `postgres_exporter` 而照搬规则。
|
||||
- 保持或显式转换 `cls/ins/ip/job`；同 IP 多实例按实例端口和标签识别，禁止把宿主 CPU/内存用量重复算成每个数据库的用量。
|
||||
- `pg:cls:partition = count(... == 0)` 在没有 primary 时可能返回空向量，后续 `!= 1` 不足以保证无主告警；增加预期集群基线/缺失检测。
|
||||
- exporter 完全不可抓取时不能指望 exporter 自己的 `*_up` 指标告警；增加 scrape `up == 0` 与 target 消失检查。
|
||||
- `PostgresReplicationBreak` 用 streaming 数量的变化检测，不能覆盖所有持续缺副本情况；补预期 replica 数与持续复制失败。
|
||||
- severity 从上游 `CRIT/WARN/INFO` 映射为既有路由使用的 `critical/warning/info`，去掉指向未部署 Pigsty UI 的链接。
|
||||
- 从 collector 白名单开始，避免默认启用所有逐库/逐查询采集。一些 collector 依赖 `monitor` schema/辅助函数；删 bootstrap SQL 时要一起裁 collector。监控辅助 SECURITY DEFINER 与 Ayatori 管理 API 是不同权限问题，不为兼容 exporter 保留整套 DBA 权限。
|
||||
- 补本地备份年龄、WAL 归档失败/积压、磁盘余量、etcd quorum/延迟，以及数据库端到端读写探测。
|
||||
|
||||
## Ayatori 与 IaC 的边界
|
||||
|
||||
IaC 仅建立实例运维必需的账号、复制账号、最小监控权限和 Ayatori 管理账号；业务数据库及 owner 不再同时放入 Pigsty `pg_users/pg_databases`。
|
||||
|
||||
Ayatori 使用既定的非 superuser `CREATEDB/CREATEROLE` 管理账号，按现有契约验证 PostgreSQL 18 的角色成员关系与所有权操作。应用凭据走 OpenBao/ESO，不进入 inventory 明文、日志或 CR。凭据发放、轮换和删除能力是否已可用要以 Ayatori 当前实现验收，不能把 Instance 注册成功当作 provisioning 已完成。
|
||||
|
||||
Backstage 不属于本裁剪仓库。PG/etcd/systemd 的创建与恢复也不塞进 Ayatori。
|
||||
|
||||
## 推荐实现切分与验收
|
||||
|
||||
1. 建立最小 inventory 模型，区分 host 与 instance；声明 prod/dev 的 uid、dataset、端口、socket、配置路径、systemd slice 和预算。所有控制文件、锁文件、日志及 pgBackRest spool 都实例化。
|
||||
2. 独立实现 shared etcd 的部署与消费者账号管理，再提取数据库 package、instance、Patroni、backup、exporter 角色；移除 upstream NODE 全局操作。初始化有前置条件，空目录才能 init，变更部署与恢复入口分开。
|
||||
3. 先在隔离环境验证同宿主 prod/dev：重复执行不重新初始化、不重启无关实例、不改变全局 PG unit、DNS 或调优；资源统计使用 cgroup/RSS/PSS，不能把 PostgreSQL 各进程共享内存重复求和。
|
||||
4. 加入两数据节点与三投票成员，验证 primary 故障、任一 etcd 故障、网络分区、进程暂停、旧主回归、入口旧连接回收和客户端重试；记录实际 RTO 与丢失事务。
|
||||
5. 触发主库切换后再次备份，并从备份恢复；模拟仓库不可达、任务失败、WAL 积压，验证报警及恢复链边界。
|
||||
6. 连接现有监控与 Ayatori，验证 create/rotate/retain/delete 契约，再安排 CNPG 分应用迁移。Gitea 迁移同时协调应用停写与其非数据库存储，但不将其文件迁移混入本次 DB 基础设施研究。
|
||||
|
||||
pg_auto_failover 保留为候选，其官方主分支安装文档声明支持 PG 13–18，但依赖带 PG 的 monitor；是否更省资源需要在同预算、同故障场景下测量。当前不建议为了组件名字少一点而舍弃已经能提取的 Patroni 配置、路由与恢复经验。见 [官方安装文档](https://github.com/hapostgres/pg_auto_failover/blob/main/docs/install.rst)。
|
||||
|
||||
本研究阶段未修改现有数据库；后续 shared etcd 已以独立 IaC 部署，详见下方实现状态。设计边界已同步到 homelab-wiki 的 `architecture/constraints.md` 与 `services/shared-postgresql.md`，文档直接发布到 main，IaC 通过独立 PR 发布；现有 CNPG 服务状态未发生变化。
|
||||
|
||||
|
||||
## 实现入口
|
||||
|
||||
共享 etcd 的首轮实现与验证见 [etcd README](../etcd/README.md)，包括独立 Bao PKI/policy Terraform、
|
||||
LXC 与成员 Ansible、消费者秘密与 RBAC、快照和隔离三成员测试。三成员已部署，PVE 两个 LXC 使用 `pve-rg` SSD DRBD；数据库 IaC 与隔离切换/恢复测试已具备，生产主从、开发实例、稳定入口和本地备份已上线，真实切换与隔离恢复通过；现有 CNPG 应用迁移未开始。具体状态见 [部署 README](README.md)。
|
||||
旧 Ansible Vault 迁移按维护者要求后置，不阻塞新增消费者直接使用 Bao。
|
||||
|
||||
2026-09-25 后续只读前置核查：laptop 为 Ubuntu 24.04.4，ZFS `data` 池当时约 526 GiB 可用，
|
||||
5432/5433/8008 未监听；CNPG 声明镜像为 `18.3-system-trixie`。新实例继续按 PG 18 主版本准备。
|
||||
此核查没有创建 PG dataset、安装数据库或迁移应用；容量与端口使用可能随后改变，执行前须再核对。
|
||||
@@ -0,0 +1,6 @@
|
||||
[defaults]
|
||||
inventory = inventory/hosts.yml
|
||||
roles_path = roles
|
||||
host_key_checking = True
|
||||
retry_files_enabled = False
|
||||
interpreter_python = auto_silent
|
||||
@@ -0,0 +1,13 @@
|
||||
---
|
||||
- name: 读取现有实例管理凭据
|
||||
ansible.builtin.import_playbook: credentials.yml
|
||||
- name: 将 Ayatori 身份交付到独立路径（不携带 superuser 凭据）
|
||||
hosts: localhost
|
||||
connection: local
|
||||
gather_facts: false
|
||||
tasks:
|
||||
- name: 逐实例 CAS 创建并核对，只接受一致的已有值
|
||||
ansible.builtin.include_tasks: tasks/ayatori-credential.yml
|
||||
loop: [prod, dev]
|
||||
loop_control:
|
||||
loop_var: pg_ayatori_instance
|
||||
@@ -0,0 +1,160 @@
|
||||
---
|
||||
- name: 配置独立备份仓库账号与工具
|
||||
hosts: pg_repository
|
||||
become: true
|
||||
gather_facts: false
|
||||
any_errors_fatal: true
|
||||
tasks:
|
||||
- name: 必须先挂载专属数据盘
|
||||
ansible.builtin.command:
|
||||
argv: [findmnt, --mountpoint, "{{ pg_repo_path }}", --noheadings]
|
||||
changed_when: false
|
||||
check_mode: false
|
||||
- name: 配置与数据库相同的软件源
|
||||
ansible.builtin.import_role:
|
||||
name: pg_packages
|
||||
tasks_from: repository
|
||||
- name: 安装备份工具（不安装数据库）
|
||||
ansible.builtin.apt:
|
||||
name: [pgbackrest, openssh-client]
|
||||
state: present
|
||||
install_recommends: false
|
||||
- name: 创建仓库组
|
||||
ansible.builtin.group:
|
||||
name: pgbackup
|
||||
system: true
|
||||
- name: 创建仓库账号
|
||||
ansible.builtin.user:
|
||||
name: pgbackup
|
||||
group: pgbackup
|
||||
home: "{{ pg_repo_path }}"
|
||||
create_home: false
|
||||
system: true
|
||||
shell: /bin/bash
|
||||
- name: 设置仓库目录权限
|
||||
ansible.builtin.file:
|
||||
path: "{{ pg_repo_path }}"
|
||||
state: directory
|
||||
owner: pgbackup
|
||||
group: pgbackup
|
||||
mode: '0700'
|
||||
|
||||
- name: 建立专用 SSH 传输身份
|
||||
hosts: pg_hosts:pg_repository
|
||||
become: true
|
||||
gather_facts: false
|
||||
any_errors_fatal: true
|
||||
vars:
|
||||
pg_transport_user: "{{ 'pgbackup' if inventory_hostname in groups['pg_repository'] else 'pgprod' }}"
|
||||
pg_transport_home: "{{ pg_repo_path if inventory_hostname in groups['pg_repository'] else pg_data_root ~ '/prod' }}"
|
||||
tasks:
|
||||
- name: 创建 SSH 目录
|
||||
ansible.builtin.file:
|
||||
path: "{{ pg_transport_home }}/.ssh"
|
||||
state: directory
|
||||
owner: "{{ pg_transport_user }}"
|
||||
group: "{{ pg_transport_user }}"
|
||||
mode: '0700'
|
||||
# community.crypto 的跨文件系统 preserved_copy 会把 ext4 属性带到 ZFS，产生 chattr 错误。
|
||||
- name: 仅在缺失时原地生成专属传输私钥
|
||||
ansible.builtin.command:
|
||||
argv: [ssh-keygen, -q, -t, ed25519, -N, '', -f, "{{ pg_transport_home }}/.ssh/id_ed25519"]
|
||||
creates: "{{ pg_transport_home }}/.ssh/id_ed25519"
|
||||
- name: 收敛私钥属主与权限
|
||||
ansible.builtin.file:
|
||||
path: "{{ pg_transport_home }}/.ssh/id_ed25519"
|
||||
owner: "{{ pg_transport_user }}"
|
||||
group: "{{ pg_transport_user }}"
|
||||
mode: '0600'
|
||||
- name: 从既有私钥派生公开部分，不重新生成密钥
|
||||
ansible.builtin.command:
|
||||
argv: [ssh-keygen, -y, -f, "{{ pg_transport_home }}/.ssh/id_ed25519"]
|
||||
register: pg_transport_key
|
||||
changed_when: false
|
||||
- name: 保存对应公钥
|
||||
ansible.builtin.copy:
|
||||
content: "{{ pg_transport_key.stdout }}\n"
|
||||
dest: "{{ pg_transport_home }}/.ssh/id_ed25519.pub"
|
||||
owner: "{{ pg_transport_user }}"
|
||||
group: "{{ pg_transport_user }}"
|
||||
mode: '0644'
|
||||
- name: 经已信任 Ansible 通道读取 SSH 主机公钥
|
||||
ansible.builtin.slurp:
|
||||
src: /etc/ssh/ssh_host_ed25519_key.pub
|
||||
register: pg_transport_host_key
|
||||
|
||||
- name: 配置专用传输授权（不允许转发和 PTY）
|
||||
hosts: pg_hosts:pg_repository
|
||||
become: true
|
||||
gather_facts: false
|
||||
any_errors_fatal: true
|
||||
vars:
|
||||
pg_transport_user: "{{ 'pgbackup' if inventory_hostname in groups['pg_repository'] else 'pgprod' }}"
|
||||
pg_transport_home: "{{ pg_repo_path if inventory_hostname in groups['pg_repository'] else pg_data_root ~ '/prod' }}"
|
||||
pg_transport_peers: >-
|
||||
{{ groups['pg_hosts'] if inventory_hostname in groups['pg_repository'] else groups['pg_repository'] }}
|
||||
tasks:
|
||||
- name: 限定专用账号的对端密钥
|
||||
ansible.builtin.copy:
|
||||
content: |
|
||||
{% for peer in pg_transport_peers %}
|
||||
restrict {{ hostvars[peer].pg_transport_key.stdout }}
|
||||
{% endfor %}
|
||||
dest: "{{ pg_transport_home }}/.ssh/authorized_keys"
|
||||
owner: "{{ pg_transport_user }}"
|
||||
group: "{{ pg_transport_user }}"
|
||||
mode: '0600'
|
||||
- name: 保持专用 SSH 信任文件只由 IaC 管理
|
||||
ansible.builtin.copy:
|
||||
content: |
|
||||
Host {{ pg_transport_peers | map('extract', hostvars, 'ansible_host') | join(' ') }}
|
||||
StrictHostKeyChecking yes
|
||||
UpdateHostKeys no
|
||||
IdentitiesOnly yes
|
||||
IdentityFile {{ pg_transport_home }}/.ssh/id_ed25519
|
||||
UserKnownHostsFile {{ pg_transport_home }}/.ssh/known_hosts
|
||||
dest: "{{ pg_transport_home }}/.ssh/config"
|
||||
owner: "{{ pg_transport_user }}"
|
||||
group: "{{ pg_transport_user }}"
|
||||
mode: '0600'
|
||||
- name: 固定对端 SSH 主机身份
|
||||
ansible.builtin.copy:
|
||||
content: |
|
||||
{% for peer in pg_transport_peers %}
|
||||
{{ hostvars[peer].ansible_host }} {{ hostvars[peer].pg_transport_host_key.content | b64decode | trim }}
|
||||
{% endfor %}
|
||||
dest: "{{ pg_transport_home }}/.ssh/known_hosts"
|
||||
owner: "{{ pg_transport_user }}"
|
||||
group: "{{ pg_transport_user }}"
|
||||
mode: '0600'
|
||||
|
||||
- name: 配置主库感知的仓库与每日备份任务（不提前启用）
|
||||
hosts: pg_repository
|
||||
become: true
|
||||
gather_facts: false
|
||||
any_errors_fatal: true
|
||||
vars:
|
||||
pg_backup_members:
|
||||
- {address: '192.168.10.127'}
|
||||
- {address: '10.60.0.20'}
|
||||
tasks:
|
||||
- name: 创建配置目录
|
||||
ansible.builtin.file:
|
||||
path: /etc/homelab-pgbackrest
|
||||
state: directory
|
||||
owner: root
|
||||
group: pgbackup
|
||||
mode: '0750'
|
||||
- name: 安装仓库配置
|
||||
ansible.builtin.template:
|
||||
src: roles/pg_instance/templates/repository.conf.j2
|
||||
dest: /etc/homelab-pgbackrest/repository.conf
|
||||
owner: root
|
||||
group: pgbackup
|
||||
mode: '0640'
|
||||
- name: 安装备份服务和 timer
|
||||
ansible.builtin.template:
|
||||
src: "roles/pg_instance/templates/backup.{{ item }}.j2"
|
||||
dest: "/etc/systemd/system/homelab-postgresql-backup.{{ item }}"
|
||||
mode: '0644'
|
||||
loop: [service, timer]
|
||||
@@ -0,0 +1,40 @@
|
||||
---
|
||||
- name: 在已启动数据库上验证首备后启用调度
|
||||
hosts: pg_repository
|
||||
become: true
|
||||
become_user: pgbackup
|
||||
gather_facts: false
|
||||
vars:
|
||||
pg_backup_command: [pgbackrest, '--config=/etc/homelab-pgbackrest/repository.conf', '--stanza=prod']
|
||||
tasks:
|
||||
- name: 检查 stanza 元数据
|
||||
ansible.builtin.stat:
|
||||
path: "{{ pg_repo_path }}/backup/prod/backup.info"
|
||||
register: pg_stanza
|
||||
- name: 首次初始化 stanza
|
||||
ansible.builtin.command:
|
||||
argv: "{{ pg_backup_command + ['stanza-create'] }}"
|
||||
when: not pg_stanza.stat.exists
|
||||
changed_when: true
|
||||
- name: 验证真实 WAL 归档（会触发 WAL 切换）
|
||||
ansible.builtin.command:
|
||||
argv: "{{ pg_backup_command + ['check'] }}"
|
||||
changed_when: false
|
||||
- name: 查询已有备份
|
||||
ansible.builtin.command:
|
||||
argv: "{{ pg_backup_command + ['--output=json', 'info'] }}"
|
||||
register: pg_backup_info
|
||||
changed_when: false
|
||||
- name: 没有成功备份时创建首个 full
|
||||
ansible.builtin.command:
|
||||
argv: "{{ pg_backup_command + ['--type=full', 'backup'] }}"
|
||||
when: (pg_backup_info.stdout | from_json)[0].backup | length == 0
|
||||
changed_when: true
|
||||
- name: 验证成功后才启用 timer
|
||||
ansible.builtin.systemd_service:
|
||||
name: homelab-postgresql-backup.timer
|
||||
daemon_reload: true
|
||||
enabled: true
|
||||
state: started
|
||||
become_user: root
|
||||
become: true
|
||||
@@ -0,0 +1,20 @@
|
||||
---
|
||||
- name: 读取已有凭据
|
||||
ansible.builtin.import_playbook: credentials.yml
|
||||
- name: 检查承载和所有权
|
||||
ansible.builtin.import_playbook: preflight.yml
|
||||
- name: 显式初始化新实例，laptop 先启动
|
||||
hosts: pg_hosts
|
||||
become: true
|
||||
serial: 1
|
||||
any_errors_fatal: true
|
||||
tasks:
|
||||
- name: 要求显式首次初始化开关
|
||||
ansible.builtin.assert:
|
||||
that: pg_allow_initialize | bool
|
||||
fail_msg: 仅首次初始化使用 -e pg_allow_initialize=true；恢复和升级不走此入口。
|
||||
- name: 初始化声明的实例
|
||||
ansible.builtin.include_tasks: tasks/bootstrap-instance.yml
|
||||
loop: "{{ pg_instances }}"
|
||||
loop_control:
|
||||
loop_var: pg_instance
|
||||
@@ -0,0 +1,70 @@
|
||||
---
|
||||
- name: 安装每日证书续签控制任务
|
||||
hosts: pg-laptop
|
||||
become: true
|
||||
gather_facts: false
|
||||
tasks:
|
||||
- name: 创建 root 管理的程序目录
|
||||
ansible.builtin.file:
|
||||
path: /opt/homelab-postgresql-controller
|
||||
state: directory
|
||||
mode: '0755'
|
||||
- name: 安装短期身份包装器
|
||||
ansible.builtin.copy:
|
||||
src: ../run.py
|
||||
dest: /opt/homelab-postgresql-controller/run.py
|
||||
owner: root
|
||||
group: root
|
||||
mode: '0755'
|
||||
- name: 安装固定 Ansible 配置副本
|
||||
ansible.builtin.copy:
|
||||
src: "{{ playbook_dir }}/"
|
||||
dest: /opt/homelab-postgresql-controller/ansible/
|
||||
owner: root
|
||||
group: root
|
||||
mode: preserve
|
||||
- name: 使用 SPIFFE 执行一次续签预检（不输出 token）
|
||||
ansible.builtin.command:
|
||||
argv: [python3, /opt/homelab-postgresql-controller/run.py, --spiffe, ansible, renew-certificates.yml]
|
||||
environment:
|
||||
PG_BAO_SPIFFE_ROLE: homelab-postgresql
|
||||
PATH: /home/panxiao81/.local/bin:/usr/local/bin:/usr/bin:/bin
|
||||
become: true
|
||||
become_user: panxiao81
|
||||
changed_when: false
|
||||
- name: 安装续签 service
|
||||
ansible.builtin.copy:
|
||||
dest: /etc/systemd/system/homelab-postgresql-renew.service
|
||||
mode: '0644'
|
||||
content: |
|
||||
[Unit]
|
||||
Description=Renew shared PostgreSQL certificates using short-lived SPIFFE identity
|
||||
After=network-online.target
|
||||
[Service]
|
||||
Type=oneshot
|
||||
User=panxiao81
|
||||
Environment=HOME=/home/panxiao81
|
||||
Environment=PATH=/home/panxiao81/.local/bin:/usr/local/bin:/usr/bin:/bin
|
||||
Environment=PG_BAO_SPIFFE_ROLE=homelab-postgresql
|
||||
ExecStart=/usr/bin/python3 /opt/homelab-postgresql-controller/run.py --spiffe ansible renew-certificates.yml
|
||||
TimeoutStartSec=14min
|
||||
UMask=0077
|
||||
- name: 安装每日 timer
|
||||
ansible.builtin.copy:
|
||||
dest: /etc/systemd/system/homelab-postgresql-renew.timer
|
||||
mode: '0644'
|
||||
content: |
|
||||
[Unit]
|
||||
Description=Daily shared PostgreSQL certificate renewal
|
||||
[Timer]
|
||||
OnCalendar=*-*-* 04:30:00 UTC
|
||||
RandomizedDelaySec=15min
|
||||
Persistent=true
|
||||
[Install]
|
||||
WantedBy=timers.target
|
||||
- name: 预检通过后启用调度
|
||||
ansible.builtin.systemd_service:
|
||||
name: homelab-postgresql-renew.timer
|
||||
daemon_reload: true
|
||||
enabled: true
|
||||
state: started
|
||||
@@ -0,0 +1,59 @@
|
||||
---
|
||||
# 只读收敛入口；秘密首次创建、删除或轮换不放进 site.yml。
|
||||
- name: 在任何主机变更前读取并校验已有凭据
|
||||
hosts: localhost
|
||||
connection: local
|
||||
gather_facts: false
|
||||
tasks:
|
||||
- name: 要求有效的显式 Bao 身份
|
||||
ansible.builtin.assert:
|
||||
that:
|
||||
- pg_bao_token | length > 0
|
||||
- pg_bao_url is match('^https://')
|
||||
no_log: true
|
||||
- name: 读取两个实例的管理凭据
|
||||
ansible.builtin.uri:
|
||||
url: "{{ pg_bao_url }}/v1/kv/data/{{ pg_secret_paths[item] }}"
|
||||
headers:
|
||||
X-Vault-Token: "{{ pg_bao_token }}"
|
||||
status_code: 200
|
||||
loop: [prod, dev]
|
||||
register: pg_secret_response
|
||||
no_log: true
|
||||
- name: 整理并验证凭据结构
|
||||
ansible.builtin.set_fact:
|
||||
pg_loaded_credentials: "{{ pg_loaded_credentials | default({}) | combine({item.item: item.json.data.data}) }}"
|
||||
loop: "{{ pg_secret_response.results }}"
|
||||
no_log: true
|
||||
- name: 拒绝缺失或短密码
|
||||
ansible.builtin.assert:
|
||||
that:
|
||||
- pg_loaded_credentials[item].superuser_password | length >= 32
|
||||
- pg_loaded_credentials[item].ayatori_password | length >= 32
|
||||
- pg_loaded_credentials[item].ayatori_username == 'ayatori'
|
||||
loop: [prod, dev]
|
||||
no_log: true
|
||||
- name: 校验生产复制与 API 凭据
|
||||
ansible.builtin.assert:
|
||||
that:
|
||||
- pg_loaded_credentials.prod.replication_password | length >= 32
|
||||
- pg_loaded_credentials.prod.rest_password | length >= 32
|
||||
no_log: true
|
||||
- name: 读取已有 etcd 消费者凭据
|
||||
ansible.builtin.uri:
|
||||
url: "{{ pg_bao_url }}/v1/kv/data/{{ pg_etcd_secret_path }}"
|
||||
headers:
|
||||
X-Vault-Token: "{{ pg_bao_token }}"
|
||||
status_code: 200
|
||||
register: pg_etcd_response
|
||||
no_log: true
|
||||
- name: 校验 etcd prefix 并交付内存引用
|
||||
ansible.builtin.assert:
|
||||
that:
|
||||
- pg_etcd_response.json.data.data.username == 'patroni-pg-prod'
|
||||
- pg_etcd_response.json.data.data.prefix == '/homelab/patroni/pg-prod/'
|
||||
no_log: true
|
||||
- name: 保存短生命周期内存引用
|
||||
ansible.builtin.set_fact:
|
||||
pg_loaded_etcd: "{{ pg_etcd_response.json.data.data }}"
|
||||
no_log: true
|
||||
@@ -0,0 +1,31 @@
|
||||
---
|
||||
# 首次创建专用密码；已存在的数据与丢失秘密不能组合为自动重置。
|
||||
- name: 检查已有实例数据
|
||||
hosts: pg_hosts
|
||||
become: true
|
||||
gather_facts: false
|
||||
tasks:
|
||||
- name: 检查各实例 PG_VERSION
|
||||
ansible.builtin.stat:
|
||||
path: "{{ pg_data_root }}/{{ item }}/data/PG_VERSION"
|
||||
loop: "{{ pg_instances }}"
|
||||
register: pg_secret_data_files
|
||||
- name: 记录现有数据边界
|
||||
ansible.builtin.set_fact:
|
||||
pg_has_data: "{{ pg_secret_data_files.results | selectattr('stat.exists') | list | length > 0 }}"
|
||||
- name: 首次将随机实例凭据写入 Bao
|
||||
hosts: localhost
|
||||
connection: local
|
||||
gather_facts: false
|
||||
tasks:
|
||||
- name: 要求显式初始化与管理身份
|
||||
ansible.builtin.assert:
|
||||
that:
|
||||
- pg_allow_initialize | bool
|
||||
- pg_bao_token | length > 0
|
||||
no_log: true
|
||||
- name: 逐实例确认元数据和 CAS 创建
|
||||
ansible.builtin.include_tasks: tasks/initialize-secret.yml
|
||||
loop: [prod, dev]
|
||||
loop_control:
|
||||
loop_var: pg_secret_instance
|
||||
@@ -0,0 +1,40 @@
|
||||
---
|
||||
pg_major: 18
|
||||
pg_patroni_version: 4.1.5
|
||||
pg_bao_url: https://bao.ad.ddupan.top:8200
|
||||
pg_bao_token: "{{ lookup('env', 'BAO_TOKEN') }}"
|
||||
pg_config_root: /etc/homelab-postgresql
|
||||
pg_data_root: /var/lib/homelab-postgresql
|
||||
pg_bin_dir: /usr/lib/postgresql/18/bin
|
||||
pg_patroni_bin: /opt/homelab-patroni/bin/patroni
|
||||
pg_allow_initialize: false
|
||||
# 仅存放 Bao 路径，值必须在控制端 no_log 读取；不从失效读请求推断秘密不存在。
|
||||
pg_secret_paths:
|
||||
prod: infra/postgresql/prod
|
||||
dev: infra/postgresql/dev
|
||||
pg_etcd_secret_path: infra/etcd/consumers/patroni-pg-prod
|
||||
pg_etcd_endpoints: ['192.168.10.127:2379', '10.60.0.20:2379', '10.60.0.21:2379']
|
||||
pg_profiles:
|
||||
prod:
|
||||
user: pgprod
|
||||
port: 5432
|
||||
api_port: 8008
|
||||
dns: pg-prod.ad.ddupan.top
|
||||
memory_high: 768M
|
||||
memory_max: 1G
|
||||
shared_buffers: 128MB
|
||||
max_connections: 80
|
||||
dev:
|
||||
user: pgdev
|
||||
port: 5433
|
||||
dns: pg-dev.ad.ddupan.top
|
||||
memory_high: 256M
|
||||
memory_max: 384M
|
||||
shared_buffers: 32MB
|
||||
max_connections: 30
|
||||
# 客户端网络范围；HBA 仍需独立的 TLS + SCRAM 身份认证。
|
||||
pg_client_cidrs: ['192.168.10.0/24', '10.60.0.0/24', '10.42.0.0/16']
|
||||
pg_replication_addresses: ['192.168.10.127/32', '10.60.0.20/32']
|
||||
pg_repo_host: 10.60.0.21
|
||||
pg_repo_user: pgbackup
|
||||
pg_repo_path: /var/lib/homelab-pgbackrest
|
||||
@@ -0,0 +1,44 @@
|
||||
---
|
||||
all:
|
||||
vars:
|
||||
ansible_user: root
|
||||
children:
|
||||
pg_hosts:
|
||||
hosts:
|
||||
pg-laptop:
|
||||
ansible_connection: local
|
||||
ansible_user: panxiao81
|
||||
ansible_host: 192.168.10.127
|
||||
pg_address: 192.168.10.127
|
||||
pg_instances: [prod, dev]
|
||||
pg-pve1:
|
||||
ansible_host: 10.60.0.20
|
||||
pg_address: 10.60.0.20
|
||||
pg_instances: [prod]
|
||||
pg_repository:
|
||||
hosts:
|
||||
pg-pve2:
|
||||
ansible_host: 10.60.0.21
|
||||
pg_pve:
|
||||
hosts:
|
||||
pve1:
|
||||
ansible_host: 192.168.10.4
|
||||
pg_lxc_id: 150
|
||||
pg_lxc_hostname: etcd-pve1
|
||||
pg_lxc_memory: 1536
|
||||
pg_lxc_mount: /var/lib/homelab-postgresql/prod
|
||||
pg_lxc_disk_gb: 16
|
||||
pve2:
|
||||
ansible_host: 192.168.10.7
|
||||
pg_lxc_id: 151
|
||||
pg_lxc_hostname: etcd-pve2
|
||||
pg_lxc_memory: 768
|
||||
pg_lxc_mount: /var/lib/homelab-pgbackrest
|
||||
pg_lxc_disk_gb: 32
|
||||
pg_proxy:
|
||||
hosts:
|
||||
pg-proxy-vyos:
|
||||
ansible_host: 192.168.10.2
|
||||
ansible_user: vyos
|
||||
ansible_connection: ssh
|
||||
ansible_python_interpreter: /usr/bin/python3
|
||||
@@ -0,0 +1,10 @@
|
||||
---
|
||||
# 新 HDD 卷初始同步曾伴随共享链路 PingAck 超时；仅约束本项目 mp0，不改全局 quorum/协议。
|
||||
- name: 限制本项目新 HDD 卷的后台同步速率
|
||||
hosts: pg_pve
|
||||
become: true
|
||||
gather_facts: false
|
||||
serial: 1
|
||||
tasks:
|
||||
- name: 收敛本项目数据卷同步限额
|
||||
ansible.builtin.include_tasks: tasks/limit-resync.yml
|
||||
@@ -0,0 +1,23 @@
|
||||
---
|
||||
- name: 检查承载前提
|
||||
ansible.builtin.import_playbook: preflight.yml
|
||||
- name: 准备数据库软件，不配置或初始化实例
|
||||
hosts: pg_hosts
|
||||
become: true
|
||||
serial: 1
|
||||
any_errors_fatal: true
|
||||
roles:
|
||||
- pg_packages
|
||||
post_tasks:
|
||||
- name: 核对包安装没有自动创建 PG18 默认实例
|
||||
ansible.builtin.stat:
|
||||
path: /var/lib/postgresql/18/main/PG_VERSION
|
||||
register: pg_package_default_cluster
|
||||
- name: 默认实例应保持未初始化
|
||||
ansible.builtin.assert:
|
||||
that: not pg_package_default_cluster.stat.exists
|
||||
fail_msg: 发现默认 PG18 main 实例，停止后续初始化；本任务不会删除已有数据。
|
||||
- name: 验证独立 Patroni 环境可运行
|
||||
ansible.builtin.command:
|
||||
argv: ["{{ pg_patroni_bin }}", --version]
|
||||
changed_when: false
|
||||
@@ -0,0 +1,28 @@
|
||||
---
|
||||
- name: 只读核对数据库宿主与数据挂载
|
||||
hosts: pg_hosts
|
||||
become: true
|
||||
tasks:
|
||||
- name: 限定已验证的平台
|
||||
ansible.builtin.assert:
|
||||
that:
|
||||
- ansible_facts.distribution == 'Ubuntu'
|
||||
- ansible_facts.distribution_version == '24.04'
|
||||
- ansible_facts.architecture == 'x86_64'
|
||||
- pg_instances | difference(['prod', 'dev']) | length == 0
|
||||
- name: 核对共置内存预算
|
||||
ansible.builtin.assert:
|
||||
that:
|
||||
- ansible_facts.memtotal_mb >= (2048 if 'dev' in pg_instances else 1536)
|
||||
fail_msg: PG 与 etcd 共置前需通过承载 IaC 扩容；不能以当前 512 MiB LXC 直接上线。
|
||||
- name: 核对每个数据父目录已单独挂载
|
||||
ansible.builtin.command:
|
||||
argv: [findmnt, --mountpoint, "{{ pg_data_root }}/{{ item }}", --noheadings, --output, 'TARGET,FSTYPE,SOURCE']
|
||||
loop: "{{ pg_instances }}"
|
||||
changed_when: false
|
||||
check_mode: false
|
||||
- name: 核对已有数据所有权
|
||||
ansible.builtin.include_tasks: tasks/preflight-instance.yml
|
||||
loop: "{{ pg_instances }}"
|
||||
loop_control:
|
||||
loop_var: pg_instance
|
||||
@@ -0,0 +1,87 @@
|
||||
---
|
||||
# 单独 HAProxy 实例复用 VyOS 的二进制与 HA 承载，避免修改发行版生成的全局模板。
|
||||
- name: 发布已经有唯一可写主库的生产入口
|
||||
hosts: pg_proxy
|
||||
become: true
|
||||
gather_facts: false
|
||||
vars:
|
||||
pg_proxy_bind: 192.168.10.2:5432
|
||||
pg_proxy_members:
|
||||
- {name: laptop, address: '192.168.10.127', port: 5432, api_port: 8008}
|
||||
- {name: pve1, address: '10.60.0.20', port: 5432, api_port: 8008}
|
||||
tasks:
|
||||
- name: 前置核对角色端点，不为未就绪实例发布入口
|
||||
ansible.builtin.uri:
|
||||
url: "http://{{ item.address }}:{{ item.api_port }}/primary"
|
||||
status_code: [200, 503]
|
||||
loop: "{{ pg_proxy_members }}"
|
||||
register: pg_proxy_primary
|
||||
check_mode: false
|
||||
- name: 必须有且只有一个 primary
|
||||
ansible.builtin.assert:
|
||||
that: pg_proxy_primary.results | selectattr('status', 'equalto', 200) | list | length == 1
|
||||
- name: 创建跨 VyOS 镜像升级保留的配置目录
|
||||
ansible.builtin.file:
|
||||
path: /config/homelab-pg
|
||||
state: directory
|
||||
owner: root
|
||||
group: root
|
||||
mode: '0755'
|
||||
- name: 写入并校验角色感知 HAProxy 配置
|
||||
ansible.builtin.template:
|
||||
src: roles/pg_instance/templates/haproxy.cfg.j2
|
||||
dest: /config/homelab-pg/haproxy.cfg
|
||||
mode: '0644'
|
||||
validate: /usr/sbin/haproxy -c -f %s
|
||||
notify: 重载数据库代理
|
||||
- name: 保存独立服务定义
|
||||
ansible.builtin.copy:
|
||||
dest: /config/homelab-pg/homelab-pg-proxy.service
|
||||
mode: '0644'
|
||||
content: |
|
||||
[Unit]
|
||||
Description=Shared PostgreSQL role-aware TCP entry point
|
||||
After=network-online.target
|
||||
Wants=network-online.target
|
||||
[Service]
|
||||
Type=simple
|
||||
DynamicUser=true
|
||||
ExecStart=/usr/sbin/haproxy -W -db -f /config/homelab-pg/haproxy.cfg
|
||||
ExecReload=/bin/kill -USR2 $MAINPID
|
||||
Restart=on-failure
|
||||
RestartSec=3
|
||||
MemoryMax=64M
|
||||
LimitNOFILE=2048
|
||||
NoNewPrivileges=true
|
||||
PrivateTmp=true
|
||||
ProtectSystem=strict
|
||||
ProtectHome=true
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
notify: 重载数据库代理
|
||||
- name: 安装服务链接
|
||||
ansible.builtin.file:
|
||||
src: /config/homelab-pg/homelab-pg-proxy.service
|
||||
dest: /etc/systemd/system/homelab-pg-proxy.service
|
||||
state: link
|
||||
- name: 通过既有 postconfig hook 在系统升级后恢复服务链接
|
||||
ansible.builtin.blockinfile:
|
||||
path: /config/scripts/vyos-postconfig-bootup.script
|
||||
marker: '# {mark} ANSIBLE HOMELAB PG PROXY'
|
||||
insertbefore: '^exit 0'
|
||||
block: |
|
||||
ln -sfn /config/homelab-pg/homelab-pg-proxy.service /etc/systemd/system/homelab-pg-proxy.service
|
||||
systemctl daemon-reload
|
||||
systemctl enable --now homelab-pg-proxy.service
|
||||
- name: 启动独立入口
|
||||
ansible.builtin.systemd_service:
|
||||
name: homelab-pg-proxy
|
||||
daemon_reload: true
|
||||
enabled: true
|
||||
state: started
|
||||
handlers:
|
||||
- name: 重载数据库代理
|
||||
ansible.builtin.systemd_service:
|
||||
name: homelab-pg-proxy
|
||||
daemon_reload: true
|
||||
state: reloaded
|
||||
@@ -0,0 +1,85 @@
|
||||
---
|
||||
# 仅扩展已经受管的两个 shared-etcd LXC；不移动/删除已有 rootfs 或数据卷。
|
||||
- name: 串行准备 standby 与备份仓库的独立 HDD 卷
|
||||
hosts: pg_pve
|
||||
become: true
|
||||
gather_facts: false
|
||||
serial: 1
|
||||
any_errors_fatal: true
|
||||
vars:
|
||||
pg_etcd_health:
|
||||
- /opt/homelab-etcd/etcdctl
|
||||
- --endpoints=https://192.168.10.127:2379,https://10.60.0.20:2379,https://10.60.0.21:2379
|
||||
- --cacert=/etc/homelab-etcd/ca.crt
|
||||
- --cert=/etc/homelab-etcd/admin.crt
|
||||
- --key=/etc/homelab-etcd/admin.key
|
||||
- endpoint
|
||||
- health
|
||||
tasks:
|
||||
- name: 核对受管容器
|
||||
ansible.builtin.command:
|
||||
argv: [pct, config, "{{ pg_lxc_id }}"]
|
||||
register: pg_lxc_config
|
||||
changed_when: false
|
||||
check_mode: false
|
||||
- name: 拒绝未知容器或已有冲突挂载
|
||||
ansible.builtin.assert:
|
||||
that:
|
||||
- pg_lxc_id in [150, 151]
|
||||
- "'shared-etcd' in pg_lxc_config.stdout"
|
||||
- "('hostname: ' ~ pg_lxc_hostname) in pg_lxc_config.stdout"
|
||||
- >-
|
||||
'mp0:' not in pg_lxc_config.stdout or
|
||||
('mp=' ~ pg_lxc_mount ~ ',') in (pg_lxc_config.stdout ~ ',')
|
||||
- "'mp0:' not in pg_lxc_config.stdout or 'mp0: pve-rg-hdd:' in pg_lxc_config.stdout"
|
||||
- name: 验证维护前全部 etcd 成员健康
|
||||
ansible.builtin.command:
|
||||
argv: "{{ pg_etcd_health }}"
|
||||
delegate_to: localhost
|
||||
changed_when: false
|
||||
check_mode: false
|
||||
- name: 首次添加独立卷
|
||||
when: "'mp0:' not in pg_lxc_config.stdout and not ansible_check_mode"
|
||||
block:
|
||||
- name: 正常停止当前单个成员
|
||||
ansible.builtin.command:
|
||||
argv: [pct, shutdown, "{{ pg_lxc_id }}", --timeout, '60']
|
||||
changed_when: true
|
||||
- name: 从指定 HDD 池创建新的 mp0
|
||||
ansible.builtin.command:
|
||||
argv:
|
||||
- pct
|
||||
- set
|
||||
- "{{ pg_lxc_id }}"
|
||||
- --mp0
|
||||
- "pve-rg-hdd:{{ pg_lxc_disk_gb }},mp={{ pg_lxc_mount }},backup=1"
|
||||
changed_when: true
|
||||
- name: 容器启动前限制新卷的初始同步速率
|
||||
ansible.builtin.include_tasks: tasks/limit-resync.yml
|
||||
always:
|
||||
- name: 查询当前状态
|
||||
ansible.builtin.command:
|
||||
argv: [pct, status, "{{ pg_lxc_id }}"]
|
||||
register: pg_lxc_status
|
||||
changed_when: false
|
||||
- name: 恢复容器运行
|
||||
ansible.builtin.command:
|
||||
argv: [pct, start, "{{ pg_lxc_id }}"]
|
||||
changed_when: true
|
||||
when: "'running' not in pg_lxc_status.stdout"
|
||||
- name: 仅上调不足的内存限额，不缩容既有预算
|
||||
ansible.builtin.command:
|
||||
argv: [pct, set, "{{ pg_lxc_id }}", --memory, "{{ pg_lxc_memory }}"]
|
||||
when: >-
|
||||
((pg_lxc_config.stdout | regex_findall('(?m)^memory: ([0-9]+)$')) | first | int) < pg_lxc_memory
|
||||
changed_when: true
|
||||
- name: 确认当前成员恢复再处理下一台
|
||||
ansible.builtin.command:
|
||||
argv: "{{ pg_etcd_health }}"
|
||||
delegate_to: localhost
|
||||
changed_when: false
|
||||
register: pg_etcd_after
|
||||
retries: 18
|
||||
delay: 5
|
||||
until: pg_etcd_after.rc == 0
|
||||
when: not ansible_check_mode
|
||||
@@ -0,0 +1,14 @@
|
||||
---
|
||||
# 只续签和重载证书，不安装包、不变更数据库参数或重启服务。
|
||||
- name: 串行检查实例证书续签窗口
|
||||
hosts: pg_hosts
|
||||
become: true
|
||||
gather_facts: false
|
||||
serial: 1
|
||||
any_errors_fatal: true
|
||||
tasks:
|
||||
- name: 逐实例续签
|
||||
ansible.builtin.include_tasks: tasks/renew-instance.yml
|
||||
loop: "{{ pg_instances }}"
|
||||
loop_control:
|
||||
loop_var: pg_instance
|
||||
@@ -0,0 +1,4 @@
|
||||
---
|
||||
collections:
|
||||
- name: community.crypto
|
||||
version: 3.2.1
|
||||
@@ -0,0 +1,69 @@
|
||||
---
|
||||
- name: 本地生成私钥
|
||||
community.crypto.openssl_privatekey:
|
||||
path: "{{ pg_config_dir }}/{{ pg_cert_name }}.key"
|
||||
type: ECC
|
||||
curve: secp256r1
|
||||
owner: root
|
||||
group: "{{ pg_profile.user }}"
|
||||
mode: '0640'
|
||||
- name: 本地生成 CSR
|
||||
community.crypto.openssl_csr:
|
||||
path: "{{ pg_config_dir }}/{{ pg_cert_name }}.csr"
|
||||
privatekey_path: "{{ pg_config_dir }}/{{ pg_cert_name }}.key"
|
||||
common_name: "{{ pg_cert_cn | default(omit, true) }}"
|
||||
use_common_name_for_san: false
|
||||
subject_alt_name: >-
|
||||
{{ ['DNS:' ~ pg_profile.dns, 'IP:' ~ pg_address] if pg_cert_name == 'server'
|
||||
else ['DNS:etcd-patroni-pg-prod'] }}
|
||||
extended_key_usage: "{{ pg_cert_eku }}"
|
||||
key_usage: [digitalSignature]
|
||||
mode: '0644'
|
||||
register: pg_csr
|
||||
- name: 检查证书文件
|
||||
ansible.builtin.stat:
|
||||
path: "{{ pg_config_dir }}/{{ pg_cert_name }}.crt"
|
||||
register: pg_cert_file
|
||||
- name: 检查续签窗口
|
||||
community.crypto.x509_certificate_info:
|
||||
path: "{{ pg_config_dir }}/{{ pg_cert_name }}.crt"
|
||||
valid_at: {renewal: '+14d'}
|
||||
register: pg_cert_info
|
||||
when: pg_cert_file.stat.exists
|
||||
- name: 需要时请求中央 CA 签发
|
||||
when: not pg_cert_file.stat.exists or pg_csr is changed or not (pg_cert_info.valid_at.renewal | default(false))
|
||||
block:
|
||||
- name: 读取公开 CSR
|
||||
ansible.builtin.slurp:
|
||||
src: "{{ pg_config_dir }}/{{ pg_cert_name }}.csr"
|
||||
register: pg_csr_content
|
||||
- name: 请求签发（凭据仅在控制端）
|
||||
ansible.builtin.uri:
|
||||
url: "{{ pg_bao_url }}/v1/pki/sign/{{ pg_cert_role }}"
|
||||
method: POST
|
||||
headers:
|
||||
X-Vault-Token: "{{ pg_bao_token }}"
|
||||
body_format: json
|
||||
body:
|
||||
csr: "{{ pg_csr_content.content | b64decode }}"
|
||||
ttl: 1440h
|
||||
status_code: 200
|
||||
delegate_to: localhost
|
||||
become: false
|
||||
register: pg_signed
|
||||
no_log: true
|
||||
- name: 保存证书和信任链
|
||||
ansible.builtin.copy:
|
||||
content: "{{ item.content }}"
|
||||
dest: "{{ pg_config_dir }}/{{ item.name }}"
|
||||
owner: root
|
||||
group: "{{ pg_profile.user }}"
|
||||
mode: '0644'
|
||||
loop:
|
||||
- name: "{{ pg_cert_name }}.crt"
|
||||
content: "{{ pg_signed.json.data.certificate }}\n{{ pg_signed.json.data.ca_chain | join('\n') }}\n"
|
||||
- name: ca.crt
|
||||
content: "{{ pg_signed.json.data.ca_chain | join('\n') }}\n"
|
||||
- name: 记录本轮已签发证书以供显式续签入口重载
|
||||
ansible.builtin.set_fact:
|
||||
pg_certificates_changed: true
|
||||
@@ -0,0 +1,80 @@
|
||||
---
|
||||
- name: 派生实例专用路径
|
||||
ansible.builtin.set_fact:
|
||||
pg_profile: "{{ pg_profiles[pg_instance] }}"
|
||||
pg_config_dir: "{{ pg_config_root }}/{{ pg_instance }}"
|
||||
pg_parent_dir: "{{ pg_data_root }}/{{ pg_instance }}"
|
||||
pg_data_dir: "{{ pg_data_root }}/{{ pg_instance }}/data"
|
||||
pg_socket_dir: "/run/homelab-postgresql-{{ pg_instance }}"
|
||||
pg_member: "{{ inventory_hostname }}"
|
||||
pg_credentials: "{{ hostvars['localhost'].pg_loaded_credentials[pg_instance] }}"
|
||||
pg_etcd_credentials: "{{ hostvars['localhost'].pg_loaded_etcd }}"
|
||||
no_log: true
|
||||
- name: 创建实例操作系统组
|
||||
ansible.builtin.group:
|
||||
name: "{{ pg_profile.user }}"
|
||||
system: true
|
||||
- name: 创建实例操作系统账号
|
||||
ansible.builtin.user:
|
||||
name: "{{ pg_profile.user }}"
|
||||
group: "{{ pg_profile.user }}"
|
||||
home: "{{ pg_parent_dir }}"
|
||||
create_home: false
|
||||
shell: /bin/bash
|
||||
system: true
|
||||
- name: 创建隔离目录
|
||||
ansible.builtin.file:
|
||||
path: "{{ item }}"
|
||||
state: directory
|
||||
owner: "{{ pg_profile.user }}"
|
||||
group: "{{ pg_profile.user }}"
|
||||
mode: '0700'
|
||||
loop:
|
||||
- "{{ pg_parent_dir }}"
|
||||
- "{{ pg_data_dir }}"
|
||||
- "{{ pg_socket_dir }}"
|
||||
- "{{ pg_parent_dir }}/backup-lock"
|
||||
- "{{ pg_parent_dir }}/backup-spool"
|
||||
- name: 保持共享配置父目录可遍历，实例目录各自受限
|
||||
ansible.builtin.file:
|
||||
path: "{{ pg_config_root }}"
|
||||
state: directory
|
||||
owner: root
|
||||
group: root
|
||||
mode: '0755'
|
||||
- name: 创建 root 管理的配置目录
|
||||
ansible.builtin.file:
|
||||
path: "{{ pg_config_dir }}"
|
||||
state: directory
|
||||
owner: root
|
||||
group: "{{ pg_profile.user }}"
|
||||
mode: '0750'
|
||||
- name: 签发实例服务器证书
|
||||
ansible.builtin.include_tasks: certificate.yml
|
||||
vars:
|
||||
pg_cert_name: server
|
||||
pg_cert_cn: "{{ pg_profile.dns }}"
|
||||
pg_cert_role: "homelab-pg-{{ pg_instance }}"
|
||||
pg_cert_eku: [serverAuth]
|
||||
- name: 签发 Patroni 独立无 CN 客户端证书
|
||||
ansible.builtin.include_tasks: certificate.yml
|
||||
vars:
|
||||
pg_cert_name: etcd
|
||||
pg_cert_cn: ''
|
||||
pg_cert_role: homelab-etcd-client
|
||||
pg_cert_eku: [clientAuth]
|
||||
when: pg_instance == 'prod'
|
||||
- name: 写入实例配置
|
||||
ansible.builtin.template:
|
||||
src: "{{ item }}.j2"
|
||||
dest: "{{ pg_config_dir }}/{{ item }}"
|
||||
owner: root
|
||||
group: "{{ pg_profile.user }}"
|
||||
mode: '0640'
|
||||
loop: "{{ ['patroni.yml', 'pgbackrest.conf'] if pg_instance == 'prod' else ['postgresql.conf', 'pg_hba.conf'] }}"
|
||||
no_log: true
|
||||
- name: 安装实例专用服务（不自动启动或重启）
|
||||
ansible.builtin.template:
|
||||
src: instance.service.j2
|
||||
dest: "/etc/systemd/system/homelab-postgresql-{{ pg_instance }}.service"
|
||||
mode: '0644'
|
||||
+14
@@ -0,0 +1,14 @@
|
||||
[Unit]
|
||||
Description=Shared PostgreSQL primary-aware full backup
|
||||
After=network-online.target
|
||||
RequiresMountsFor={{ pg_repo_path }}
|
||||
[Service]
|
||||
Type=oneshot
|
||||
User=pgbackup
|
||||
Group=pgbackup
|
||||
ExecStart=/usr/bin/pgbackrest --config=/etc/homelab-pgbackrest/repository.conf --stanza=prod --type=full backup
|
||||
TimeoutStartSec=2h
|
||||
UMask=0077
|
||||
Nice=10
|
||||
IOSchedulingClass=best-effort
|
||||
IOSchedulingPriority=7
|
||||
@@ -0,0 +1,8 @@
|
||||
[Unit]
|
||||
Description=Daily shared PostgreSQL full backup
|
||||
[Timer]
|
||||
OnCalendar=*-*-* 03:40:00 UTC
|
||||
RandomizedDelaySec=10min
|
||||
Persistent=true
|
||||
[Install]
|
||||
WantedBy=timers.target
|
||||
@@ -0,0 +1,19 @@
|
||||
# 参考 Pigsty v4.5.0 的 Patroni 角色检查与旧连接摘除行为。
|
||||
global
|
||||
maxconn 256
|
||||
log stdout format raw local0
|
||||
stats socket /tmp/homelab-pg-haproxy.sock mode 600 level admin
|
||||
defaults
|
||||
mode tcp
|
||||
timeout connect 3s
|
||||
timeout client 1h
|
||||
timeout server 1h
|
||||
timeout check 3s
|
||||
listen pg-prod
|
||||
bind {{ pg_proxy_bind }}
|
||||
option httpchk GET /primary
|
||||
http-check expect status 200
|
||||
default-server inter 2s fall 3 rise 2 on-marked-down shutdown-sessions
|
||||
{% for member in pg_proxy_members %}
|
||||
server {{ member.name }} {{ member.address }}:{{ member.port }} check port {{ member.api_port }}
|
||||
{% endfor %}
|
||||
+33
@@ -0,0 +1,33 @@
|
||||
[Unit]
|
||||
Description=Homelab PostgreSQL {{ pg_instance }}{{ ' through Patroni' if pg_instance == 'prod' else '' }}
|
||||
After=network-online.target
|
||||
Wants=network-online.target
|
||||
RequiresMountsFor={{ pg_parent_dir }}
|
||||
[Service]
|
||||
User={{ pg_profile.user }}
|
||||
Group={{ pg_profile.user }}
|
||||
UMask=0077
|
||||
RuntimeDirectory=homelab-postgresql-{{ pg_instance }}
|
||||
RuntimeDirectoryMode=0700
|
||||
{% if pg_instance == 'prod' %}
|
||||
Type=simple
|
||||
ExecStart={{ pg_patroni_bin }} {{ pg_config_dir }}/patroni.yml
|
||||
KillMode=process
|
||||
{% else %}
|
||||
Type=simple
|
||||
ExecStart={{ pg_bin_dir }}/postgres -D {{ pg_data_dir }} -c config_file={{ pg_config_dir }}/postgresql.conf
|
||||
KillSignal=SIGINT
|
||||
{% endif %}
|
||||
Restart=on-failure
|
||||
RestartSec=5
|
||||
TimeoutStopSec=120
|
||||
MemoryHigh={{ pg_profile.memory_high }}
|
||||
MemoryMax={{ pg_profile.memory_max }}
|
||||
OOMPolicy=stop
|
||||
NoNewPrivileges=true
|
||||
ProtectSystem=strict
|
||||
ProtectHome=true
|
||||
PrivateTmp=true
|
||||
ReadWritePaths={{ pg_parent_dir }} {{ pg_config_dir }} {{ pg_socket_dir }}
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
@@ -0,0 +1,99 @@
|
||||
# 实例化部署；不引用 Pigsty 的全局 /pg 或宿主内存自动调优。
|
||||
name: {{ pg_member | to_json }}
|
||||
namespace: /homelab/patroni/
|
||||
scope: pg-prod
|
||||
restapi:
|
||||
listen: {{ pg_address }}:{{ pg_profile.api_port }}
|
||||
connect_address: {{ pg_address }}:{{ pg_profile.api_port }}
|
||||
authentication:
|
||||
username: patroni
|
||||
password: {{ pg_credentials.rest_password | to_json }}
|
||||
etcd3:
|
||||
hosts: {{ pg_etcd_endpoints | to_json }}
|
||||
protocol: https
|
||||
username: {{ pg_etcd_credentials.username | to_json }}
|
||||
password: {{ pg_etcd_credentials.password | to_json }}
|
||||
cacert: {{ pg_config_dir }}/ca.crt
|
||||
cert: {{ pg_config_dir }}/etcd.crt
|
||||
key: {{ pg_config_dir }}/etcd.key
|
||||
bootstrap:
|
||||
dcs:
|
||||
ttl: 30
|
||||
loop_wait: 10
|
||||
retry_timeout: 10
|
||||
maximum_lag_on_failover: 1048576
|
||||
check_timeline: true
|
||||
synchronous_mode: false
|
||||
failsafe_mode: false
|
||||
postgresql:
|
||||
use_pg_rewind: true
|
||||
use_slots: true
|
||||
parameters:
|
||||
max_connections: {{ pg_profile.max_connections }}
|
||||
wal_level: replica
|
||||
hot_standby: 'on'
|
||||
wal_log_hints: 'on'
|
||||
max_wal_senders: 5
|
||||
max_replication_slots: 5
|
||||
max_slot_wal_keep_size: 2GB
|
||||
max_wal_size: 1GB
|
||||
min_wal_size: 80MB
|
||||
archive_mode: 'on'
|
||||
archive_timeout: 300s
|
||||
archive_command: "pgbackrest --config={{ pg_config_dir }}/pgbackrest.conf --stanza=prod archive-push %p"
|
||||
initdb:
|
||||
- encoding: UTF8
|
||||
- locale: C.UTF-8
|
||||
- data-checksums
|
||||
postgresql:
|
||||
listen: {{ pg_address }}:{{ pg_profile.port }}
|
||||
connect_address: {{ pg_address }}:{{ pg_profile.port }}
|
||||
data_dir: {{ pg_data_dir }}
|
||||
bin_dir: {{ pg_bin_dir }}
|
||||
pgpass: {{ pg_parent_dir }}/pgpass
|
||||
use_unix_socket: true
|
||||
use_unix_socket_repl: true
|
||||
authentication:
|
||||
superuser:
|
||||
username: {{ pg_profile.user }}
|
||||
password: {{ pg_credentials.superuser_password | to_json }}
|
||||
replication:
|
||||
username: replicator
|
||||
password: {{ pg_credentials.replication_password | to_json }}
|
||||
sslmode: verify-full
|
||||
sslrootcert: {{ pg_config_dir }}/ca.crt
|
||||
parameters:
|
||||
unix_socket_directories: {{ pg_socket_dir }}
|
||||
unix_socket_permissions: '0700'
|
||||
password_encryption: scram-sha-256
|
||||
shared_buffers: {{ pg_profile.shared_buffers }}
|
||||
work_mem: 4MB
|
||||
maintenance_work_mem: 64MB
|
||||
ssl: 'on'
|
||||
ssl_cert_file: {{ pg_config_dir }}/server.crt
|
||||
ssl_key_file: {{ pg_config_dir }}/server.key
|
||||
ssl_ca_file: {{ pg_config_dir }}/ca.crt
|
||||
log_statement: none
|
||||
log_min_error_statement: panic
|
||||
pg_hba:
|
||||
- local all {{ pg_profile.user }} peer
|
||||
- local replication {{ pg_profile.user }} peer
|
||||
- local replication replicator scram-sha-256
|
||||
{% for address in pg_replication_addresses %}
|
||||
- hostssl replication replicator {{ address }} scram-sha-256
|
||||
- hostssl all {{ pg_profile.user }} {{ address }} scram-sha-256
|
||||
{% endfor %}
|
||||
{% for cidr in pg_client_cidrs %}
|
||||
- hostssl all all {{ cidr }} scram-sha-256
|
||||
{% endfor %}
|
||||
- host all all 0.0.0.0/0 reject
|
||||
- host all all ::/0 reject
|
||||
remove_data_directory_on_rewind_failure: false
|
||||
remove_data_directory_on_diverged_timelines: false
|
||||
watchdog:
|
||||
mode: 'off'
|
||||
tags:
|
||||
nofailover: false
|
||||
noloadbalance: false
|
||||
clonefrom: false
|
||||
nosync: false
|
||||
@@ -0,0 +1,6 @@
|
||||
local all {{ pg_profile.user }} peer
|
||||
{% for cidr in pg_client_cidrs %}
|
||||
hostssl all all {{ cidr }} scram-sha-256
|
||||
{% endfor %}
|
||||
host all all 0.0.0.0/0 reject
|
||||
host all all ::/0 reject
|
||||
+22
@@ -0,0 +1,22 @@
|
||||
[global]
|
||||
{% if pg_repo_remote | default(true) %}
|
||||
repo1-host={{ pg_repo_host }}
|
||||
repo1-host-user={{ pg_repo_user }}
|
||||
repo1-host-config=/etc/homelab-pgbackrest/repository.conf
|
||||
{% endif %}
|
||||
repo1-path={{ pg_repo_path }}
|
||||
repo1-retention-full=3
|
||||
process-max=1
|
||||
compress-type=zst
|
||||
compress-level=3
|
||||
start-fast=y
|
||||
log-level-console=warn
|
||||
log-level-file=off
|
||||
lock-path={{ pg_parent_dir }}/backup-lock
|
||||
spool-path={{ pg_parent_dir }}/backup-spool
|
||||
# 不设置 archive-push-queue-max；超限丢弃 WAL 会破坏恢复链。
|
||||
[prod]
|
||||
pg1-path={{ pg_data_dir }}
|
||||
pg1-port={{ pg_profile.port }}
|
||||
pg1-socket-path={{ pg_socket_dir }}
|
||||
pg1-user={{ pg_profile.user }}
|
||||
+18
@@ -0,0 +1,18 @@
|
||||
listen_addresses = '{{ pg_address }}'
|
||||
port = {{ pg_profile.port }}
|
||||
unix_socket_directories = '{{ pg_socket_dir }}'
|
||||
unix_socket_permissions = 0700
|
||||
hba_file = '{{ pg_config_dir }}/pg_hba.conf'
|
||||
password_encryption = 'scram-sha-256'
|
||||
shared_buffers = '{{ pg_profile.shared_buffers }}'
|
||||
max_connections = {{ pg_profile.max_connections }}
|
||||
work_mem = '4MB'
|
||||
maintenance_work_mem = '32MB'
|
||||
max_wal_size = '512MB'
|
||||
ssl = on
|
||||
ssl_cert_file = '{{ pg_config_dir }}/server.crt'
|
||||
ssl_key_file = '{{ pg_config_dir }}/server.key'
|
||||
ssl_ca_file = '{{ pg_config_dir }}/ca.crt'
|
||||
archive_mode = off
|
||||
log_statement = none
|
||||
log_min_error_statement = panic
|
||||
+21
@@ -0,0 +1,21 @@
|
||||
# 备份从仓库发起，发现实际 primary，避免切换后定时器仍依赖旧主。
|
||||
[global]
|
||||
repo1-path={{ pg_repo_path }}
|
||||
repo1-retention-full=3
|
||||
process-max=1
|
||||
compress-type=zst
|
||||
compress-level=3
|
||||
start-fast=y
|
||||
log-level-console=info
|
||||
log-level-file=off
|
||||
lock-path={{ pg_repo_path }}/lock
|
||||
[prod]
|
||||
{% for member in pg_backup_members %}
|
||||
pg{{ loop.index }}-host={{ member.address }}
|
||||
pg{{ loop.index }}-host-user=pgprod
|
||||
pg{{ loop.index }}-host-config=/etc/homelab-postgresql/prod/pgbackrest.conf
|
||||
pg{{ loop.index }}-path=/var/lib/homelab-postgresql/prod/data
|
||||
pg{{ loop.index }}-port=5432
|
||||
pg{{ loop.index }}-socket-path=/run/homelab-postgresql-prod
|
||||
pg{{ loop.index }}-user=pgprod
|
||||
{% endfor %}
|
||||
@@ -0,0 +1,40 @@
|
||||
---
|
||||
- name: 配置官方软件源
|
||||
ansible.builtin.import_tasks: repository.yml
|
||||
- name: 安装集群管理基础包（不安装默认主版本元包）
|
||||
ansible.builtin.apt:
|
||||
name: postgresql-common
|
||||
state: present
|
||||
- name: 查询已安装包以避免重复制造临时配置变更
|
||||
ansible.builtin.package_facts:
|
||||
manager: apt
|
||||
- name: 临时禁止包安装自动建立默认 main 实例
|
||||
when: >-
|
||||
['postgresql-18', 'postgresql-client-18', 'pgbackrest']
|
||||
| difference(ansible_facts.packages.keys() | list) | length > 0
|
||||
block:
|
||||
- name: 确保包配置片段目录存在
|
||||
ansible.builtin.file:
|
||||
path: /etc/postgresql-common/createcluster.d
|
||||
state: directory
|
||||
mode: '0755'
|
||||
- name: 禁止本次 apt postinst 自动初始化
|
||||
ansible.builtin.copy:
|
||||
dest: /etc/postgresql-common/createcluster.d/90-homelab-no-auto.conf
|
||||
content: "create_main_cluster = false\n"
|
||||
mode: '0644'
|
||||
- name: 安装明确的 PG 主版本与备份工具
|
||||
ansible.builtin.apt:
|
||||
name: ['postgresql-18', 'postgresql-client-18', pgbackrest]
|
||||
state: present
|
||||
install_recommends: false
|
||||
always:
|
||||
- name: 撤回临时包安装选项
|
||||
ansible.builtin.file:
|
||||
path: /etc/postgresql-common/createcluster.d/90-homelab-no-auto.conf
|
||||
state: absent
|
||||
- name: 在独立 venv 安装固定 Patroni 版本
|
||||
ansible.builtin.pip:
|
||||
name: 'patroni[etcd3,psycopg3]=={{ pg_patroni_version }}'
|
||||
virtualenv: /opt/homelab-patroni
|
||||
virtualenv_command: python3 -m venv
|
||||
@@ -0,0 +1,51 @@
|
||||
---
|
||||
# 共用宿主机上其他第三方源故障不能阻塞本项目；仅刷新 Ubuntu/PGDG，仍强制签名验证。
|
||||
- name: 仅刷新 Ubuntu 官方基础软件源 # noqa: command-instead-of-module
|
||||
ansible.builtin.command:
|
||||
argv:
|
||||
- apt-get
|
||||
- -o
|
||||
- Dir::Etc::sourcelist=/etc/apt/sources.list.d/ubuntu.sources
|
||||
- -o
|
||||
- Dir::Etc::sourceparts=-
|
||||
- -o
|
||||
- APT::Get::List-Cleanup=0
|
||||
- -o
|
||||
- APT::Update::Error-Mode=any
|
||||
- update
|
||||
changed_when: false
|
||||
- name: 安装软件源及 Python 依赖
|
||||
ansible.builtin.apt:
|
||||
name: [ca-certificates, python3-debian, python3-venv, python3-cryptography, acl]
|
||||
state: present
|
||||
- name: 安装官方 PGDG 公钥
|
||||
ansible.builtin.get_url:
|
||||
url: https://www.postgresql.org/media/keys/ACCC4CF8.asc
|
||||
dest: /usr/share/keyrings/homelab-pgdg.asc
|
||||
mode: '0644'
|
||||
register: pg_repo_key
|
||||
retries: 3
|
||||
delay: 5
|
||||
until: pg_repo_key is succeeded
|
||||
- name: 声明 PGDG 官方软件源
|
||||
ansible.builtin.deb822_repository:
|
||||
name: homelab-pgdg
|
||||
uris: https://apt.postgresql.org/pub/repos/apt
|
||||
suites: noble-pgdg
|
||||
components: main
|
||||
architectures: amd64
|
||||
signed_by: /usr/share/keyrings/homelab-pgdg.asc
|
||||
- name: 仅刷新本项目 PGDG 软件源 # noqa: command-instead-of-module
|
||||
ansible.builtin.command:
|
||||
argv:
|
||||
- apt-get
|
||||
- -o
|
||||
- Dir::Etc::sourcelist=/etc/apt/sources.list.d/homelab-pgdg.sources
|
||||
- -o
|
||||
- Dir::Etc::sourceparts=-
|
||||
- -o
|
||||
- APT::Get::List-Cleanup=0
|
||||
- -o
|
||||
- APT::Update::Error-Mode=any
|
||||
- update
|
||||
changed_when: false
|
||||
@@ -0,0 +1,17 @@
|
||||
---
|
||||
- name: 检查承载前提
|
||||
ansible.builtin.import_playbook: preflight.yml
|
||||
- name: 验证秘密可用
|
||||
ansible.builtin.import_playbook: credentials.yml
|
||||
- name: 配置数据库实例（不初始化、不启动、不自动重启）
|
||||
hosts: pg_hosts
|
||||
become: true
|
||||
roles:
|
||||
- pg_packages
|
||||
tasks:
|
||||
- name: 配置声明的实例
|
||||
ansible.builtin.include_role:
|
||||
name: pg_instance
|
||||
loop: "{{ pg_instances }}"
|
||||
loop_control:
|
||||
loop_var: pg_instance
|
||||
@@ -0,0 +1,57 @@
|
||||
---
|
||||
# 只创建专属 dataset；PVE 新挂载需串行重启 etcd 成员，单独处理，不混入日常数据库收敛。
|
||||
- name: 创建 laptop 独立生产与开发 dataset
|
||||
hosts: pg-laptop
|
||||
become: true
|
||||
gather_facts: false
|
||||
tasks:
|
||||
- name: 查询专属 ZFS dataset
|
||||
ansible.builtin.command:
|
||||
argv: [zfs, list, -H, -o, name, "data/homelab-pg-{{ item }}"]
|
||||
loop: [prod, dev]
|
||||
register: pg_datasets
|
||||
changed_when: false
|
||||
failed_when: pg_datasets.rc not in [0, 1]
|
||||
check_mode: false
|
||||
- name: 检查新 dataset 挂载路径未被占用
|
||||
ansible.builtin.stat:
|
||||
path: "{{ pg_data_root }}/{{ item.item }}"
|
||||
loop: "{{ pg_datasets.results }}"
|
||||
when: item.rc == 1
|
||||
register: pg_new_mounts
|
||||
- name: 禁止覆盖已有目录
|
||||
ansible.builtin.assert:
|
||||
that: not item.stat.exists
|
||||
loop: "{{ pg_new_mounts.results }}"
|
||||
when: not (item.skipped | default(false))
|
||||
- name: 新建本项目专用 ZFS dataset
|
||||
ansible.builtin.command:
|
||||
argv:
|
||||
- zfs
|
||||
- create
|
||||
- -o
|
||||
- "mountpoint={{ pg_data_root }}/{{ item.item }}"
|
||||
- -o
|
||||
- recordsize=16K
|
||||
- -o
|
||||
- compression=lz4
|
||||
- -o
|
||||
- atime=off
|
||||
- -o
|
||||
- "quota={{ '32G' if item.item == 'prod' else '8G' }}"
|
||||
- -o
|
||||
- org.homelab:owner=shared-postgresql
|
||||
- "data/homelab-pg-{{ item.item }}"
|
||||
loop: "{{ pg_datasets.results }}"
|
||||
when: item.rc == 1
|
||||
changed_when: true
|
||||
- name: 核对所有权与挂载路径
|
||||
ansible.builtin.command:
|
||||
argv: [zfs, get, -H, -o, value, 'org.homelab:owner,mountpoint', "data/homelab-pg-{{ item }}"]
|
||||
loop: [prod, dev]
|
||||
register: pg_dataset_owner
|
||||
changed_when: false
|
||||
failed_when: >-
|
||||
pg_dataset_owner.rc != 0 or
|
||||
pg_dataset_owner.stdout_lines != ['shared-postgresql', pg_data_root ~ '/' ~ item]
|
||||
when: not ansible_check_mode
|
||||
@@ -0,0 +1,37 @@
|
||||
---
|
||||
- name: 读取独立管理凭据元数据
|
||||
ansible.builtin.uri:
|
||||
url: "{{ pg_bao_url }}/v1/kv/metadata/infra/postgresql/ayatori/{{ pg_ayatori_instance }}"
|
||||
headers: {X-Vault-Token: "{{ pg_bao_token }}"}
|
||||
status_code: [200, 404]
|
||||
register: pg_admin_metadata
|
||||
no_log: true
|
||||
- name: 只在明确缺失时写入独立的用户名密码
|
||||
ansible.builtin.uri:
|
||||
url: "{{ pg_bao_url }}/v1/kv/data/infra/postgresql/ayatori/{{ pg_ayatori_instance }}"
|
||||
method: POST
|
||||
headers: {X-Vault-Token: "{{ pg_bao_token }}"}
|
||||
body_format: json
|
||||
body:
|
||||
options: {cas: 0}
|
||||
data:
|
||||
username: "{{ pg_loaded_credentials[pg_ayatori_instance].ayatori_username }}"
|
||||
password: "{{ pg_loaded_credentials[pg_ayatori_instance].ayatori_password }}"
|
||||
status_code: 200
|
||||
when: pg_admin_metadata.status == 404
|
||||
changed_when: true
|
||||
no_log: true
|
||||
- name: 回读已有管理凭据
|
||||
ansible.builtin.uri:
|
||||
url: "{{ pg_bao_url }}/v1/kv/data/infra/postgresql/ayatori/{{ pg_ayatori_instance }}"
|
||||
headers: {X-Vault-Token: "{{ pg_bao_token }}"}
|
||||
status_code: 200
|
||||
register: pg_admin_readback
|
||||
no_log: true
|
||||
- name: 不隐式覆盖或轮换不一致凭据
|
||||
ansible.builtin.assert:
|
||||
that:
|
||||
- pg_admin_readback.json.data.data.keys() | sort == ['password', 'username']
|
||||
- pg_admin_readback.json.data.data.username == pg_loaded_credentials[pg_ayatori_instance].ayatori_username
|
||||
- pg_admin_readback.json.data.data.password == pg_loaded_credentials[pg_ayatori_instance].ayatori_password
|
||||
no_log: true
|
||||
@@ -0,0 +1,172 @@
|
||||
---
|
||||
- name: 设置实例初始化参数
|
||||
ansible.builtin.set_fact:
|
||||
pg_profile: "{{ pg_profiles[pg_instance] }}"
|
||||
pg_parent_dir: "{{ pg_data_root }}/{{ pg_instance }}"
|
||||
pg_data_dir: "{{ pg_data_root }}/{{ pg_instance }}/data"
|
||||
pg_config_dir: "{{ pg_config_root }}/{{ pg_instance }}"
|
||||
pg_bootstrap_password: "{{ hostvars['localhost'].pg_loaded_credentials[pg_instance].ayatori_password }}"
|
||||
pg_socket_dir: "/run/homelab-postgresql-{{ pg_instance }}"
|
||||
no_log: true
|
||||
- name: 检查版本标记
|
||||
ansible.builtin.stat:
|
||||
path: "{{ pg_data_dir }}/PG_VERSION"
|
||||
register: pg_version_file
|
||||
- name: 核对未初始化目录为空
|
||||
ansible.builtin.find:
|
||||
paths: "{{ pg_data_dir }}"
|
||||
hidden: true
|
||||
file_type: any
|
||||
register: pg_initial_files
|
||||
when: not pg_version_file.stat.exists
|
||||
- name: 禁止覆盖不完整或未知数据
|
||||
ansible.builtin.assert:
|
||||
that: pg_version_file.stat.exists or pg_initial_files.matched == 0
|
||||
- name: 初始化独立开发实例
|
||||
when: pg_instance == 'dev' and not pg_version_file.stat.exists
|
||||
block:
|
||||
- name: 临时写入开发管理员密码
|
||||
ansible.builtin.copy:
|
||||
content: "{{ hostvars['localhost'].pg_loaded_credentials.dev.superuser_password }}\n"
|
||||
dest: "{{ pg_parent_dir }}/.init-password"
|
||||
owner: "{{ pg_profile.user }}"
|
||||
mode: '0600'
|
||||
no_log: true
|
||||
- name: 仅在空目录初始化
|
||||
ansible.builtin.command:
|
||||
argv:
|
||||
- "{{ pg_bin_dir }}/initdb"
|
||||
- -D
|
||||
- "{{ pg_data_dir }}"
|
||||
- --username={{ pg_profile.user }}
|
||||
- --pwfile={{ pg_parent_dir }}/.init-password
|
||||
- --auth-local=peer
|
||||
- --auth-host=scram-sha-256
|
||||
- --encoding=UTF8
|
||||
- --locale=C.UTF-8
|
||||
- --data-checksums
|
||||
become: true
|
||||
become_user: "{{ pg_profile.user }}"
|
||||
changed_when: true
|
||||
no_log: true
|
||||
always:
|
||||
- name: 删除临时初始化密码文件
|
||||
ansible.builtin.file:
|
||||
path: "{{ pg_parent_dir }}/.init-password"
|
||||
state: absent
|
||||
- name: 登记专属数据目录（启动失败后仍可辨认来源）
|
||||
ansible.builtin.copy:
|
||||
content: "shared-postgresql/{{ pg_instance }}\n"
|
||||
dest: "{{ pg_parent_dir }}/.homelab-owned"
|
||||
owner: root
|
||||
mode: '0644'
|
||||
- name: 启动配置完成的新实例
|
||||
ansible.builtin.systemd_service:
|
||||
name: "homelab-postgresql-{{ pg_instance }}"
|
||||
daemon_reload: true
|
||||
enabled: true
|
||||
state: started
|
||||
- name: 等待本机 SQL 可连接
|
||||
ansible.builtin.command:
|
||||
argv:
|
||||
- '{{ pg_bin_dir }}/psql'
|
||||
- -X
|
||||
- -At
|
||||
- -h
|
||||
- '{{ pg_socket_dir }}'
|
||||
- -p
|
||||
- '{{ pg_profile.port | string }}'
|
||||
- -d
|
||||
- postgres
|
||||
- -c
|
||||
- SELECT 1
|
||||
become: true
|
||||
become_user: "{{ pg_profile.user }}"
|
||||
register: pg_ready
|
||||
until: pg_ready.rc == 0
|
||||
retries: 30
|
||||
delay: 2
|
||||
changed_when: false
|
||||
- name: 核对本节点是否可写
|
||||
ansible.builtin.command:
|
||||
argv:
|
||||
- '{{ pg_bin_dir }}/psql'
|
||||
- -X
|
||||
- -At
|
||||
- -h
|
||||
- '{{ pg_socket_dir }}'
|
||||
- -p
|
||||
- '{{ pg_profile.port | string }}'
|
||||
- -d
|
||||
- postgres
|
||||
- -c
|
||||
- SELECT NOT pg_is_in_recovery()
|
||||
become: true
|
||||
become_user: "{{ pg_profile.user }}"
|
||||
register: pg_is_primary
|
||||
changed_when: false
|
||||
- name: 在可写实例创建 Ayatori 最小管理账号
|
||||
when: pg_is_primary.stdout | trim == 't'
|
||||
block:
|
||||
- name: 查询 Ayatori 账号
|
||||
ansible.builtin.command:
|
||||
argv:
|
||||
- '{{ pg_bin_dir }}/psql'
|
||||
- -X
|
||||
- -At
|
||||
- -h
|
||||
- '{{ pg_socket_dir }}'
|
||||
- -p
|
||||
- '{{ pg_profile.port | string }}'
|
||||
- -d
|
||||
- postgres
|
||||
- -c
|
||||
- SELECT count(*) FROM pg_roles WHERE rolname = 'ayatori'
|
||||
become: true
|
||||
become_user: "{{ pg_profile.user }}"
|
||||
register: pg_ayatori_exists
|
||||
changed_when: false
|
||||
- name: 首次建立受限角色（不静默轮换已存在密码）
|
||||
ansible.builtin.command:
|
||||
argv:
|
||||
- '{{ pg_bin_dir }}/psql'
|
||||
- -X
|
||||
- -v
|
||||
- ON_ERROR_STOP=1
|
||||
- -h
|
||||
- '{{ pg_socket_dir }}'
|
||||
- -p
|
||||
- '{{ pg_profile.port | string }}'
|
||||
- -d
|
||||
- postgres
|
||||
stdin: >-
|
||||
CREATE ROLE ayatori LOGIN NOSUPERUSER CREATEDB CREATEROLE NOREPLICATION NOBYPASSRLS
|
||||
PASSWORD '{{ pg_bootstrap_password | replace("'", "''") }}';
|
||||
become: true
|
||||
become_user: "{{ pg_profile.user }}"
|
||||
when: pg_ayatori_exists.stdout | trim == '0'
|
||||
changed_when: true
|
||||
no_log: true
|
||||
- name: 以实际保存密码及 verify-full 验证权限
|
||||
ansible.builtin.command:
|
||||
argv:
|
||||
- "{{ pg_bin_dir }}/psql"
|
||||
- -X
|
||||
- -At
|
||||
- -c
|
||||
- >-
|
||||
SELECT NOT rolsuper AND rolcreatedb AND rolcreaterole AND NOT pg_is_in_recovery()
|
||||
FROM pg_roles WHERE rolname = current_user
|
||||
environment:
|
||||
PGHOST: "{{ pg_profile.dns }}"
|
||||
PGHOSTADDR: "{{ pg_address }}"
|
||||
PGPORT: "{{ pg_profile.port | string }}"
|
||||
PGDATABASE: postgres
|
||||
PGUSER: ayatori
|
||||
PGPASSWORD: "{{ hostvars['localhost'].pg_loaded_credentials[pg_instance].ayatori_password }}"
|
||||
PGSSLMODE: verify-full
|
||||
PGSSLROOTCERT: "{{ pg_config_dir }}/ca.crt"
|
||||
register: pg_ayatori_check
|
||||
changed_when: false
|
||||
failed_when: pg_ayatori_check.rc != 0 or pg_ayatori_check.stdout | trim != 't'
|
||||
no_log: true
|
||||
@@ -0,0 +1,46 @@
|
||||
---
|
||||
- name: 核对秘密元数据（非 404 错误不能视为不存在）
|
||||
ansible.builtin.uri:
|
||||
url: "{{ pg_bao_url }}/v1/kv/metadata/{{ pg_secret_paths[pg_secret_instance] }}"
|
||||
headers:
|
||||
X-Vault-Token: "{{ pg_bao_token }}"
|
||||
status_code: [200, 404]
|
||||
register: pg_secret_metadata
|
||||
no_log: true
|
||||
- name: 明确不存在时创建一次
|
||||
when: pg_secret_metadata.status == 404
|
||||
block:
|
||||
- name: 拒绝为已有实例重造丢失秘密
|
||||
ansible.builtin.assert:
|
||||
that: >-
|
||||
groups['pg_hosts'] | map('extract', hostvars, 'pg_has_data')
|
||||
| select('equalto', true) | list | length == 0
|
||||
- name: 随机生成只在本次执行内存存在的值
|
||||
ansible.builtin.set_fact:
|
||||
pg_new_secret:
|
||||
ayatori_username: ayatori
|
||||
ayatori_password: "{{ lookup('password', '/dev/null', length=48, chars=['ascii_letters', 'digits']) }}"
|
||||
superuser_password: "{{ lookup('password', '/dev/null', length=48, chars=['ascii_letters', 'digits']) }}"
|
||||
replication_password: "{{ lookup('password', '/dev/null', length=48, chars=['ascii_letters', 'digits']) }}"
|
||||
rest_password: "{{ lookup('password', '/dev/null', length=48, chars=['ascii_letters', 'digits']) }}"
|
||||
no_log: true
|
||||
- name: 使用 KV v2 CAS=0 防覆盖写入
|
||||
ansible.builtin.uri:
|
||||
url: "{{ pg_bao_url }}/v1/kv/data/{{ pg_secret_paths[pg_secret_instance] }}"
|
||||
method: POST
|
||||
headers:
|
||||
X-Vault-Token: "{{ pg_bao_token }}"
|
||||
body_format: json
|
||||
body:
|
||||
options: {cas: 0}
|
||||
data: "{{ pg_new_secret }}"
|
||||
status_code: 200
|
||||
changed_when: true
|
||||
no_log: true
|
||||
- name: 确认已有秘密数据可读，软删除必须失败
|
||||
ansible.builtin.uri:
|
||||
url: "{{ pg_bao_url }}/v1/kv/data/{{ pg_secret_paths[pg_secret_instance] }}"
|
||||
headers:
|
||||
X-Vault-Token: "{{ pg_bao_token }}"
|
||||
status_code: 200
|
||||
no_log: true
|
||||
@@ -0,0 +1,34 @@
|
||||
---
|
||||
- name: 从受管容器配置定位资源
|
||||
ansible.builtin.command:
|
||||
argv: [pct, config, "{{ pg_lxc_id }}"]
|
||||
register: pg_resync_config
|
||||
changed_when: false
|
||||
check_mode: false
|
||||
- name: 核对容器和专属挂载
|
||||
ansible.builtin.assert:
|
||||
that:
|
||||
- pg_lxc_id in [150, 151]
|
||||
- "'shared-etcd' in pg_resync_config.stdout"
|
||||
- "('mp=' ~ pg_lxc_mount ~ ',') in pg_resync_config.stdout"
|
||||
- "pg_resync_config.stdout | regex_findall('(?m)^mp0: pve-rg-hdd:(pm-[0-9a-f]+)_') | length == 1"
|
||||
- name: 提取明确匹配的 LINSTOR resource definition
|
||||
ansible.builtin.set_fact:
|
||||
pg_resync_resource: "{{ (pg_resync_config.stdout | regex_findall('(?m)^mp0: pve-rg-hdd:(pm-[0-9a-f]+)_'))[0] }}"
|
||||
- name: 读取本资源已有配置
|
||||
ansible.builtin.command:
|
||||
argv: [linstor, resource-definition, list-properties, "{{ pg_resync_resource }}"]
|
||||
register: pg_resync_properties
|
||||
changed_when: false
|
||||
check_mode: false
|
||||
- name: 每个新卷将自适应同步速率上限设为 10 MiB/s
|
||||
ansible.builtin.command:
|
||||
argv:
|
||||
- linstor
|
||||
- resource-definition
|
||||
- set-property
|
||||
- "{{ pg_resync_resource }}"
|
||||
- DrbdOptions/PeerDevice/c-max-rate
|
||||
- '10240'
|
||||
when: pg_resync_properties.stdout is not search('c-max-rate\s*│\s*10240\s*│')
|
||||
changed_when: true
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user