Compare commits
2
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 panxiao81
|
f00b5e3fc2
|
||
|
panxiao81
|
9a800f55fc
|
@@ -1,32 +0,0 @@
|
||||
---
|
||||
name: dind-fuse-image
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
push:
|
||||
branches: [main]
|
||||
paths:
|
||||
- platform/gitea-runner/images/dind-fuse/**
|
||||
- .gitea/workflows/dind-fuse-image.yml
|
||||
|
||||
jobs:
|
||||
publish:
|
||||
runs-on: [self-hosted, pod]
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- name: Log in to the Gitea container registry
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
registry: git.ddupan.top
|
||||
username: ${{ gitea.actor }}
|
||||
password: ${{ secrets.REGISTRY_TOKEN }}
|
||||
|
||||
- name: Build and publish
|
||||
uses: docker/build-push-action@v6
|
||||
with:
|
||||
context: platform/gitea-runner/images/dind-fuse
|
||||
push: true
|
||||
tags: |
|
||||
git.ddupan.top/panxiao81/dind-fuse:29.7.1
|
||||
git.ddupan.top/panxiao81/dind-fuse:latest
|
||||
@@ -1,24 +0,0 @@
|
||||
---
|
||||
name: kata-runner-smoke
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
push:
|
||||
branches:
|
||||
- poc/kata-runner-smoke
|
||||
paths:
|
||||
- .gitea/workflows/kata-runner-smoke.yml
|
||||
|
||||
jobs:
|
||||
smoke:
|
||||
runs-on: [self-hosted, pod]
|
||||
steps:
|
||||
- name: Verify isolated job environment
|
||||
shell: sh
|
||||
run: |
|
||||
set -eu
|
||||
uname -a
|
||||
grep -q '^ID=alpine$' /etc/os-release
|
||||
test -f /.dockerenv
|
||||
test ! -e /dev/kvm
|
||||
printf 'kata ephemeral runner job ok\n'
|
||||
@@ -1,63 +0,0 @@
|
||||
---
|
||||
name: kind-on-kata-smoke
|
||||
|
||||
on:
|
||||
push:
|
||||
branches:
|
||||
- poc/kind-on-kata
|
||||
paths:
|
||||
- .gitea/workflows/kind-on-kata-smoke.yml
|
||||
workflow_dispatch:
|
||||
|
||||
jobs:
|
||||
smoke:
|
||||
runs-on: [self-hosted, pod]
|
||||
steps:
|
||||
- name: Create nested kind cluster
|
||||
shell: sh
|
||||
env:
|
||||
KIND_VERSION: v0.27.0
|
||||
KIND_NODE_IMAGE: kindest/node:v1.32.2@sha256:f226345927d7e348497136874b6d207e0b32cc52154ad8323129352923a3142f
|
||||
run: |
|
||||
set -eu
|
||||
apk add --no-cache ca-certificates curl docker-cli
|
||||
curl --retry 5 --retry-all-errors --connect-timeout 15 -fsSLo /tmp/kind \
|
||||
"https://kind.sigs.k8s.io/dl/${KIND_VERSION}/kind-linux-amd64"
|
||||
curl --retry 5 --retry-all-errors --connect-timeout 15 -fsSLo /tmp/kind.sha256sum \
|
||||
"https://kind.sigs.k8s.io/dl/${KIND_VERSION}/kind-linux-amd64.sha256sum"
|
||||
expected="$(awk '{print $1}' /tmp/kind.sha256sum)"
|
||||
printf '%s %s\n' "$expected" /tmp/kind | sha256sum -c -
|
||||
install -m 0755 /tmp/kind /usr/local/bin/kind
|
||||
docker info --format 'kernel={{.KernelVersion}} driver={{.Driver}}'
|
||||
test "$(docker info --format '{{.Driver}}')" = overlay2
|
||||
|
||||
cleanup() {
|
||||
kind delete cluster --name nested >/dev/null 2>&1 || true
|
||||
}
|
||||
trap cleanup EXIT
|
||||
|
||||
cat >/tmp/kind-config.yaml <<'EOF'
|
||||
kind: Cluster
|
||||
apiVersion: kind.x-k8s.io/v1alpha4
|
||||
name: nested
|
||||
nodes:
|
||||
- role: control-plane
|
||||
extraMounts:
|
||||
- hostPath: /dev/kmsg
|
||||
containerPath: /dev/kmsg
|
||||
EOF
|
||||
kind create cluster -v 9 --retain --config /tmp/kind-config.yaml --image "$KIND_NODE_IMAGE" --wait 5m
|
||||
docker exec nested-control-plane kubectl \
|
||||
--kubeconfig=/etc/kubernetes/admin.conf wait \
|
||||
--for=condition=Ready node/nested-control-plane --timeout=2m
|
||||
docker exec nested-control-plane kubectl \
|
||||
--kubeconfig=/etc/kubernetes/admin.conf run smoke \
|
||||
--image=docker.io/library/busybox:1.37 --restart=Never \
|
||||
--command -- sh -c 'echo kind-on-kata-ok'
|
||||
docker exec nested-control-plane kubectl \
|
||||
--kubeconfig=/etc/kubernetes/admin.conf wait \
|
||||
--for=jsonpath='{.status.phase}'=Succeeded pod/smoke --timeout=2m
|
||||
test "$(docker exec nested-control-plane kubectl \
|
||||
--kubeconfig=/etc/kubernetes/admin.conf logs smoke)" = kind-on-kata-ok
|
||||
kind delete cluster --name nested
|
||||
trap - EXIT
|
||||
@@ -18,13 +18,3 @@ OCI digest 固定镜像。
|
||||
|
||||
Flux 等待 ExternalSecret 和 Deployment 就绪；任何前置缺失都会使该
|
||||
Kustomization 保持 NotReady，而不会回退到明文 Secret。
|
||||
|
||||
## 镜像更新
|
||||
|
||||
`panxiao81/backstage` 的 CI 在 main push 后只推送 `:latest`。
|
||||
`platform/flux-image-automation` 跟踪 `latest` 背后的 digest，由 `flux-bot`
|
||||
直接提交到本仓库 main，不经过 PR。`deployment.yaml` 中 image 行的
|
||||
`$imagepolicy` 注释是 setter 标记，删除后自动更新会静默停止。
|
||||
|
||||
旧 digest 失去 tag 后会被 zot GC 回收（24h），因此不要把 Deployment 手工固定到
|
||||
一个已不是 `latest` 的 digest 并长期保留。
|
||||
|
||||
@@ -27,7 +27,7 @@ spec:
|
||||
type: RuntimeDefault
|
||||
containers:
|
||||
- name: backstage
|
||||
image: zot.ad.ddupan.top/panxiao81/backstage:latest@sha256:da55be5c2f5c8b33de2d87ee0ef02edeacf52a349c8b19a5fac9be4152b2723d # {"$imagepolicy": "flux-system:backstage"}
|
||||
image: zot.ad.ddupan.top/panxiao81/backstage@sha256:e5a12550726f19a680bc7c40e2cc07cc624318f9279ee121814d293a006ef210
|
||||
imagePullPolicy: IfNotPresent
|
||||
env:
|
||||
- name: BACKSTAGE_BASE_URL
|
||||
|
||||
+6
-61
@@ -1,69 +1,14 @@
|
||||
# Hydra 与独立 IAM 登录接入
|
||||
# Hydra 与 OIDC Login/Consent PoC
|
||||
|
||||
本目录提供独立 Hydra 签发服务。当前分支将实验性 Hydra 登录入口接至 Spring `iam-login`
|
||||
开发实例，由其执行 AD 密码、WebAuthn 和授权确认；不改变 issuer、Gitea 登录源或数据库。
|
||||
旧 Go OIDC 适配器继续部署以便回退，Gitea 的直接 Authelia 登录源也保留。
|
||||
该切换已于 2026-10-01 经 PR #168 合并并由 Flux 应用；下面原有 Go/Authelia 说明保留为回退路径资料。
|
||||
本目录提供独立 Hydra 签发服务，以及一个薄的 **OIDC 上游适配器**。当前上游配置为
|
||||
Authelia；适配器不连接 LDAP，也不管理用户目录。Samba AD、密码和 MFA 继续由现有
|
||||
Authelia 链路负责。第一轮只接入人类和 Gitea，不实现 agent 动态授权。
|
||||
|
||||
```text
|
||||
Gitea → Hydra → iam-login（Tailscale 开发实例）→ Samba AD + WebAuthn
|
||||
Gitea → Hydra → OIDC Login/Consent → Authelia → Samba AD
|
||||
← OIDC ← 经验证的上游身份 ← OIDC callback
|
||||
```
|
||||
|
||||
## Spring 开发路径验收
|
||||
|
||||
源码版本：[iam-login 5f49a7c](https://git.ddupan.top/panxiao81/iam-login/commit/5f49a7c)。
|
||||
开发 UI 是 `https://laptop.tail7e769.ts.net:18082`，已有 Passkey RP ID 保持不变。
|
||||
Hydra 回调为 `/oauth2/start`、`/oauth2/consent`、`/oauth2/logout`；默认注销回到 `/signin`。
|
||||
浏览器需要 Tailscale 连通性；开发服务暂由本地 JVM 容器运行，并非生产 Native 部署。
|
||||
|
||||
开发服务通过 `kubectl -n hydra port-forward --address 127.0.0.1 service/hydra-admin 18445:4445`
|
||||
访问私有 Admin API。转发需在 Hydra 滚动更新后重连；验收运行器会循环重建该通道。
|
||||
该通道与本地容器是临时验收依赖，必须保持运行；不修改 NetworkPolicy、
|
||||
不新增 Admin HTTPRoute。临时运行配置与旧版回退备份仅保存在受限本地运行目录，不提交秘密。
|
||||
|
||||
验收前已只读核对目标 Gitea 账号的现有 Hydra 外部关联，并将旧 sub 显式绑定到 AD
|
||||
稳定主体；不按邮箱重建关联、不修改 Gitea 账号或仓库权限。客户端管理仅允许有效 MFA 且
|
||||
直接属于 AD Domain Admins 的用户；这是验收期明确指定的粗粒度管理组。
|
||||
|
||||
### 客户端管理 `/console`
|
||||
|
||||
iam-login 的客户端管理 API 只接受 Hydra 签发、带 `iam.clients.manage` scope 的 access token；
|
||||
`/console` 是调用该 API 的前端，在 Hydra 中登记为普通 **public** 客户端 `iam-admin-ui`。
|
||||
该 scope 只在 consent 时发给 `iam-admin-ui` 且直接属于管理组的用户，规则在 iam-login。
|
||||
|
||||
浏览器直接向 Hydra 换取 token，因此 `hydra.yaml` 为 public 端口开启 CORS，只允许开发实例
|
||||
origin；不放行 cookie 凭据。Gitea 等服务端客户端不受影响。
|
||||
|
||||
`iam-admin-ui` 无 secret，与 `gitea` 一样经 Admin 带外登记（它是 public 客户端，iam-login API
|
||||
看不到也删不掉它，恢复同样走此通道）：
|
||||
|
||||
```sh
|
||||
curl -fsS -X POST http://127.0.0.1:18445/admin/clients -H 'Content-Type: application/json' -d '{
|
||||
"client_id": "iam-admin-ui", "client_name": "IAM 管理",
|
||||
"redirect_uris": ["https://laptop.tail7e769.ts.net:18082/console/callback"],
|
||||
"grant_types": ["authorization_code"], "response_types": ["code"],
|
||||
"scope": "openid iam.clients.manage", "token_endpoint_auth_method": "none",
|
||||
"subject_type": "public", "metadata": {"iam_login_enabled": true}}'
|
||||
```
|
||||
|
||||
开发实例另需 `iam.clients.admin-ui-client-id=iam-admin-ui`；未配置时 `/console` 不提供，API
|
||||
无法获得可用 token（fail closed）。
|
||||
|
||||
从 Gitea `/user/oauth2/hydra` 发起，应进入新密码/Passkey 页，确认授权后返回原账号，核对
|
||||
仓库权限。当前 Gitea client 未登记 front/back-channel 或 post-logout 回调，不能声称 Gitea 会话会
|
||||
随 IAM 注销。应用的注销协议兼容性需单独验收。旧 `authelia` 登录源始终保留。
|
||||
|
||||
开发配置关闭 Boot 默认 LDAP health indicator：它未配置目录连接，不对应同用户 bind 的
|
||||
AD 仓储。总健康检查仍验证数据库，真实 AD 认证由本次人类登录验收，不将 readiness 当作
|
||||
AD 凭据有效性的证明。
|
||||
|
||||
Hydra 的 login/consent URL 是全局配置，本次影响全部经 Hydra 的登录请求（当前接入 Gitea），
|
||||
不影响直接走 Authelia 的应用。不要在无法访问 Tailscale 开发实例时合并此切换。
|
||||
验收失败时 revert 本次回调变更，恢复旧 Go 的 `/login`、`/consent`；不删除客户端、签名密钥、
|
||||
数据库、用户关联或 Passkey。最终 `auth.ddupan.top` 同源部署另开 PR，不在此次切换范围内。
|
||||
|
||||
## 原 Go/Authelia 路径与回退资料
|
||||
|
||||
目标入口：
|
||||
|
||||
- `https://hydra.ad.ddupan.top`：Hydra 公共 OAuth2/OIDC endpoint。
|
||||
|
||||
+2
-18
@@ -1,20 +1,6 @@
|
||||
serve:
|
||||
public:
|
||||
port: 4444
|
||||
# The iam-login /console admin UI is a public OIDC client that exchanges its code from the
|
||||
# browser, so only that origin may call the public endpoints cross-origin. Server-side
|
||||
# clients such as Gitea are unaffected by CORS.
|
||||
cors:
|
||||
enabled: true
|
||||
allowed_origins:
|
||||
- https://laptop.tail7e769.ts.net:18082
|
||||
allowed_methods:
|
||||
- GET
|
||||
- POST
|
||||
allowed_headers:
|
||||
- Authorization
|
||||
- Content-Type
|
||||
allow_credentials: false
|
||||
admin:
|
||||
port: 4445
|
||||
tls:
|
||||
@@ -26,10 +12,8 @@ urls:
|
||||
self:
|
||||
issuer: https://hydra.ad.ddupan.top
|
||||
public: https://hydra.ad.ddupan.top
|
||||
login: https://laptop.tail7e769.ts.net:18082/oauth2/start
|
||||
consent: https://laptop.tail7e769.ts.net:18082/oauth2/consent
|
||||
logout: https://laptop.tail7e769.ts.net:18082/oauth2/logout
|
||||
post_logout_redirect: https://laptop.tail7e769.ts.net:18082/signin
|
||||
login: https://hydra-login.ad.ddupan.top/login
|
||||
consent: https://hydra-login.ad.ddupan.top/consent
|
||||
ttl:
|
||||
access_token: 15m
|
||||
id_token: 15m
|
||||
|
||||
@@ -1,16 +0,0 @@
|
||||
apiVersion: kustomize.toolkit.fluxcd.io/v1
|
||||
kind: Kustomization
|
||||
metadata:
|
||||
name: flux-image-automation
|
||||
namespace: flux-system
|
||||
spec:
|
||||
dependsOn:
|
||||
- name: external-secrets
|
||||
interval: 10m
|
||||
path: ./platform/flux-image-automation
|
||||
prune: true
|
||||
sourceRef:
|
||||
kind: GitRepository
|
||||
name: flux-system
|
||||
timeout: 5m
|
||||
wait: true
|
||||
File diff suppressed because it is too large
Load Diff
@@ -8,7 +8,6 @@ resources:
|
||||
- apps/external-secrets.yaml
|
||||
- apps/gitea.yaml
|
||||
- apps/backstage.yaml
|
||||
- apps/flux-image-automation.yaml
|
||||
- apps/http-echo.yaml
|
||||
- apps/openebs.yaml
|
||||
- apps/nats.yaml
|
||||
|
||||
@@ -27,9 +27,6 @@ homelab_dns:
|
||||
- { zone: ad.ddupan.top, name: zot, type: A, values: [192.168.10.127] }
|
||||
- { zone: ad.ddupan.top, name: zot-push, type: A, values: [192.168.10.127] }
|
||||
|
||||
- { zone: ad.ddupan.top, name: pg-prod, type: A, values: [192.168.10.2] }
|
||||
- { zone: ad.ddupan.top, name: pg-dev, type: A, values: [192.168.10.127] }
|
||||
|
||||
split_horizon:
|
||||
# backends records the current adoption boundary. obj is deliberately not
|
||||
# emitted to CoreDNS yet, preserving the current pod resolver behaviour.
|
||||
|
||||
@@ -1,6 +0,0 @@
|
||||
.terraform/
|
||||
*.tfstate*
|
||||
*.tfplan
|
||||
*.tfvars
|
||||
!*.tfvars.example
|
||||
__pycache__/
|
||||
@@ -1,165 +0,0 @@
|
||||
# Homelab shared etcd
|
||||
|
||||
独立于 PostgreSQL 与 k3s 的三成员协调服务。Ansible 管主机、证书文件和 etcd 账号；
|
||||
独立 Terraform root 管现有 Bao PKI 下的签发角色与 policy，不管理 CA 或秘密值。
|
||||
旧 Ansible Vault 不在本次迁移范围。
|
||||
|
||||
状态：2026-09-25 已部署三成员 etcd，启用 Bao mTLS 和 RBAC；三个端点健康检查通过。
|
||||
PVE 两个 LXC 的 rootfs 已通过 `pct move-volume` 从 local-lvm 迁入 `pve-rg` SSD DRBD 池，
|
||||
保留容器与 etcd 数据。当前 PVE LXC 迁卷要求停止容器，维护入口按成员串行执行，回归健康后再处理下一个。
|
||||
|
||||
| 成员 | 承载 | 地址 | 资源 |
|
||||
| --- | --- | --- | --- |
|
||||
| etcd-laptop | laptop systemd | 192.168.10.127 | MemoryHigh 256 MiB / MemoryMax 384 MiB |
|
||||
| etcd-pve1 | pve1 新无特权 LXC 150 | 10.60.0.20 | 1 core / 1536 MiB / 8 GiB SSD DRBD |
|
||||
| etcd-pve2 | pve2 新无特权 LXC 151 | 10.60.0.21 | 1 core / 768 MiB / 8 GiB SSD DRBD |
|
||||
|
||||
资源限额是初始测试预算，不是已测容量。LXC 已增加 PG standby/备份的独立 HDD mp0 与内存上限，生产 PG standby 与备份仓库已启动。
|
||||
etcd 自身的 MemoryHigh/MemoryMax 仍为 256/384 MiB。labnet 依赖现有 VyOS 路由；切换 bare-metal 时复用服务角色，更换主机 inventory。
|
||||
|
||||
## 接入方式与认证
|
||||
|
||||
客户端使用三个 `https://地址:2379` endpoint，校验 Bao 中央 CA，出示独立客户端证书。
|
||||
成员间 2380 也开启 mTLS，额外限制 peer CN 为 `homelab-etcd-peer`。
|
||||
管理员证书 CN 为 `root`，只保存在受管成员 root 可读的文件中；对应 etcd root 用户不设置密码。
|
||||
服务进程只能读取自己的 server/peer 私钥，不能读取管理员私钥。
|
||||
|
||||
Patroni 走 etcd3 gateway，使用无 CN 的客户端证书及独立 username/password 完成 RBAC。
|
||||
实测带 CN 的证书会被 gateway 拒绝；管理员 CN=root 证书仅用于原生 etcdctl。
|
||||
成员另有无 CN 的 gateway 证书供内部 gateway→gRPC 连接及密码核对使用，不能将管理员证书复用到 HTTP API。首个消费者为
|
||||
`patroni-pg-prod`，只允许 `/homelab/patroni/pg-prod/`。其 Bao KV v2 路径为
|
||||
`kv/infra/etcd/consumers/patroni-pg-prod`，包含 username/password/prefix；不在本文复制实际值。
|
||||
配置 Patroni 时对应 `namespace: /homelab/patroni/` 与 `scope: pg-prod`。
|
||||
|
||||
第一次接入应由管理流程交付 CA、客户端证书及秘密引用，然后使用自己的身份对自己的 prefix
|
||||
执行一次 put/get/delete。跨 prefix 应被拒绝；不要把 root 证书复制给应用。
|
||||
生产消费者证书已签发，Patroni 已接入并验证主从自动切换。
|
||||
|
||||
## 安装与维护入口
|
||||
|
||||
需要 Ansible、community.crypto，以及目标机 Python/systemd。当前二进制固定为 etcd 3.7.2
|
||||
linux-amd64，并固定官方发布资产 SHA-256；版本升级必须单独评估兼容性、备份与 quorum。
|
||||
Terraform 使用已登录的 Vault 兼容 provider 身份；Ansible 读取控制端 `BAO_TOKEN` 环境变量，
|
||||
token 不下发目标机。`homelab-etcd-provisioner` policy 不自动绑定到现有身份，先由 Bao 管理流程授权。
|
||||
|
||||
```bash
|
||||
cd infrastructure/etcd/terraform
|
||||
terraform init
|
||||
terraform plan
|
||||
# 审阅计划后 apply；只新增此服务的 PKI roles/policies。
|
||||
|
||||
cd ../ansible
|
||||
ansible-galaxy collection install -r requirements.yml
|
||||
ansible-playbook lxc.yml --check
|
||||
ansible-playbook lxc.yml
|
||||
ansible-playbook site.yml
|
||||
ansible-playbook bootstrap-auth.yml -e etcd_bootstrap_auth=true
|
||||
ansible-playbook consumers.yml
|
||||
ansible-playbook verify.yml
|
||||
```
|
||||
|
||||
Terraform 使用现有 S3 tfstate bucket，独立 key `etcd/terraform.tfstate`，启用原生 lockfile；不复用 Bao 的 state。
|
||||
从仓库根目录运行 `python3 infrastructure/etcd/run.py terraform <子命令>` 或
|
||||
`python3 infrastructure/etcd/run.py ansible <playbook.yml>`，复用当前 Bao 会话并在内存中取得所需凭据。
|
||||
`lxc.yml` 是创建入口，不接管已有未知 VMID，也不自动调整运行中容器的网络与容量；配置漂移会报错。
|
||||
部署前核对模板与 `pve-rg` 存储可用性、IP 冲突、SSH host key、laptop sudo 和网络访问控制。
|
||||
数据库角色不会创建/删除这些 LXC 或 etcd 成员。
|
||||
|
||||
`site.yml` 管安装、CSR 本地生成、Bao 签发、配置和逐成员激活。全体健康检查通过才记录激活指纹；
|
||||
上次中断后重跑仍能识别尚未激活的磁盘配置。已有集群每个成员激活后检查健康，再处理下一个。
|
||||
首次建群先让三个服务启动，再检查 quorum。`bootstrap-auth.yml` 与日常收敛分开，认证已启用时不重建。
|
||||
首次认证启用前应限制客户端网络，不能把“已有中央 CA 签名”当作业务授权。
|
||||
|
||||
`consumers.yml` 首次生成 48 字符随机密码并以 KV v2 CAS=0 写入，之后只复用。
|
||||
元数据存在但数据被删除、Bao 403/超时、或 etcd 用户存在但秘密丢失时均失败，不自动换密码。
|
||||
并发 CAS 冲突会中止此次执行，重跑读取胜出的版本。既有额外角色或不同 prefix 权限也报错，
|
||||
不默默扩大权限。账号删除、prefix 迁移和密码轮换均不包含在普通收敛里。
|
||||
密码使用 stdin 送给 etcdctl，涉及秘密的任务 `no_log: true`；禁止用 `--diff` 打印未来消费者秘密文件。
|
||||
|
||||
证书有效期 60 天，`site.yml` 在剩余不足 14 天时续签；需要 Bao 的控制端身份。
|
||||
续签 IaC 已实现于 `terraform/renewal.tf`、`controller/` 和 `ansible/controller.yml`：
|
||||
独立 cert auth 挂载信任中央 CA，并限制 laptop 的 DNS SAN；用现有 peer 客户端证书登录，
|
||||
换取 10 分钟 token，仅允许签发 server/peer/admin/gateway 四类证书，不授予 KV 读取权限。
|
||||
控制端每日运行现有健康检查与串行部署，结束撤销 token；不保存长期 token，不依赖人工 OIDC 会话。
|
||||
控制程序安装为 root 管理的固定副本，以 panxiao81 身份复用现有 Ansible/SSH/sudo 环境。
|
||||
|
||||
**机器身份和 timer 已启用**（2026-09-25）：维护者恢复 OIDC 登录后，Terraform 新增三个
|
||||
续签资源，`controller.yml` 登录预检通过。首次实际运行三成员均 `changed=0`，服务 Result=success。
|
||||
Bao cert auth 需要非空 CN 作为 identity alias，不能使用无 CN 的 gateway 证书；因此登录使用
|
||||
同一主机已有 peer 证书，仍由 `allowed_dns_sans=etcd-laptop` 限制身份，未扩大到其他成员。
|
||||
手动触发 `systemctl start homelab-etcd-renew.service`，检查 journal 和
|
||||
`/var/lib/homelab-etcd-controller/last-success`；每日 timer 04:10 UTC 加随机延迟。
|
||||
本流程也续签自身登录所用 peer 证书；若超过有效期仍未修复，则需要管理身份重新签发。
|
||||
成员启动本身仅用本地证书，不实时依赖 Bao。
|
||||
|
||||
## 备份与故障处理
|
||||
|
||||
每个成员有每日一次的 `homelab-etcd-snapshot.timer`，快照保存在各自
|
||||
`/var/backups/homelab-etcd`，root-only，保留最近 3 个成功快照；snapshot status 验证成功才淘汰旧快照。
|
||||
首备可执行 `systemctl start homelab-etcd-snapshot.service`，用 journal 和 `etcdutl snapshot status`
|
||||
检查，不根据文件名推断备份成功。三处本地快照不等于异地备份。
|
||||
|
||||
quota 初始 256 MiB，按 1 小时保留进行自动 compaction；defrag 另择维护窗口逐成员做，
|
||||
不能三个成员同时执行。metrics 绑定 loopback 与成员内网地址的 2381 端口，独立 listener 不提供 KV API；
|
||||
与既有 LAN exporter 一样通过内网 HTTP 采集，禁止将端口映射到公网。
|
||||
现有 VictoriaMetrics 使用 `platform/observability/metrics/scrapes/shared-etcd.yaml` 采集，
|
||||
对应 VMRule 覆盖成员不可采集、少于两个可采集成员、无 leader、容量、WAL fsync 和频繁选举。
|
||||
采集失败不等于 quorum 丢失，需结合管理员 endpoint health 判断。
|
||||
备份年龄、证书到期与续签检查的主机采集已于 2026-09-27 部署，
|
||||
告警规则已随 [PR #166](https://git.ddupan.top/panxiao81/homelab-infra/pulls/166) 合并并由 Flux 接管；
|
||||
部署、阈值和排障见 [维护监控](../shared-data-monitoring/README.md)。既有 Alertmanager 已接入 Telegram，
|
||||
维护者已收到本次成员无 leader 与频繁选举通知；接收器配置由现有监控栈维护。
|
||||
|
||||
### DRBD I/O 故障导致成员根卷只读
|
||||
|
||||
`SharedEtcdNoLeader` 是单成员指标，不等于全体没有 leader。先检查三个 endpoint，
|
||||
并检查宿主机内核的 DRBD quorum、PingAck、I/O error 和 ext4 journal 日志。
|
||||
`findmnt` 显示 `rw,emergency_ro` 仍表示文件系统已因错误停止写入；DRBD 恢复 quorum 不会自动修复它。
|
||||
|
||||
2026-09-25 17:16 UTC 起，CT150 根卷发生 quorum 短暂丢失、journal I/O 错误和 emergency_ro，
|
||||
另外两个 etcd 成员保持健康。期间两个新 HDD 数据卷初始同步伴随多个 DRBD 资源 PingAck 超时。
|
||||
只对本项目两个新数据卷设置 `DrbdOptions/PeerDevice/c-max-rate=10240`（10 MiB/s），
|
||||
入口为 `../shared-postgresql/ansible/limit-resync.yml`，创建卷流程也复用该任务。
|
||||
限速后短期未见新增超时，尚不构成唯一根因证明；未调整全局协议、quorum 或超时。
|
||||
|
||||
恢复顺序：确认另外两个成员健康并保存快照，正常停止故障容器，确认根卷卸载、DRBD quorum
|
||||
和副本 UpToDate，再执行 `pct fsck 150 --device rootfs --force 1`。fsck 返回 1 表示已修复，
|
||||
PVE 包装命令仍会报非零；需再次检查返回 0 才启动容器。不能在已挂载卷上执行 fsck，不能直接强制 remount。
|
||||
本次修复后 CT150 启动，三个 endpoint 健康且 Raft term/index 一致；两个容器根卷与 mp0 均可写。
|
||||
频繁选举告警含 15 分钟历史窗口，应核对最新计数和健康状态，不通过关闭规则消除通知。
|
||||
|
||||
- 单成员故障：先确认其他两成员仍健康，修复原成员；普通部署不删数据目录。
|
||||
- 成员永久丢失：需单独执行 member remove/add 流程并同步 inventory，不用重新 init 覆盖。
|
||||
- 全集群丢失：先恢复 Bao/信任根与恢复凭据，隔离全部旧成员，用同一有效快照恢复新逻辑集群。
|
||||
开放客户端前逐个协调消费者状态。Patroni 必须核对真实主库、timeline、lease/DCS 状态；
|
||||
不因旧快照宣称某节点为主就允许它写入。完整跨服务灾难恢复演练尚未完成。
|
||||
|
||||
## 验证
|
||||
|
||||
```bash
|
||||
# 本机 loopback 三节点，临时测试 CA 与假的 Bao HTTP 服务，不接触线上秘密。
|
||||
ETCD_TEST_BIN=/path/to/etcd-v3.7.2-linux-amd64 python3 tests/integration.py
|
||||
cd terraform && terraform validate
|
||||
cd ../ansible && ansible-playbook --syntax-check site.yml
|
||||
```
|
||||
|
||||
集成测试验证实际模板启动、mTLS、认证初始化幂等、消费者幂等、CAS 首次创建、秘密缺失失败关闭、
|
||||
prefix 隔离、gateway 账号登录和单成员故障。假 Bao 不验证真实 PKI 签发或中央 policy 的权限，
|
||||
也不证明 PVE SSD DRBD 在业务负载下的延迟、systemd 内存预算、在线证书轮换和生产网络符合要求。
|
||||
|
||||
源码边界与 PostgreSQL 后续设计见 [研究记录](../shared-postgresql/RESEARCH.md)。
|
||||
|
||||
2026-09-25 验证结果：Terraform validate 通过；Ansible lint 零文件级问题；隔离三成员测试通过，
|
||||
认证初始化与消费者第二次执行均 `changed=0`，包含 gateway 写入、密码漂移拒绝、快照离线恢复、
|
||||
停止一成员后继续写入。测试低负载 RSS 约 36.6–40.4 MiB/成员，仅作开销参考。
|
||||
现场验证：Bao PKI roles/policies 已应用，三成员已签发证书并启动；认证初始化与
|
||||
`patroni-pg-prod` 账号创建成功，随机密码已保存 Bao，实际 gateway 登录验证通过。
|
||||
两台 LXC 的 rootfs 均为 `pve-rg`，运行状态正常；迁移后全部端点成功提交健康探测。
|
||||
现有监控已接入；自动续签已启用；备份/证书年龄告警于 2026-09-27 补齐并启用。PostgreSQL 部署状态见其 README。
|
||||
|
||||
监控接入验收（2026-09-25）：三个 `up{job="shared-etcd"}` 均为 1，六条规则 health=ok，
|
||||
三端点健康检查通过。新增续签调度四项失败/成功路径测试通过，Terraform validate 与 Ansible lint 通过，
|
||||
隔离三成员测试再次通过。[IaC PR #159](https://git.ddupan.top/panxiao81/homelab-infra/pulls/159) 已合并。
|
||||
2026-09-25 合并后核实：Flux observability Ready，应用版本 `f6d12d6`，
|
||||
etcd/PG 的两份 VMRule 与两份 VMStaticScrape 均已纳入 inventory，状态 operational；
|
||||
三个 etcd 与两个 Patroni 抓取目标均 up=1，相关 11 条告警 health=ok、inactive。备份/证书年龄告警不包含在这六条规则中。
|
||||
@@ -1,6 +0,0 @@
|
||||
[defaults]
|
||||
inventory = inventory/hosts.yml
|
||||
roles_path = roles
|
||||
retry_files_enabled = False
|
||||
host_key_checking = True
|
||||
interpreter_python = auto_silent
|
||||
@@ -1,60 +0,0 @@
|
||||
---
|
||||
# 与常规 site 分离，防止重建/恢复时默默创建新的认证域。
|
||||
- name: 初始化共享 etcd 认证
|
||||
hosts: etcd[0]
|
||||
become: true
|
||||
gather_facts: false
|
||||
pre_tasks:
|
||||
- name: 加载共享默认参数
|
||||
ansible.builtin.import_role:
|
||||
name: shared_etcd
|
||||
tasks_from: context
|
||||
environment:
|
||||
ETCDCTL_ENDPOINTS: "https://{{ etcd_address }}:{{ etcd_client_port }}"
|
||||
ETCDCTL_CACERT: "{{ etcd_config_dir }}/ca.crt"
|
||||
ETCDCTL_CERT: "{{ etcd_config_dir }}/admin.crt"
|
||||
ETCDCTL_KEY: "{{ etcd_config_dir }}/admin.key"
|
||||
tasks:
|
||||
- name: 读取认证状态
|
||||
ansible.builtin.command:
|
||||
argv: ["{{ etcd_install_dir }}/etcdctl", --write-out=json, auth, status]
|
||||
changed_when: false
|
||||
register: etcd_auth_status
|
||||
check_mode: false
|
||||
|
||||
- name: 初始化管理员与认证
|
||||
when: not ((etcd_auth_status.stdout | from_json).enabled | default(false))
|
||||
block:
|
||||
- name: 要求显式初始化参数
|
||||
ansible.builtin.assert:
|
||||
that: etcd_bootstrap_auth | default(false) | bool
|
||||
fail_msg: 首次初始化需要 -e etcd_bootstrap_auth=true；常规运行不能重建认证。
|
||||
|
||||
- name: 读取现有用户
|
||||
ansible.builtin.command:
|
||||
argv: ["{{ etcd_install_dir }}/etcdctl", --write-out=json, user, list]
|
||||
changed_when: false
|
||||
register: etcd_users
|
||||
|
||||
- name: 创建仅证书认证的 root 用户
|
||||
ansible.builtin.command:
|
||||
argv: ["{{ etcd_install_dir }}/etcdctl", user, add, root, --no-password]
|
||||
when: "'root' not in ((etcd_users.stdout | from_json).users | default([], true))"
|
||||
changed_when: true
|
||||
|
||||
- name: 读取管理员角色
|
||||
ansible.builtin.command:
|
||||
argv: ["{{ etcd_install_dir }}/etcdctl", --write-out=json, user, get, root]
|
||||
changed_when: false
|
||||
register: etcd_root_roles
|
||||
|
||||
- name: 授予 root 管理角色
|
||||
ansible.builtin.command:
|
||||
argv: ["{{ etcd_install_dir }}/etcdctl", user, grant-role, root, root]
|
||||
when: "'root' not in ((etcd_root_roles.stdout | from_json).roles | default([], true))"
|
||||
changed_when: true
|
||||
|
||||
- name: 开启认证
|
||||
ansible.builtin.command:
|
||||
argv: ["{{ etcd_install_dir }}/etcdctl", auth, enable]
|
||||
changed_when: true
|
||||
@@ -1,37 +0,0 @@
|
||||
---
|
||||
- name: 收敛消费者账号及独立 prefix
|
||||
hosts: etcd[0]
|
||||
become: true
|
||||
gather_facts: false
|
||||
pre_tasks:
|
||||
- name: 加载共享默认参数
|
||||
ansible.builtin.import_role:
|
||||
name: shared_etcd
|
||||
tasks_from: context
|
||||
environment:
|
||||
ETCDCTL_ENDPOINTS: "https://{{ etcd_address }}:{{ etcd_client_port }}"
|
||||
ETCDCTL_CACERT: "{{ etcd_config_dir }}/ca.crt"
|
||||
ETCDCTL_CERT: "{{ etcd_config_dir }}/admin.crt"
|
||||
ETCDCTL_KEY: "{{ etcd_config_dir }}/admin.key"
|
||||
tasks:
|
||||
- name: 检查认证状态
|
||||
ansible.builtin.command:
|
||||
argv: ["{{ etcd_install_dir }}/etcdctl", --write-out=json, auth, status]
|
||||
register: etcd_auth_status
|
||||
changed_when: false
|
||||
check_mode: false
|
||||
|
||||
- name: 要求认证已启用
|
||||
ansible.builtin.assert:
|
||||
that:
|
||||
- ((etcd_auth_status.stdout | from_json).enabled | default(false))
|
||||
- etcd_consumers | map(attribute='name') | unique | length == etcd_consumers | length
|
||||
- etcd_bao_token | length > 0
|
||||
no_log: true
|
||||
|
||||
- name: 逐个收敛消费者
|
||||
ansible.builtin.include_tasks: tasks/consumer.yml
|
||||
loop: "{{ etcd_consumers }}"
|
||||
loop_control:
|
||||
loop_var: etcd_consumer
|
||||
label: "{{ etcd_consumer.name }}"
|
||||
@@ -1,74 +0,0 @@
|
||||
---
|
||||
# 仅在 Terraform 身份配置完成后部署；预检失败不启用定时器。
|
||||
- name: 安装独立于人工会话的续签调度
|
||||
hosts: etcd-laptop
|
||||
become: true
|
||||
gather_facts: false
|
||||
tasks:
|
||||
- name: 创建 root 管理的程序目录
|
||||
ansible.builtin.file:
|
||||
path: /opt/homelab-etcd-controller
|
||||
state: directory
|
||||
mode: '0755'
|
||||
- name: 安装证书登录与调度程序
|
||||
ansible.builtin.copy:
|
||||
src: "../controller/{{ item }}"
|
||||
dest: "/opt/homelab-etcd-controller/{{ item }}"
|
||||
owner: root
|
||||
group: root
|
||||
mode: '0755'
|
||||
loop: [login.py, renew.py]
|
||||
- name: 以机器身份预检（不输出 token）
|
||||
ansible.builtin.command:
|
||||
argv: [/usr/bin/python3, /opt/homelab-etcd-controller/login.py]
|
||||
changed_when: false
|
||||
no_log: true
|
||||
- name: 安装固定的 Ansible 配置副本
|
||||
ansible.builtin.copy:
|
||||
src: "{{ playbook_dir }}/"
|
||||
dest: /opt/homelab-etcd-controller/ansible/
|
||||
owner: root
|
||||
group: root
|
||||
mode: preserve
|
||||
- name: 创建调度状态目录
|
||||
ansible.builtin.file:
|
||||
path: /var/lib/homelab-etcd-controller
|
||||
state: directory
|
||||
owner: panxiao81
|
||||
group: panxiao81
|
||||
mode: '0700'
|
||||
- name: 安装续签 oneshot unit
|
||||
ansible.builtin.copy:
|
||||
dest: /etc/systemd/system/homelab-etcd-renew.service
|
||||
mode: '0644'
|
||||
content: |
|
||||
[Unit]
|
||||
Description=Renew shared etcd certificates through Bao machine identity
|
||||
After=network-online.target
|
||||
[Service]
|
||||
Type=oneshot
|
||||
User=panxiao81
|
||||
Environment=HOME=/home/panxiao81
|
||||
Environment=PATH=/home/panxiao81/.local/bin:/usr/local/bin:/usr/bin:/bin
|
||||
ExecStart=/usr/bin/python3 /opt/homelab-etcd-controller/renew.py
|
||||
TimeoutStartSec=30min
|
||||
UMask=0077
|
||||
- name: 安装每日续签 timer
|
||||
ansible.builtin.copy:
|
||||
dest: /etc/systemd/system/homelab-etcd-renew.timer
|
||||
mode: '0644'
|
||||
content: |
|
||||
[Unit]
|
||||
Description=Daily shared etcd certificate renewal check
|
||||
[Timer]
|
||||
OnCalendar=*-*-* 04:10:00 UTC
|
||||
RandomizedDelaySec=15min
|
||||
Persistent=true
|
||||
[Install]
|
||||
WantedBy=timers.target
|
||||
- name: 启用续签调度
|
||||
ansible.builtin.systemd_service:
|
||||
name: homelab-etcd-renew.timer
|
||||
daemon_reload: true
|
||||
enabled: true
|
||||
state: started
|
||||
@@ -1,20 +0,0 @@
|
||||
---
|
||||
etcd_bao_url: https://bao.ad.ddupan.top:8200
|
||||
etcd_bao_pki_mount: pki
|
||||
etcd_bao_kv_mount: kv
|
||||
etcd_bao_secret_base: infra/etcd/consumers
|
||||
# 只在控制端使用，不下发 Bao token。也不读取或复制历史 .vault_pass。
|
||||
etcd_bao_token: "{{ lookup('env', 'BAO_TOKEN') }}"
|
||||
etcd_consumers:
|
||||
- name: patroni-pg-prod
|
||||
prefix: /homelab/patroni/pg-prod/
|
||||
etcd_lxc_template: laptop:vztmpl/ubuntu-24.04-standard_24.04-2_amd64.tar.zst
|
||||
etcd_lxc_storage: pve-rg
|
||||
etcd_lxc_bridge: labnet
|
||||
etcd_lxc_gateway: 10.60.0.1
|
||||
etcd_lxc_memory: 512
|
||||
etcd_lxc_disk_gb: 8
|
||||
etcd_lxc_pubkey: "{{ lookup('file', '~/.ssh/id_ed25519.pub') }}"
|
||||
|
||||
# 与现有 LAN exporter 相同的内网采集边界；禁止映射到公网。
|
||||
etcd_metrics_urls: "http://127.0.0.1:2381,http://{{ etcd_address }}:2381"
|
||||
@@ -1,33 +0,0 @@
|
||||
---
|
||||
# 已部署成员；共置 PG standby/备份的承载预算，数据库数据用独立 HDD mp0。
|
||||
all:
|
||||
vars:
|
||||
ansible_user: root
|
||||
children:
|
||||
etcd:
|
||||
hosts:
|
||||
etcd-laptop:
|
||||
ansible_connection: local
|
||||
ansible_host: 192.168.10.127
|
||||
ansible_user: panxiao81
|
||||
etcd_address: 192.168.10.127
|
||||
etcd-pve1:
|
||||
ansible_host: 10.60.0.20
|
||||
etcd_address: 10.60.0.20
|
||||
etcd-pve2:
|
||||
ansible_host: 10.60.0.21
|
||||
etcd_address: 10.60.0.21
|
||||
etcd_pve:
|
||||
hosts:
|
||||
pve1:
|
||||
ansible_host: 192.168.10.4
|
||||
etcd_lxc_vmid: 150
|
||||
etcd_lxc_hostname: etcd-pve1
|
||||
etcd_lxc_address: 10.60.0.20/24
|
||||
etcd_lxc_memory: 1536
|
||||
pve2:
|
||||
ansible_host: 192.168.10.7
|
||||
etcd_lxc_vmid: 151
|
||||
etcd_lxc_hostname: etcd-pve2
|
||||
etcd_lxc_address: 10.60.0.21/24
|
||||
etcd_lxc_memory: 768
|
||||
@@ -1,126 +0,0 @@
|
||||
---
|
||||
# 只创建已声明且尚不存在的 LXC；不接管未知 VMID，不重启现有容器。
|
||||
- name: 创建独立 etcd LXC
|
||||
hosts: etcd_pve
|
||||
become: true
|
||||
gather_facts: false
|
||||
tasks:
|
||||
- name: 读取全局资源，避免 VMID 在其他节点已占用
|
||||
ansible.builtin.command:
|
||||
argv: [pvesh, get, /cluster/resources, --type, vm, --output-format, json]
|
||||
register: etcd_pve_resources
|
||||
changed_when: false
|
||||
check_mode: false
|
||||
|
||||
- name: 保存同号资源
|
||||
ansible.builtin.set_fact:
|
||||
etcd_lxc_existing: >-
|
||||
{{ etcd_pve_resources.stdout
|
||||
| from_json
|
||||
| selectattr('vmid', 'equalto', etcd_lxc_vmid)
|
||||
| list }}
|
||||
|
||||
- name: 拒绝接管未知资源
|
||||
ansible.builtin.assert:
|
||||
that:
|
||||
- >-
|
||||
etcd_lxc_existing | length == 0 or
|
||||
(etcd_lxc_existing[0].type == 'lxc' and etcd_lxc_existing[0].node == inventory_hostname
|
||||
and etcd_lxc_existing[0].name == etcd_lxc_hostname
|
||||
and 'shared-etcd' in (etcd_lxc_existing[0].tags | default('')))
|
||||
|
||||
- name: 新建无特权 LXC
|
||||
when: etcd_lxc_existing | length == 0 and not ansible_check_mode
|
||||
block:
|
||||
- name: 暂存 SSH 公钥
|
||||
ansible.builtin.copy:
|
||||
content: "{{ etcd_lxc_pubkey }}\n"
|
||||
dest: /run/shared-etcd-bootstrap.pub
|
||||
mode: '0600'
|
||||
- name: 创建声明的容器
|
||||
ansible.builtin.command:
|
||||
argv:
|
||||
- pct
|
||||
- create
|
||||
- "{{ etcd_lxc_vmid }}"
|
||||
- "{{ etcd_lxc_template }}"
|
||||
- --hostname
|
||||
- "{{ etcd_lxc_hostname }}"
|
||||
- --unprivileged
|
||||
- '1'
|
||||
- --cores
|
||||
- '1'
|
||||
- --memory
|
||||
- "{{ etcd_lxc_memory }}"
|
||||
- --swap
|
||||
- '0'
|
||||
- --rootfs
|
||||
- "{{ etcd_lxc_storage }}:{{ etcd_lxc_disk_gb }}"
|
||||
- --net0
|
||||
- "name=eth0,bridge={{ etcd_lxc_bridge }},ip={{ etcd_lxc_address }},gw={{ etcd_lxc_gateway }},type=veth"
|
||||
- --nameserver
|
||||
- 192.168.10.5
|
||||
- --searchdomain
|
||||
- ad.ddupan.top
|
||||
- --ssh-public-keys
|
||||
- /run/shared-etcd-bootstrap.pub
|
||||
- --onboot
|
||||
- '1'
|
||||
- --tags
|
||||
- ansible;shared-etcd
|
||||
changed_when: true
|
||||
always:
|
||||
- name: 删除暂存公钥
|
||||
ansible.builtin.file:
|
||||
path: /run/shared-etcd-bootstrap.pub
|
||||
state: absent
|
||||
|
||||
- name: 读取容器配置
|
||||
ansible.builtin.command:
|
||||
argv: [pct, config, "{{ etcd_lxc_vmid }}"]
|
||||
changed_when: false
|
||||
register: etcd_lxc_config
|
||||
when: not ansible_check_mode or etcd_lxc_existing | length > 0
|
||||
|
||||
- name: 配置漂移先报错，不直接改运行中的网络/资源
|
||||
ansible.builtin.assert:
|
||||
that:
|
||||
- >-
|
||||
('ip=' ~ etcd_lxc_address ~ ',') in etcd_lxc_config.stdout or
|
||||
('ip=' ~ etcd_lxc_address ~ '\n') in etcd_lxc_config.stdout
|
||||
- "('bridge=' ~ etcd_lxc_bridge ~ ',') in etcd_lxc_config.stdout"
|
||||
- "'unprivileged: 1' in etcd_lxc_config.stdout"
|
||||
- "('memory: ' ~ etcd_lxc_memory) in etcd_lxc_config.stdout"
|
||||
- "('rootfs: ' ~ etcd_lxc_storage ~ ':') in etcd_lxc_config.stdout"
|
||||
when: etcd_lxc_config is not skipped
|
||||
|
||||
- name: 读取容器运行状态
|
||||
ansible.builtin.command:
|
||||
argv: [pct, status, "{{ etcd_lxc_vmid }}"]
|
||||
changed_when: false
|
||||
register: etcd_lxc_status
|
||||
when: not ansible_check_mode
|
||||
|
||||
- name: 启动容器
|
||||
ansible.builtin.command:
|
||||
argv: [pct, start, "{{ etcd_lxc_vmid }}"]
|
||||
changed_when: true
|
||||
when: not ansible_check_mode and 'running' not in etcd_lxc_status.stdout
|
||||
|
||||
- name: 通过可信宿主机取得容器 SSH 公钥
|
||||
ansible.builtin.command:
|
||||
argv: [pct, exec, "{{ etcd_lxc_vmid }}", --, cat, /etc/ssh/ssh_host_ed25519_key.pub]
|
||||
register: etcd_lxc_hostkey
|
||||
changed_when: false
|
||||
retries: 12
|
||||
delay: 5
|
||||
until: etcd_lxc_hostkey.rc == 0
|
||||
when: not ansible_check_mode
|
||||
|
||||
- name: 保存经宿主机验证的 SSH host key
|
||||
ansible.builtin.known_hosts:
|
||||
name: "{{ etcd_lxc_address.split('/')[0] }}"
|
||||
key: "{{ etcd_lxc_address.split('/')[0] }} {{ etcd_lxc_hostkey.stdout }}"
|
||||
delegate_to: localhost
|
||||
become: false
|
||||
when: not ansible_check_mode
|
||||
@@ -1,90 +0,0 @@
|
||||
---
|
||||
# 首次部署承载调整，逐节点迁移；普通 lxc.yml 不自动移动磁盘。
|
||||
- name: 逐个将新 etcd 容器迁入声明的 SSD 池
|
||||
hosts: etcd_pve
|
||||
become: true
|
||||
gather_facts: false
|
||||
serial: 1
|
||||
any_errors_fatal: true
|
||||
vars:
|
||||
etcd_move_health_command:
|
||||
- /opt/homelab-etcd/etcdctl
|
||||
- --endpoints=https://192.168.10.127:2379,https://10.60.0.20:2379,https://10.60.0.21:2379
|
||||
- --cacert=/etc/homelab-etcd/ca.crt
|
||||
- --cert=/etc/homelab-etcd/admin.crt
|
||||
- --key=/etc/homelab-etcd/admin.key
|
||||
- endpoint
|
||||
- health
|
||||
tasks:
|
||||
- name: 核对容器配置
|
||||
ansible.builtin.command:
|
||||
argv: [pct, config, "{{ etcd_lxc_vmid }}"]
|
||||
register: etcd_move_config
|
||||
changed_when: false
|
||||
check_mode: false
|
||||
|
||||
- name: 限定本项目新建容器与允许的源池
|
||||
ansible.builtin.assert:
|
||||
that:
|
||||
- etcd_lxc_vmid in [150, 151]
|
||||
- "('hostname: ' ~ etcd_lxc_hostname) in etcd_move_config.stdout"
|
||||
- "'shared-etcd' in etcd_move_config.stdout"
|
||||
- "'rootfs: local-lvm:' in etcd_move_config.stdout or 'rootfs: pve-rg:' in etcd_move_config.stdout"
|
||||
- etcd_lxc_storage == 'pve-rg'
|
||||
|
||||
- name: 验证迁移前全部端点健康
|
||||
ansible.builtin.command:
|
||||
argv: "{{ etcd_move_health_command }}"
|
||||
delegate_to: localhost
|
||||
changed_when: false
|
||||
check_mode: false
|
||||
|
||||
- name: 迁移当前尚在本地池的根卷
|
||||
when: "'rootfs: local-lvm:' in etcd_move_config.stdout and not ansible_check_mode"
|
||||
block:
|
||||
- name: 创建已验证的集群快照
|
||||
ansible.builtin.command:
|
||||
argv: [systemctl, start, homelab-etcd-snapshot.service]
|
||||
delegate_to: localhost
|
||||
changed_when: true
|
||||
|
||||
- name: 正常关闭一个容器
|
||||
ansible.builtin.command:
|
||||
argv: [pct, shutdown, "{{ etcd_lxc_vmid }}", --timeout, '60']
|
||||
changed_when: true
|
||||
|
||||
- name: 复制成功后移除该新建容器的原卷
|
||||
ansible.builtin.command:
|
||||
argv:
|
||||
- pct
|
||||
- move-volume
|
||||
- "{{ etcd_lxc_vmid }}"
|
||||
- rootfs
|
||||
- "{{ etcd_lxc_storage }}"
|
||||
- --delete
|
||||
- '1'
|
||||
- --bwlimit
|
||||
- '32768'
|
||||
changed_when: true
|
||||
always:
|
||||
- name: 核对容器运行状态
|
||||
ansible.builtin.command:
|
||||
argv: [pct, status, "{{ etcd_lxc_vmid }}"]
|
||||
register: etcd_move_status
|
||||
changed_when: false
|
||||
- name: 重新启动容器
|
||||
ansible.builtin.command:
|
||||
argv: [pct, start, "{{ etcd_lxc_vmid }}"]
|
||||
when: "'running' not in etcd_move_status.stdout"
|
||||
changed_when: true
|
||||
|
||||
- name: 等待当前成员回归
|
||||
ansible.builtin.command:
|
||||
argv: "{{ etcd_move_health_command }}"
|
||||
delegate_to: localhost
|
||||
changed_when: false
|
||||
register: etcd_move_health
|
||||
retries: 18
|
||||
delay: 5
|
||||
until: etcd_move_health.rc == 0
|
||||
when: not ansible_check_mode
|
||||
@@ -1,4 +0,0 @@
|
||||
---
|
||||
collections:
|
||||
- name: community.crypto
|
||||
version: 3.2.1
|
||||
@@ -1,17 +0,0 @@
|
||||
---
|
||||
etcd_version: 3.7.2
|
||||
etcd_archive_checksum: sha256:3a3679bc51a4ee9d30bccea1da7cd4fe62c6fc1d2ca1255068d2c53bf3026135
|
||||
etcd_install_dir: /opt/homelab-etcd
|
||||
etcd_config_dir: /etc/homelab-etcd
|
||||
etcd_data_dir: /var/lib/homelab-etcd
|
||||
etcd_cluster_token: homelab-shared-etcd-v1
|
||||
etcd_client_port: 2379
|
||||
etcd_peer_port: 2380
|
||||
etcd_metrics_port: 2381
|
||||
etcd_quota_bytes: 268435456
|
||||
etcd_memory_high: 256M
|
||||
etcd_memory_max: 384M
|
||||
etcd_certificate_ttl: 1440h
|
||||
etcd_renew_before: +14d
|
||||
etcd_snapshot_dir: /var/backups/homelab-etcd
|
||||
etcd_snapshot_keep: 3
|
||||
@@ -1,93 +0,0 @@
|
||||
---
|
||||
- name: 在成员本地生成私钥
|
||||
community.crypto.openssl_privatekey:
|
||||
path: "{{ etcd_config_dir }}/{{ etcd_cert.name }}.key"
|
||||
type: ECC
|
||||
curve: secp256r1
|
||||
owner: root
|
||||
group: "{{ etcd_cert.group }}"
|
||||
mode: '0640'
|
||||
|
||||
- name: 本地生成 CSR
|
||||
community.crypto.openssl_csr:
|
||||
path: "{{ etcd_config_dir }}/{{ etcd_cert.name }}.csr"
|
||||
privatekey_path: "{{ etcd_config_dir }}/{{ etcd_cert.name }}.key"
|
||||
common_name: "{{ etcd_cert.cn | default(omit, true) }}"
|
||||
subject_alt_name: >-
|
||||
{{ ['IP:' ~ etcd_address, 'DNS:' ~ inventory_hostname] if etcd_cert.name in ['server', 'peer']
|
||||
else (['DNS:' ~ inventory_hostname] if etcd_cert.name == 'gateway' else []) }}
|
||||
use_common_name_for_san: false
|
||||
extended_key_usage: "{{ etcd_cert.eku }}"
|
||||
key_usage: [digitalSignature]
|
||||
mode: '0644'
|
||||
register: etcd_csr_state
|
||||
|
||||
- name: 检查已有证书
|
||||
ansible.builtin.stat:
|
||||
path: "{{ etcd_config_dir }}/{{ etcd_cert.name }}.crt"
|
||||
register: etcd_cert_file
|
||||
|
||||
- name: 检查续签窗口
|
||||
community.crypto.x509_certificate_info:
|
||||
path: "{{ etcd_config_dir }}/{{ etcd_cert.name }}.crt"
|
||||
valid_at:
|
||||
renewal: "{{ etcd_renew_before }}"
|
||||
register: etcd_cert_info
|
||||
when: etcd_cert_file.stat.exists
|
||||
|
||||
- name: 通过中央 CA 签发需更新的证书
|
||||
when: >-
|
||||
not etcd_cert_file.stat.exists or etcd_csr_state is changed or
|
||||
not (etcd_cert_info.valid_at.renewal | default(false))
|
||||
block:
|
||||
- name: 仅在确需签发时要求 Bao 凭据
|
||||
ansible.builtin.assert:
|
||||
that: etcd_bao_token | length > 0
|
||||
no_log: true
|
||||
|
||||
- name: 读取 CSR 公共内容
|
||||
ansible.builtin.slurp:
|
||||
src: "{{ etcd_config_dir }}/{{ etcd_cert.name }}.csr"
|
||||
register: etcd_csr
|
||||
|
||||
- name: 控制端提交 Bao 签名请求
|
||||
ansible.builtin.uri:
|
||||
url: "{{ etcd_bao_url }}/v1/{{ etcd_bao_pki_mount }}/sign/{{ etcd_cert.role }}"
|
||||
method: POST
|
||||
headers:
|
||||
X-Vault-Token: "{{ etcd_bao_token }}"
|
||||
body_format: json
|
||||
body:
|
||||
csr: "{{ etcd_csr.content | b64decode }}"
|
||||
ttl: "{{ etcd_certificate_ttl }}"
|
||||
status_code: 200
|
||||
delegate_to: localhost
|
||||
become: false
|
||||
register: etcd_signed
|
||||
no_log: true
|
||||
when: not ansible_check_mode
|
||||
|
||||
- name: 保存签发的证书链
|
||||
ansible.builtin.copy:
|
||||
content: |
|
||||
{{ etcd_signed.json.data.certificate }}
|
||||
{{ etcd_signed.json.data.ca_chain | join('\n') }}
|
||||
dest: "{{ etcd_config_dir }}/{{ etcd_cert.name }}.crt"
|
||||
owner: root
|
||||
group: "{{ etcd_cert.group }}"
|
||||
mode: '0644'
|
||||
when: not ansible_check_mode
|
||||
|
||||
- name: 保存中央 CA 信任链
|
||||
ansible.builtin.copy:
|
||||
content: |
|
||||
{{ etcd_signed.json.data.ca_chain | join('\n') }}
|
||||
dest: "{{ etcd_config_dir }}/ca.crt"
|
||||
owner: root
|
||||
group: homelab-etcd
|
||||
mode: '0644'
|
||||
when: not ansible_check_mode
|
||||
|
||||
- name: 标记证书已更新
|
||||
ansible.builtin.set_fact:
|
||||
etcd_certificates_changed: true
|
||||
@@ -1,3 +0,0 @@
|
||||
---
|
||||
# 只加载 role defaults，供运维入口复用。
|
||||
[]
|
||||
@@ -1,134 +0,0 @@
|
||||
---
|
||||
- name: 验证拓扑和签发配置
|
||||
ansible.builtin.assert:
|
||||
that:
|
||||
- groups['etcd'] | length == 3
|
||||
- groups['etcd'] | map('extract', hostvars, 'etcd_address') | unique | length == 3
|
||||
- ansible_facts['architecture'] == 'x86_64'
|
||||
- etcd_bao_url is match('^https://')
|
||||
- etcd_archive_checksum is match('^sha256:[a-f0-9]{64}$')
|
||||
no_log: true
|
||||
|
||||
- name: 安装证书处理依赖
|
||||
ansible.builtin.package:
|
||||
name: [python3-cryptography, openssl]
|
||||
state: present
|
||||
|
||||
- name: 创建独立 etcd 组
|
||||
ansible.builtin.group:
|
||||
name: homelab-etcd
|
||||
system: true
|
||||
|
||||
- name: 创建独立 etcd 用户
|
||||
ansible.builtin.user:
|
||||
name: homelab-etcd
|
||||
group: homelab-etcd
|
||||
system: true
|
||||
shell: /usr/sbin/nologin
|
||||
create_home: false
|
||||
|
||||
- name: 创建受管目录
|
||||
ansible.builtin.file:
|
||||
path: "{{ item.path }}"
|
||||
state: directory
|
||||
owner: "{{ item.owner }}"
|
||||
group: homelab-etcd
|
||||
mode: "{{ item.mode }}"
|
||||
loop:
|
||||
- {path: "{{ etcd_config_dir }}", owner: root, mode: '0750'}
|
||||
- {path: "{{ etcd_data_dir }}", owner: homelab-etcd, mode: '0700'}
|
||||
- {path: "{{ etcd_install_dir }}", owner: root, mode: '0755'}
|
||||
|
||||
- name: 下载固定版本及校验归档
|
||||
ansible.builtin.get_url:
|
||||
url: >-
|
||||
https://github.com/etcd-io/etcd/releases/download/v{{ etcd_version }}/etcd-v{{ etcd_version }}-linux-amd64.tar.gz
|
||||
dest: "{{ etcd_install_dir }}/etcd-v{{ etcd_version }}.tar.gz"
|
||||
checksum: "{{ etcd_archive_checksum }}"
|
||||
mode: '0644'
|
||||
register: etcd_download
|
||||
retries: 3
|
||||
delay: 5
|
||||
until: etcd_download is succeeded
|
||||
|
||||
- name: 展开固定版本
|
||||
ansible.builtin.unarchive:
|
||||
src: "{{ etcd_install_dir }}/etcd-v{{ etcd_version }}.tar.gz"
|
||||
dest: "{{ etcd_install_dir }}"
|
||||
remote_src: true
|
||||
creates: "{{ etcd_install_dir }}/etcd-v{{ etcd_version }}-linux-amd64/etcd"
|
||||
|
||||
- name: 安装版本链接
|
||||
ansible.builtin.file:
|
||||
src: "{{ etcd_install_dir }}/etcd-v{{ etcd_version }}-linux-amd64/{{ item }}"
|
||||
dest: "{{ etcd_install_dir }}/{{ item }}"
|
||||
state: link
|
||||
loop: [etcd, etcdctl, etcdutl]
|
||||
register: etcd_binary_links
|
||||
|
||||
- name: 签发成员与管理员证书
|
||||
ansible.builtin.include_tasks: certificate.yml
|
||||
loop:
|
||||
- {name: server, role: homelab-etcd-server, cn: "{{ inventory_hostname }}", eku: [serverAuth], group: homelab-etcd}
|
||||
- {name: peer, role: homelab-etcd-peer, cn: homelab-etcd-peer, eku: [serverAuth, clientAuth], group: homelab-etcd}
|
||||
- {name: gateway, role: homelab-etcd-gateway, cn: "", eku: [clientAuth], group: homelab-etcd}
|
||||
- {name: admin, role: homelab-etcd-admin, cn: root, eku: [clientAuth], group: root}
|
||||
loop_control:
|
||||
loop_var: etcd_cert
|
||||
|
||||
- name: 写入独立 etcd 配置
|
||||
ansible.builtin.template:
|
||||
src: etcd.yml.j2
|
||||
dest: "{{ etcd_config_dir }}/etcd.yml"
|
||||
owner: root
|
||||
group: homelab-etcd
|
||||
mode: '0640'
|
||||
register: etcd_config_file
|
||||
|
||||
- name: 写入独立 systemd unit
|
||||
ansible.builtin.template:
|
||||
src: homelab-etcd.service.j2
|
||||
dest: /etc/systemd/system/homelab-etcd.service
|
||||
mode: '0644'
|
||||
register: etcd_unit
|
||||
|
||||
# 跨失败重跑记录激活状态，防止上一轮写文件后中断导致漏掉必要重启。
|
||||
- name: 计算受管文件校验和
|
||||
ansible.builtin.stat:
|
||||
path: "{{ item }}"
|
||||
checksum_algorithm: sha256
|
||||
loop:
|
||||
- "{{ etcd_config_dir }}/etcd.yml"
|
||||
- "{{ etcd_config_dir }}/server.crt"
|
||||
- "{{ etcd_config_dir }}/peer.crt"
|
||||
- "{{ etcd_config_dir }}/gateway.crt"
|
||||
- "{{ etcd_config_dir }}/ca.crt"
|
||||
- /etc/systemd/system/homelab-etcd.service
|
||||
register: etcd_managed_files
|
||||
|
||||
- name: 计算期望激活指纹
|
||||
ansible.builtin.set_fact:
|
||||
etcd_config_fingerprint: >-
|
||||
{{ ((etcd_managed_files.results | map(attribute='stat.checksum') | list | join(':'))
|
||||
~ ':' ~ etcd_version) | hash('sha256') }}
|
||||
when: not ansible_check_mode
|
||||
|
||||
- name: 检查已激活指纹
|
||||
ansible.builtin.stat:
|
||||
path: "{{ etcd_config_dir }}/activated.sha256"
|
||||
register: etcd_activated_file
|
||||
|
||||
- name: 读取已激活指纹
|
||||
ansible.builtin.slurp:
|
||||
src: "{{ etcd_config_dir }}/activated.sha256"
|
||||
register: etcd_activated
|
||||
when: etcd_activated_file.stat.exists
|
||||
|
||||
- name: 判断是否需要滚动激活
|
||||
ansible.builtin.set_fact:
|
||||
etcd_config_changed: >-
|
||||
{{ not etcd_activated_file.stat.exists or
|
||||
(etcd_activated.content | default('') | b64decode | trim) != etcd_config_fingerprint | default('check-mode') }}
|
||||
|
||||
- name: 管理本地快照任务
|
||||
ansible.builtin.import_tasks: snapshot.yml
|
||||
@@ -1,26 +0,0 @@
|
||||
---
|
||||
- name: 创建仅 root 可访问的快照目录
|
||||
ansible.builtin.file:
|
||||
path: "{{ etcd_snapshot_dir }}"
|
||||
state: directory
|
||||
owner: root
|
||||
group: root
|
||||
mode: '0700'
|
||||
|
||||
- name: 写入快照脚本和 systemd 任务
|
||||
ansible.builtin.template:
|
||||
src: "{{ item.src }}"
|
||||
dest: "{{ item.dest }}"
|
||||
mode: "{{ item.mode }}"
|
||||
loop:
|
||||
- {src: snapshot.sh.j2, dest: "{{ etcd_install_dir }}/snapshot", mode: '0700'}
|
||||
- {src: snapshot.service.j2, dest: /etc/systemd/system/homelab-etcd-snapshot.service, mode: '0644'}
|
||||
- {src: snapshot.timer.j2, dest: /etc/systemd/system/homelab-etcd-snapshot.timer, mode: '0644'}
|
||||
register: etcd_snapshot_units
|
||||
|
||||
- name: 启用本地快照计划
|
||||
ansible.builtin.systemd_service:
|
||||
name: homelab-etcd-snapshot.timer
|
||||
daemon_reload: "{{ etcd_snapshot_units is changed }}"
|
||||
enabled: true
|
||||
state: started
|
||||
@@ -1,33 +0,0 @@
|
||||
# Ansible 管理；与 k3s、数据库生命周期独立。
|
||||
name: {{ inventory_hostname | to_json }}
|
||||
data-dir: {{ etcd_data_dir | to_json }}
|
||||
listen-client-urls: https://{{ etcd_address }}:{{ etcd_client_port }}
|
||||
advertise-client-urls: https://{{ etcd_address }}:{{ etcd_client_port }}
|
||||
listen-peer-urls: https://{{ etcd_address }}:{{ etcd_peer_port }}
|
||||
initial-advertise-peer-urls: https://{{ etcd_address }}:{{ etcd_peer_port }}
|
||||
initial-cluster: "{% for member in groups['etcd'] %}{{ member }}=https://{{ hostvars[member].etcd_address }}:{{ etcd_peer_port }}{{ ',' if not loop.last else '' }}{% endfor %}"
|
||||
initial-cluster-token: {{ etcd_cluster_token | to_json }}
|
||||
initial-cluster-state: new
|
||||
# 已存在的数据目录优先；成员替换必须走单独 runbook，不删除数据重建。
|
||||
client-transport-security:
|
||||
cert-file: {{ etcd_config_dir }}/server.crt
|
||||
key-file: {{ etcd_config_dir }}/server.key
|
||||
client-cert-file: {{ etcd_config_dir }}/gateway.crt
|
||||
client-key-file: {{ etcd_config_dir }}/gateway.key
|
||||
trusted-ca-file: {{ etcd_config_dir }}/ca.crt
|
||||
client-cert-auth: true
|
||||
peer-transport-security:
|
||||
cert-file: {{ etcd_config_dir }}/peer.crt
|
||||
key-file: {{ etcd_config_dir }}/peer.key
|
||||
trusted-ca-file: {{ etcd_config_dir }}/ca.crt
|
||||
client-cert-auth: true
|
||||
allowed-cn: [homelab-etcd-peer]
|
||||
# 独立 metrics listener 仅提供指标/健康，不开放 KV API；只绑定受管内网地址。
|
||||
listen-metrics-urls: {{ etcd_metrics_urls | default("http://127.0.0.1:" ~ etcd_metrics_port) | to_json }}
|
||||
quota-backend-bytes: {{ etcd_quota_bytes }}
|
||||
auto-compaction-mode: periodic
|
||||
auto-compaction-retention: '1h'
|
||||
heartbeat-interval: 100
|
||||
election-timeout: 1000
|
||||
logger: zap
|
||||
log-level: info
|
||||
@@ -1,23 +0,0 @@
|
||||
[Unit]
|
||||
Description=Homelab shared etcd
|
||||
Wants=network-online.target
|
||||
After=network-online.target
|
||||
|
||||
[Service]
|
||||
User=homelab-etcd
|
||||
Group=homelab-etcd
|
||||
ExecStart={{ etcd_install_dir }}/etcd --config-file={{ etcd_config_dir }}/etcd.yml
|
||||
Restart=on-failure
|
||||
RestartSec=5
|
||||
TimeoutStopSec=60
|
||||
MemoryHigh={{ etcd_memory_high }}
|
||||
MemoryMax={{ etcd_memory_max }}
|
||||
UMask=0077
|
||||
NoNewPrivileges=true
|
||||
ProtectSystem=strict
|
||||
ProtectHome=true
|
||||
PrivateTmp=true
|
||||
ReadWritePaths={{ etcd_data_dir }}
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
@@ -1,11 +0,0 @@
|
||||
[Unit]
|
||||
Description=Snapshot homelab shared etcd
|
||||
After=homelab-etcd.service
|
||||
|
||||
[Service]
|
||||
Type=oneshot
|
||||
ExecStart={{ etcd_install_dir }}/snapshot
|
||||
User=root
|
||||
UMask=0077
|
||||
TimeoutStartSec=300
|
||||
Nice=10
|
||||
@@ -1,20 +0,0 @@
|
||||
#!/bin/bash
|
||||
set -euo pipefail
|
||||
umask 077
|
||||
export ETCDCTL_ENDPOINTS="https://{{ etcd_address }}:{{ etcd_client_port }}"
|
||||
export ETCDCTL_CACERT="{{ etcd_config_dir }}/ca.crt"
|
||||
export ETCDCTL_CERT="{{ etcd_config_dir }}/admin.crt"
|
||||
export ETCDCTL_KEY="{{ etcd_config_dir }}/admin.key"
|
||||
repo="{{ etcd_snapshot_dir }}"
|
||||
exec 9>"$repo/.lock"
|
||||
flock -n 9 || exit 0
|
||||
output="$repo/$(date -u +%Y%m%dT%H%M%SZ).db"
|
||||
trap 'rm -f "$output.partial" "$output.partial.part"' EXIT
|
||||
{{ etcd_install_dir }}/etcdctl snapshot save "$output.partial"
|
||||
{{ etcd_install_dir }}/etcdutl snapshot status "$output.partial" >/dev/null
|
||||
mv "$output.partial" "$output"
|
||||
# 只有新快照成功且验证可读才清理旧备份。目录仅由此任务管理。
|
||||
mapfile -t snapshots < <(find "$repo" -maxdepth 1 -type f -name '????????T??????Z.db' -printf '%f\n' | sort -r)
|
||||
for old in "${snapshots[@]:{{ etcd_snapshot_keep }}}"; do
|
||||
rm -- "$repo/$old"
|
||||
done
|
||||
@@ -1,10 +0,0 @@
|
||||
[Unit]
|
||||
Description=Daily homelab etcd snapshot
|
||||
|
||||
[Timer]
|
||||
OnCalendar=*-*-* 03:20:00 UTC
|
||||
RandomizedDelaySec=300
|
||||
Persistent=true
|
||||
|
||||
[Install]
|
||||
WantedBy=timers.target
|
||||
@@ -1,66 +0,0 @@
|
||||
---
|
||||
# 先配置全体，再滚动启动：首次集群形成不能在第一个节点等待 quorum。
|
||||
- name: 配置 shared etcd 成员
|
||||
hosts: etcd
|
||||
become: true
|
||||
roles:
|
||||
- shared_etcd
|
||||
|
||||
- name: 滚动启动并验证 shared etcd
|
||||
hosts: etcd
|
||||
become: true
|
||||
serial: 1
|
||||
pre_tasks:
|
||||
- name: 加载共享默认参数
|
||||
ansible.builtin.import_role:
|
||||
name: shared_etcd
|
||||
tasks_from: context
|
||||
tasks:
|
||||
- name: 启动已配置的成员
|
||||
ansible.builtin.systemd_service:
|
||||
name: homelab-etcd
|
||||
enabled: true
|
||||
daemon_reload: true
|
||||
state: "{{ 'restarted' if etcd_config_changed | bool else 'started' }}"
|
||||
- name: 等待本机客户端端口
|
||||
ansible.builtin.wait_for:
|
||||
host: "{{ etcd_address }}"
|
||||
port: "{{ etcd_client_port }}"
|
||||
timeout: 60
|
||||
when: not ansible_check_mode
|
||||
|
||||
- name: 已有集群每次激活后等待本成员恢复 quorum 通信
|
||||
ansible.builtin.command:
|
||||
argv:
|
||||
- "{{ etcd_install_dir }}/etcdctl"
|
||||
- --endpoints=https://{{ etcd_address }}:{{ etcd_client_port }}
|
||||
- --cacert={{ etcd_config_dir }}/ca.crt
|
||||
- --cert={{ etcd_config_dir }}/admin.crt
|
||||
- --key={{ etcd_config_dir }}/admin.key
|
||||
- endpoint
|
||||
- health
|
||||
changed_when: false
|
||||
register: etcd_member_health
|
||||
retries: 12
|
||||
delay: 5
|
||||
until: etcd_member_health.rc == 0
|
||||
when: etcd_activated_file.stat.exists and not ansible_check_mode
|
||||
|
||||
- name: 核对全部成员
|
||||
ansible.builtin.import_playbook: verify.yml
|
||||
|
||||
- name: 记录已成功激活的配置
|
||||
hosts: etcd
|
||||
become: true
|
||||
pre_tasks:
|
||||
- name: 加载共享默认参数
|
||||
ansible.builtin.import_role:
|
||||
name: shared_etcd
|
||||
tasks_from: context
|
||||
tasks:
|
||||
- name: 写入激活指纹（全体健康检查通过后）
|
||||
ansible.builtin.copy:
|
||||
content: "{{ etcd_config_fingerprint }}\n"
|
||||
dest: "{{ etcd_config_dir }}/activated.sha256"
|
||||
mode: '0644'
|
||||
when: not ansible_check_mode
|
||||
@@ -1,170 +0,0 @@
|
||||
---
|
||||
- name: 验证消费者范围
|
||||
ansible.builtin.assert:
|
||||
that:
|
||||
- etcd_consumer.name is match('^[a-z][a-z0-9-]+$')
|
||||
- etcd_consumer.name != 'root'
|
||||
- etcd_consumer.prefix is match('^/homelab/[a-zA-Z0-9/_-]+/$')
|
||||
- etcd_consumer.prefix | length > 10
|
||||
|
||||
- name: 读取已有用户和角色
|
||||
ansible.builtin.command:
|
||||
argv: ["{{ etcd_install_dir }}/etcdctl", --write-out=json, "{{ item }}", list]
|
||||
loop: [user, role]
|
||||
register: etcd_identities
|
||||
changed_when: false
|
||||
check_mode: false
|
||||
|
||||
# 404 data 也可能代表被删除/销毁的旧秘密；只有 metadata 不存在才允许生成。
|
||||
- name: 控制端读取 Bao 秘密元数据
|
||||
ansible.builtin.uri:
|
||||
url: "{{ etcd_bao_url }}/v1/{{ etcd_bao_kv_mount }}/metadata/{{ etcd_bao_secret_base }}/{{ etcd_consumer.name }}"
|
||||
headers:
|
||||
X-Vault-Token: "{{ etcd_bao_token }}"
|
||||
status_code: [200, 404]
|
||||
delegate_to: localhost
|
||||
become: false
|
||||
no_log: true
|
||||
register: etcd_secret_metadata
|
||||
check_mode: false
|
||||
|
||||
- name: 禁止已有用户丢失秘密后自动换密码
|
||||
ansible.builtin.assert:
|
||||
that: >-
|
||||
etcd_secret_metadata.status == 200 or
|
||||
etcd_consumer.name not in ((etcd_identities.results[0].stdout | from_json).users | default([], true))
|
||||
fail_msg: etcd 用户已存在但 Bao 秘密缺失；需恢复原秘密或执行显式轮换。
|
||||
|
||||
- name: 首次创建随机秘密且禁止覆盖已有版本
|
||||
ansible.builtin.uri:
|
||||
url: "{{ etcd_bao_url }}/v1/{{ etcd_bao_kv_mount }}/data/{{ etcd_bao_secret_base }}/{{ etcd_consumer.name }}"
|
||||
method: POST
|
||||
headers:
|
||||
X-Vault-Token: "{{ etcd_bao_token }}"
|
||||
body_format: json
|
||||
body:
|
||||
options: {cas: 0}
|
||||
data:
|
||||
username: "{{ etcd_consumer.name }}"
|
||||
password: "{{ lookup('ansible.builtin.password', '/dev/null', length=48, chars=['ascii_letters', 'digits']) }}"
|
||||
prefix: "{{ etcd_consumer.prefix }}"
|
||||
status_code: 200
|
||||
delegate_to: localhost
|
||||
become: false
|
||||
no_log: true
|
||||
changed_when: true
|
||||
when:
|
||||
- etcd_secret_metadata.status == 404
|
||||
- not ansible_check_mode
|
||||
|
||||
- name: 控制端读取既有秘密
|
||||
ansible.builtin.uri:
|
||||
url: "{{ etcd_bao_url }}/v1/{{ etcd_bao_kv_mount }}/data/{{ etcd_bao_secret_base }}/{{ etcd_consumer.name }}"
|
||||
headers:
|
||||
X-Vault-Token: "{{ etcd_bao_token }}"
|
||||
status_code: 200
|
||||
delegate_to: localhost
|
||||
become: false
|
||||
register: etcd_consumer_secret
|
||||
no_log: true
|
||||
when: etcd_secret_metadata.status == 200 or not ansible_check_mode
|
||||
check_mode: false
|
||||
|
||||
- name: 核对已保存秘密归属
|
||||
ansible.builtin.assert:
|
||||
that:
|
||||
- etcd_consumer_secret.json.data.data.username == etcd_consumer.name
|
||||
- etcd_consumer_secret.json.data.data.prefix == etcd_consumer.prefix
|
||||
- etcd_consumer_secret.json.data.data.password | length >= 32
|
||||
no_log: true
|
||||
when: etcd_secret_metadata.status == 200 or not ansible_check_mode
|
||||
|
||||
- name: 新建消费者用户（密码只通过 stdin 传递）
|
||||
ansible.builtin.command:
|
||||
argv: ["{{ etcd_install_dir }}/etcdctl", user, add, "{{ etcd_consumer.name }}", --interactive=false]
|
||||
stdin: "{{ etcd_consumer_secret.json.data.data.password }}"
|
||||
no_log: true
|
||||
changed_when: true
|
||||
when:
|
||||
- etcd_consumer.name not in ((etcd_identities.results[0].stdout | from_json).users | default([], true))
|
||||
- not ansible_check_mode
|
||||
|
||||
- name: 新建消费者角色
|
||||
ansible.builtin.command:
|
||||
argv: ["{{ etcd_install_dir }}/etcdctl", role, add, "{{ etcd_consumer.name }}"]
|
||||
changed_when: true
|
||||
when: etcd_consumer.name not in ((etcd_identities.results[1].stdout | from_json).roles | default([], true))
|
||||
|
||||
- name: 读取角色权限
|
||||
ansible.builtin.command:
|
||||
argv: ["{{ etcd_install_dir }}/etcdctl", --write-out=json, role, get, "{{ etcd_consumer.name }}"]
|
||||
register: etcd_role_state
|
||||
changed_when: false
|
||||
when: >-
|
||||
not ansible_check_mode or
|
||||
etcd_consumer.name in ((etcd_identities.results[1].stdout | from_json).roles | default([], true))
|
||||
|
||||
- name: 拒绝不符合声明的既有权限（不自动扩大或删除）
|
||||
ansible.builtin.assert:
|
||||
that:
|
||||
- >-
|
||||
((etcd_role_state.stdout | from_json).perm | default([], true)) in
|
||||
[[], [{'permType': 2, 'key': etcd_consumer.prefix | b64encode,
|
||||
'range_end': (etcd_consumer.prefix[:-1] ~ '0') | b64encode}]]
|
||||
fail_msg: 既有角色权限与声明不同，请显式审查权限迁移。
|
||||
when: etcd_role_state is not skipped
|
||||
|
||||
- name: 授予唯一 prefix 读写权限
|
||||
ansible.builtin.command:
|
||||
argv:
|
||||
- "{{ etcd_install_dir }}/etcdctl"
|
||||
- role
|
||||
- grant-permission
|
||||
- "{{ etcd_consumer.name }}"
|
||||
- readwrite
|
||||
- "{{ etcd_consumer.prefix }}"
|
||||
- --prefix=true
|
||||
changed_when: true
|
||||
when:
|
||||
- etcd_role_state is not skipped
|
||||
- ((etcd_role_state.stdout | from_json).perm | default([], true)) | length == 0
|
||||
|
||||
- name: 读取用户角色
|
||||
ansible.builtin.command:
|
||||
argv: ["{{ etcd_install_dir }}/etcdctl", --write-out=json, user, get, "{{ etcd_consumer.name }}"]
|
||||
register: etcd_user_state
|
||||
changed_when: false
|
||||
when: >-
|
||||
not ansible_check_mode or
|
||||
etcd_consumer.name in ((etcd_identities.results[0].stdout | from_json).users | default([], true))
|
||||
|
||||
- name: 拒绝消费者已有额外角色
|
||||
ansible.builtin.assert:
|
||||
that: >-
|
||||
((etcd_user_state.stdout | from_json).roles | default([], true))
|
||||
| difference([etcd_consumer.name]) | length == 0
|
||||
when: etcd_user_state is not skipped
|
||||
|
||||
- name: 绑定消费者角色
|
||||
ansible.builtin.command:
|
||||
argv: ["{{ etcd_install_dir }}/etcdctl", user, grant-role, "{{ etcd_consumer.name }}", "{{ etcd_consumer.name }}"]
|
||||
changed_when: true
|
||||
when:
|
||||
- etcd_user_state is not skipped
|
||||
- etcd_consumer.name not in ((etcd_user_state.stdout | from_json).roles | default([], true))
|
||||
|
||||
# 使用 gateway 的真实密码登录来核对 Bao 与 etcd 一致性，不能只看用户已存在。
|
||||
- name: 验证消费者密码可经 gateway 登录
|
||||
ansible.builtin.uri:
|
||||
url: "https://{{ etcd_address }}:{{ etcd_client_port }}/v3/auth/authenticate"
|
||||
method: POST
|
||||
client_cert: "{{ etcd_config_dir }}/gateway.crt"
|
||||
client_key: "{{ etcd_config_dir }}/gateway.key"
|
||||
ca_path: "{{ etcd_config_dir }}/ca.crt"
|
||||
body_format: json
|
||||
body:
|
||||
name: "{{ etcd_consumer.name }}"
|
||||
password: "{{ etcd_consumer_secret.json.data.data.password }}"
|
||||
status_code: 200
|
||||
no_log: true
|
||||
when: not ansible_check_mode
|
||||
@@ -1,29 +0,0 @@
|
||||
---
|
||||
- name: 验证 shared etcd 全部端点
|
||||
hosts: etcd[0]
|
||||
become: true
|
||||
gather_facts: false
|
||||
pre_tasks:
|
||||
- name: 加载共享默认参数
|
||||
ansible.builtin.import_role:
|
||||
name: shared_etcd
|
||||
tasks_from: context
|
||||
tasks:
|
||||
- name: 通过管理员 mTLS 检查所有端点健康
|
||||
ansible.builtin.command:
|
||||
argv:
|
||||
- "{{ etcd_install_dir }}/etcdctl"
|
||||
- >-
|
||||
--endpoints={{ groups['etcd'] | map('extract', hostvars, 'etcd_address')
|
||||
| map('regex_replace', '^(.*)$', 'https://\1:' ~ etcd_client_port) | join(',') }}
|
||||
- --cacert={{ etcd_config_dir }}/ca.crt
|
||||
- --cert={{ etcd_config_dir }}/admin.crt
|
||||
- --key={{ etcd_config_dir }}/admin.key
|
||||
- endpoint
|
||||
- health
|
||||
changed_when: false
|
||||
register: etcd_health
|
||||
retries: 12
|
||||
delay: 5
|
||||
until: etcd_health.rc == 0
|
||||
when: not ansible_check_mode
|
||||
@@ -1,30 +0,0 @@
|
||||
#!/usr/bin/python3
|
||||
"""仅由受控调用者捕获 stdout；不得手动运行以免在终端输出短期 token。"""
|
||||
import json
|
||||
import ssl
|
||||
import sys
|
||||
import urllib.request
|
||||
|
||||
|
||||
def main():
|
||||
context = ssl.create_default_context()
|
||||
context.load_cert_chain('/etc/homelab-etcd/peer.crt', '/etc/homelab-etcd/peer.key')
|
||||
request = urllib.request.Request(
|
||||
'https://bao.ad.ddupan.top:8200/v1/auth/homelab-etcd-renewal/login',
|
||||
data=json.dumps({'name': 'etcd-laptop'}).encode(),
|
||||
headers={'Content-Type': 'application/json'}, method='POST',
|
||||
)
|
||||
with urllib.request.urlopen(request, context=context, timeout=30) as response:
|
||||
token = json.load(response)['auth']['client_token']
|
||||
if not isinstance(token, str) or not token:
|
||||
raise ValueError('empty token')
|
||||
sys.stdout.write(token)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
try:
|
||||
main()
|
||||
except Exception:
|
||||
# Bao 响应和异常对象可能带敏感内容，不写入 journal。
|
||||
sys.stderr.write('Bao certificate login failed\n')
|
||||
sys.exit(1)
|
||||
@@ -1,51 +0,0 @@
|
||||
#!/usr/bin/python3
|
||||
"""每日一次，机器证书换短期 token；使用现有串行 Ansible 收敛，不常驻 agent。"""
|
||||
import fcntl
|
||||
import os
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import sys
|
||||
import urllib.request
|
||||
|
||||
ROOT = Path('/opt/homelab-etcd-controller')
|
||||
|
||||
|
||||
def main():
|
||||
with open('/var/lib/homelab-etcd-controller/renew.lock', 'a') as lock:
|
||||
try:
|
||||
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
||||
except BlockingIOError:
|
||||
return 0
|
||||
login = subprocess.run(
|
||||
['sudo', '-n', '/usr/bin/python3', str(ROOT / 'login.py')],
|
||||
capture_output=True, text=True, timeout=40,
|
||||
)
|
||||
if login.returncode or not login.stdout.strip():
|
||||
print('Bao 机器证书登录失败；保留现有证书与运行中的 etcd。', file=sys.stderr)
|
||||
return 1
|
||||
token = login.stdout.strip()
|
||||
env = dict(os.environ, BAO_TOKEN=token)
|
||||
try:
|
||||
for play in ['verify.yml', 'site.yml']:
|
||||
result = subprocess.run(
|
||||
['/home/panxiao81/.local/bin/ansible-playbook', play],
|
||||
cwd=ROOT / 'ansible', env=env, timeout=750,
|
||||
)
|
||||
if result.returncode:
|
||||
return result.returncode
|
||||
Path('/var/lib/homelab-etcd-controller/last-success').touch()
|
||||
return 0
|
||||
finally:
|
||||
request = urllib.request.Request(
|
||||
'https://bao.ad.ddupan.top:8200/v1/auth/token/revoke-self',
|
||||
data=b'{}', headers={'X-Vault-Token': token}, method='POST',
|
||||
)
|
||||
try:
|
||||
with urllib.request.urlopen(request, timeout=15):
|
||||
pass
|
||||
except Exception:
|
||||
print('短期 token 撤销未确认，将由 TTL 自动失效。', file=sys.stderr)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
raise SystemExit(main())
|
||||
@@ -1,43 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""从当前 Bao 登录会话向子进程传递凭据，不写临时秘密文件或输出秘密。"""
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
ROOT = Path(__file__).resolve().parent
|
||||
|
||||
|
||||
def main():
|
||||
if len(sys.argv) < 3 or sys.argv[1] not in ('terraform', 'ansible'):
|
||||
raise SystemExit('用法：python3 run.py terraform <args> | ansible <playbook> [args]')
|
||||
env = dict(os.environ)
|
||||
env.setdefault('BAO_ADDR', 'https://bao.ad.ddupan.top:8200')
|
||||
token = env.get('BAO_TOKEN') or env.get('VAULT_TOKEN')
|
||||
if not token:
|
||||
token = Path('~/.vault-token').expanduser().read_text().strip()
|
||||
env['BAO_TOKEN'] = env['VAULT_TOKEN'] = token
|
||||
if sys.argv[1] == 'terraform':
|
||||
response = subprocess.run(
|
||||
['bao', 'kv', 'get', '-format=json', 'kv/k8s/seaweedfs-s3'],
|
||||
env=env, capture_output=True, text=True,
|
||||
)
|
||||
if response.returncode:
|
||||
raise SystemExit('读取 tfstate 受限身份失败；请检查 Bao 登录和授权。')
|
||||
config = json.loads(response.stdout)['data']['data']['seaweedfs_s3_config']
|
||||
config = json.loads(config) if isinstance(config, str) else config
|
||||
identities = [i for i in config['identities'] if i['name'] == 'terraform']
|
||||
if len(identities) != 1 or len(identities[0]['credentials']) != 1:
|
||||
raise SystemExit('tfstate 身份不唯一，拒绝猜测凭据。')
|
||||
credential = identities[0]['credentials'][0]
|
||||
env['AWS_ACCESS_KEY_ID'] = credential['accessKey']
|
||||
env['AWS_SECRET_ACCESS_KEY'] = credential['secretKey']
|
||||
command, cwd = ['terraform', *sys.argv[2:]], ROOT / 'terraform'
|
||||
else:
|
||||
command, cwd = ['ansible-playbook', *sys.argv[2:]], ROOT / 'ansible'
|
||||
raise SystemExit(subprocess.run(command, cwd=cwd, env=env).returncode)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
-22
@@ -1,22 +0,0 @@
|
||||
# This file is maintained automatically by "terraform init".
|
||||
# Manual edits may be lost in future updates.
|
||||
|
||||
provider "registry.terraform.io/hashicorp/vault" {
|
||||
version = "4.8.0"
|
||||
constraints = "~> 4.0"
|
||||
hashes = [
|
||||
"h1:aHqgWQhDBMeZO9iUKwJYMlh4q+xNMUlMIcjRbF4d02Y=",
|
||||
"zh:269ab13433f67684012ae7e15876532b0312f5d0d2002a9cf9febb1279ce5ea6",
|
||||
"zh:4babc95bf0c40eb85005db1dc2ca403c46be4a71dd3e409db3711a56f7a5ca0e",
|
||||
"zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3",
|
||||
"zh:86e27c1c625ecc24446a11eeffc3ac319b36c2b4e51251db8579256a0dbcf136",
|
||||
"zh:a32f31da94824009e26b077374440b52098aecb93c92ff55dc3d31dd37c4ea25",
|
||||
"zh:be0a18c6c0425518bab4fbffd82078b82036a88503b5d76064de551c9f646cbf",
|
||||
"zh:be5a77fdfd36863ebeec79cd12b1d13322ffad6821d157a0b279789fa06b5937",
|
||||
"zh:be8317d142a3caad74c7d936039ae27076a1b2b8312ef5208e2871a5f525977c",
|
||||
"zh:c94a84895a3d9954b80e983eed4603330a5cdbbd8eef5b3c99278c2d1402ef3c",
|
||||
"zh:de1fb712784dd8415f011ca5346a34f87fab6046c730557615247e511dbc7d98",
|
||||
"zh:e3eafae7da550f86cae395d6660b2a0e93ec8d2b0e0e5ef982ec762e961fc952",
|
||||
"zh:ff35fb1ab6add288f0f368981e56f780b50405accd1937131cba1137999c8d83",
|
||||
]
|
||||
}
|
||||
@@ -1,15 +0,0 @@
|
||||
# 与既有服务复用受限 tfstate 身份，使用独立对象与原生锁；不复用 Bao 的 state。
|
||||
terraform {
|
||||
backend "s3" {
|
||||
bucket = "tfstate"
|
||||
key = "etcd/terraform.tfstate"
|
||||
endpoints = { s3 = "https://s3.ad.ddupan.top" }
|
||||
region = "us-east-1"
|
||||
use_path_style = true
|
||||
skip_credentials_validation = true
|
||||
skip_metadata_api_check = true
|
||||
skip_region_validation = true
|
||||
skip_requesting_account_id = true
|
||||
use_lockfile = true
|
||||
}
|
||||
}
|
||||
@@ -1,93 +0,0 @@
|
||||
terraform {
|
||||
required_version = ">= 1.10"
|
||||
required_providers {
|
||||
vault = {
|
||||
source = "hashicorp/vault"
|
||||
version = "~> 4.0"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
provider "vault" {
|
||||
address = var.bao_address
|
||||
}
|
||||
|
||||
variable "bao_address" {
|
||||
type = string
|
||||
default = "https://bao.ad.ddupan.top:8200"
|
||||
}
|
||||
variable "pki_mount" {
|
||||
type = string
|
||||
default = "pki"
|
||||
}
|
||||
variable "kv_mount" {
|
||||
type = string
|
||||
default = "kv"
|
||||
}
|
||||
variable "member_names" {
|
||||
type = list(string)
|
||||
default = ["etcd-laptop", "etcd-pve1", "etcd-pve2"]
|
||||
}
|
||||
variable "consumer_names" {
|
||||
type = set(string)
|
||||
default = ["patroni-pg-prod"]
|
||||
}
|
||||
|
||||
# 仅管理既有 PKI 下的新 role/policy，不纳管 CA 私钥、mount 或秘密值。
|
||||
locals {
|
||||
certificate_roles = {
|
||||
server = { names = var.member_names, server = true, client = false, ips = true }
|
||||
peer = { names = concat(var.member_names, ["homelab-etcd-peer"]), server = true, client = true, ips = true }
|
||||
admin = { names = ["root"], server = false, client = true, ips = false }
|
||||
gateway = { names = var.member_names, server = false, client = true, ips = false }
|
||||
client = { names = [for name in var.consumer_names : "etcd-${name}"], server = false, client = true, ips = false }
|
||||
}
|
||||
}
|
||||
resource "vault_pki_secret_backend_role" "etcd" {
|
||||
for_each = local.certificate_roles
|
||||
backend = var.pki_mount
|
||||
name = "homelab-etcd-${each.key}"
|
||||
allowed_domains = each.value.names
|
||||
allow_bare_domains = true
|
||||
allow_subdomains = false
|
||||
allow_glob_domains = false
|
||||
allow_any_name = false
|
||||
allow_localhost = false
|
||||
allow_wildcard_certificates = false
|
||||
allow_ip_sans = each.value.ips
|
||||
server_flag = each.value.server
|
||||
client_flag = each.value.client
|
||||
key_type = "ec"
|
||||
key_bits = 256
|
||||
ttl = 5184000
|
||||
max_ttl = 5184000
|
||||
require_cn = !contains(["gateway", "client"], each.key)
|
||||
use_csr_common_name = true
|
||||
use_csr_sans = true
|
||||
}
|
||||
|
||||
resource "vault_policy" "etcd_provisioner" {
|
||||
name = "homelab-etcd-provisioner"
|
||||
policy = <<-EOT
|
||||
path "${var.pki_mount}/sign/homelab-etcd-*" {
|
||||
capabilities = ["update"]
|
||||
}
|
||||
path "${var.kv_mount}/metadata/infra/etcd/consumers/*" {
|
||||
capabilities = ["read"]
|
||||
}
|
||||
path "${var.kv_mount}/data/infra/etcd/consumers/*" {
|
||||
capabilities = ["create", "read", "update"]
|
||||
}
|
||||
EOT
|
||||
}
|
||||
|
||||
# 身份绑定沿用既有控制端认证方式，创建 policy 不自动授权任何身份。
|
||||
resource "vault_policy" "etcd_consumer" {
|
||||
for_each = var.consumer_names
|
||||
name = "homelab-etcd-${each.key}"
|
||||
policy = <<-EOT
|
||||
path "${var.kv_mount}/data/infra/etcd/consumers/${each.key}" {
|
||||
capabilities = ["read"]
|
||||
}
|
||||
EOT
|
||||
}
|
||||
@@ -1,28 +0,0 @@
|
||||
# 独立挂载：不接管全局认证，不保存长期 token，也不授权 KV 消费者秘密读取。
|
||||
data "vault_generic_secret" "etcd_ca" {
|
||||
path = "${var.pki_mount}/cert/ca"
|
||||
}
|
||||
|
||||
resource "vault_auth_backend" "etcd_renewal" {
|
||||
type = "cert"
|
||||
path = "homelab-etcd-renewal"
|
||||
}
|
||||
|
||||
resource "vault_policy" "etcd_renewal" {
|
||||
name = "homelab-etcd-renewal"
|
||||
policy = join("\n", concat([
|
||||
for kind in ["server", "peer", "admin", "gateway"] :
|
||||
"path \"${var.pki_mount}/sign/homelab-etcd-${kind}\" { capabilities = [\"update\"] }"
|
||||
], ["path \"auth/token/revoke-self\" { capabilities = [\"update\"] }"]))
|
||||
}
|
||||
|
||||
resource "vault_cert_auth_backend_role" "etcd_renewal" {
|
||||
backend = vault_auth_backend.etcd_renewal.path
|
||||
name = "etcd-laptop"
|
||||
certificate = data.vault_generic_secret.etcd_ca.data["certificate"]
|
||||
allowed_dns_sans = ["etcd-laptop"]
|
||||
token_policies = [vault_policy.etcd_renewal.name]
|
||||
token_no_default_policy = true
|
||||
token_ttl = 600
|
||||
token_max_ttl = 900
|
||||
}
|
||||
@@ -1,233 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""临时三成员 mTLS/RBAC 集成测试；仅绑定 loopback，不访问生产 Bao。"""
|
||||
import base64
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import re
|
||||
import shutil
|
||||
import ssl
|
||||
import subprocess
|
||||
import tempfile
|
||||
import threading
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
||||
|
||||
import jinja2
|
||||
import yaml
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
BIN = Path(os.environ.get('ETCD_TEST_BIN', '/tmp/etcd-v3.7.2-linux-amd64'))
|
||||
|
||||
|
||||
def run(argv, **kwargs):
|
||||
return subprocess.run([str(x) for x in argv], capture_output=True, text=True, check=True, **kwargs)
|
||||
|
||||
|
||||
class FakeBao(BaseHTTPRequestHandler):
|
||||
records = {}
|
||||
versions = {}
|
||||
writes = 0
|
||||
deny = False
|
||||
|
||||
def log_message(self, *_):
|
||||
pass
|
||||
|
||||
def respond(self, status, body):
|
||||
self.send_response(status)
|
||||
self.send_header('Content-Type', 'application/json')
|
||||
self.end_headers()
|
||||
self.wfile.write(json.dumps(body).encode())
|
||||
|
||||
def do_GET(self):
|
||||
if self.deny:
|
||||
return self.respond(403, {'errors': ['permission denied']})
|
||||
name = self.path.rsplit('/', 1)[-1]
|
||||
if '/metadata/' in self.path and name in self.versions:
|
||||
return self.respond(200, {'data': {'current_version': self.versions[name]}})
|
||||
if '/data/' in self.path and name in self.records:
|
||||
return self.respond(200, {'data': {'data': self.records[name]}})
|
||||
self.respond(404, {'errors': []})
|
||||
|
||||
def do_POST(self):
|
||||
body = json.loads(self.rfile.read(int(self.headers['Content-Length'])))
|
||||
name = self.path.rsplit('/', 1)[-1]
|
||||
if body['options']['cas'] != 0 or name in self.versions:
|
||||
return self.respond(400, {'errors': ['CAS mismatch']})
|
||||
self.records[name] = body['data']
|
||||
self.versions[name] = 1
|
||||
type(self).writes += 1
|
||||
self.respond(200, {'data': {'version': 1}})
|
||||
|
||||
|
||||
def main():
|
||||
for key in ('HTTP_PROXY', 'HTTPS_PROXY', 'ALL_PROXY', 'http_proxy', 'https_proxy', 'all_proxy'):
|
||||
os.environ.pop(key, None)
|
||||
os.environ['NO_PROXY'] = '*'
|
||||
processes = []
|
||||
handles = []
|
||||
server = None
|
||||
with tempfile.TemporaryDirectory(prefix='shared-etcd-test-') as work:
|
||||
w = Path(work)
|
||||
w.chmod(0o700)
|
||||
try:
|
||||
run(['openssl', 'req', '-x509', '-newkey', 'rsa:2048', '-nodes', '-keyout', w/'ca.key',
|
||||
'-out', w/'ca.crt', '-days', '1', '-subj', '/CN=isolated-test-ca'])
|
||||
|
||||
def cert(name, cn, eku, ip=None):
|
||||
run(['openssl', 'req', '-new', '-newkey', 'rsa:2048', '-nodes', '-keyout', w/f'{name}.key',
|
||||
'-out', w/f'{name}.csr', '-subj', f'/CN={cn}' if cn else '/'])
|
||||
ext = w/f'{name}.ext'
|
||||
ext.write_text(f'extendedKeyUsage={eku}\n' + (f'subjectAltName=IP:{ip},IP:127.0.0.1\n' if ip else ''))
|
||||
run(['openssl', 'x509', '-req', '-in', w/f'{name}.csr', '-CA', w/'ca.crt',
|
||||
'-CAkey', w/'ca.key', '-CAcreateserial', '-out', w/f'{name}.crt', '-days', '1', '-extfile', ext])
|
||||
|
||||
cert('admin', 'root', 'clientAuth')
|
||||
cert('client', '', 'clientAuth')
|
||||
cert('gateway', '', 'clientAuth')
|
||||
env = jinja2.Environment(loader=jinja2.FileSystemLoader(ROOT/'ansible/roles/shared_etcd/templates'), undefined=jinja2.StrictUndefined)
|
||||
env.filters['to_json'] = json.dumps
|
||||
defaults = yaml.safe_load((ROOT/'ansible/roles/shared_etcd/defaults/main.yml').read_text())
|
||||
names = ['test1', 'test2', 'test3']
|
||||
hosts = {name: {'etcd_address': f'127.0.0.{i+2}'} for i, name in enumerate(names)}
|
||||
for name in names:
|
||||
d = w/name
|
||||
d.mkdir()
|
||||
for kind, cn, eku in [('server', name, 'serverAuth'), ('peer', 'homelab-etcd-peer', 'serverAuth,clientAuth')]:
|
||||
cert(f'{name}-{kind}', cn, eku, hosts[name]['etcd_address'])
|
||||
for suffix in ['crt', 'key']:
|
||||
shutil.copy(w/f'{name}-{kind}.{suffix}', d/f'{kind}.{suffix}')
|
||||
shutil.copy(w/'ca.crt', d/'ca.crt')
|
||||
for suffix in ['crt', 'key']:
|
||||
shutil.copy(w/f'gateway.{suffix}', d/f'gateway.{suffix}')
|
||||
values = dict(defaults, inventory_hostname=name, groups={'etcd': names}, hostvars=hosts,
|
||||
etcd_address=hosts[name]['etcd_address'], etcd_config_dir=str(d), etcd_data_dir=str(d/'data'),
|
||||
etcd_client_port=22379, etcd_peer_port=22380, etcd_metrics_port=0)
|
||||
config = env.get_template('etcd.yml.j2').render(**values)
|
||||
# 三成员共享进程命名空间，metrics 用独立 loopback IP。
|
||||
config = config.replace('http://127.0.0.1:0', f"http://{hosts[name]['etcd_address']}:22381")
|
||||
(d/'config.yml').write_text(config)
|
||||
handle = (d/'etcd.log').open('w')
|
||||
handles.append(handle)
|
||||
processes.append(subprocess.Popen([str(BIN/'etcd'), '--config-file='+str(d/'config.yml')], stdout=handle, stderr=handle))
|
||||
|
||||
base_env = dict(os.environ, ETCDCTL_ENDPOINTS='https://127.0.0.2:22379', ETCDCTL_CACERT=str(w/'ca.crt'),
|
||||
ETCDCTL_CERT=str(w/'admin.crt'), ETCDCTL_KEY=str(w/'admin.key'), ETCDCTL_DIAL_TIMEOUT='2s', ETCDCTL_COMMAND_TIMEOUT='3s')
|
||||
|
||||
def ctl(*args, input=None, env=None):
|
||||
return run([BIN/'etcdctl', *args], input=input, env=env or base_env)
|
||||
|
||||
for attempt in range(5):
|
||||
try:
|
||||
ctl('endpoint', 'health')
|
||||
break
|
||||
except subprocess.CalledProcessError:
|
||||
if any(p.poll() is not None for p in processes):
|
||||
|
||||
for name in names:
|
||||
print((w/name/'etcd.log').read_text()[-4000:])
|
||||
raise RuntimeError('test etcd exited during startup')
|
||||
time.sleep(0.2)
|
||||
else:
|
||||
|
||||
print((w/'test1'/'etcd.log').read_text()[-5000:])
|
||||
raise RuntimeError('test quorum did not form')
|
||||
server = ThreadingHTTPServer(('127.0.0.1', 0), FakeBao)
|
||||
threading.Thread(target=server.serve_forever, daemon=True).start()
|
||||
inventory = w/'hosts.yml'
|
||||
inventory.write_text(yaml.safe_dump({'all': {'children': {'etcd': {'hosts': {'test1': {'ansible_connection': 'local'}}}}}}))
|
||||
extra = {
|
||||
'ansible_become': False, 'etcd_address': '127.0.0.2', 'etcd_client_port': 22379,
|
||||
'etcd_install_dir': str(BIN), 'etcd_config_dir': str(w), 'etcd_bootstrap_auth': True,
|
||||
'etcd_bao_url': f'http://127.0.0.1:{server.server_port}', 'etcd_bao_token': 'isolated-test-token',
|
||||
'etcd_bao_kv_mount': 'kv', 'etcd_bao_secret_base': 'infra/etcd/consumers',
|
||||
'etcd_consumers': [{'name': 'patroni-pg-prod', 'prefix': '/homelab/patroni/pg-prod/'}],
|
||||
}
|
||||
(w/'extra.json').write_text(json.dumps(extra))
|
||||
ansible_env = dict(os.environ, ANSIBLE_LOCAL_TEMP=str(w/'ansible-tmp'), ANSIBLE_NOCOLOR='1',
|
||||
ANSIBLE_ROLES_PATH=str(ROOT/'ansible/roles'))
|
||||
|
||||
def play(name, fail=False):
|
||||
r = subprocess.run(['ansible-playbook', '-i', str(inventory), str(ROOT/'ansible'/name), '-e', '@'+str(w/'extra.json')],
|
||||
capture_output=True, text=True, env=ansible_env)
|
||||
(w/(name+'.log')).write_text(r.stdout+r.stderr)
|
||||
if fail:
|
||||
assert r.returncode != 0, 'expected a fail-closed playbook error'
|
||||
elif r.returncode:
|
||||
# Tasks use no_log for secret data; retain useful task/line diagnostic.
|
||||
print(r.stdout[-5000:]);print(r.stderr[-2000:])
|
||||
raise RuntimeError(name+' failed')
|
||||
return r.stdout
|
||||
|
||||
play('bootstrap-auth.yml')
|
||||
assert re.search(r'changed=0\s', play('bootstrap-auth.yml'))
|
||||
play('consumers.yml')
|
||||
secret = FakeBao.records['patroni-pg-prod']['password']
|
||||
assert len(secret) == 48
|
||||
assert re.search(r'changed=0\s', play('consumers.yml'))
|
||||
assert FakeBao.writes == 1
|
||||
role = json.loads(ctl('--write-out=json', 'role', 'get', 'patroni-pg-prod').stdout)
|
||||
assert role['perm'][0]['key'] == base64.b64encode(b'/homelab/patroni/pg-prod/').decode()
|
||||
client_env = dict(base_env, ETCDCTL_CERT=str(w/'client.crt'), ETCDCTL_KEY=str(w/'client.key'),
|
||||
ETCDCTL_USER='patroni-pg-prod', ETCDCTL_PASSWORD=secret)
|
||||
ctl('put', '/homelab/patroni/pg-prod/test', 'ok', env=client_env)
|
||||
try:
|
||||
ctl('put', '/homelab/other/test', 'denied', env=client_env)
|
||||
raise AssertionError('cross-prefix write succeeded')
|
||||
except subprocess.CalledProcessError:
|
||||
pass
|
||||
# 模拟 Patroni 的 gateway 协议，证明 mTLS + username/password 可组合使用。
|
||||
ctx = ssl.create_default_context(cafile=str(w/'ca.crt'))
|
||||
ctx.load_cert_chain(w/'client.crt', w/'client.key')
|
||||
body = json.dumps({'name':'patroni-pg-prod', 'password':secret}).encode()
|
||||
req = urllib.request.Request('https://127.0.0.2:22379/v3/auth/authenticate', data=body, headers={'Content-Type':'application/json'})
|
||||
with urllib.request.urlopen(req, context=ctx) as response:
|
||||
token = json.load(response)['token']
|
||||
gateway_body = json.dumps({'key': base64.b64encode(b'/homelab/patroni/pg-prod/gateway').decode(),
|
||||
'value': base64.b64encode(b'ok').decode()}).encode()
|
||||
req = urllib.request.Request('https://127.0.0.2:22379/v3/kv/put', data=gateway_body,
|
||||
headers={'Content-Type': 'application/json', 'Authorization': token})
|
||||
with urllib.request.urlopen(req, context=ctx) as response:
|
||||
assert response.status == 200
|
||||
# 对已有用户，秘密值被意外覆盖也必须失败，而不是悄悄改 etcd 密码。
|
||||
FakeBao.records['patroni-pg-prod']['password'] = 'x' * 48
|
||||
play('consumers.yml', fail=True)
|
||||
FakeBao.records['patroni-pg-prod']['password'] = secret
|
||||
FakeBao.deny = True
|
||||
play('consumers.yml', fail=True)
|
||||
FakeBao.deny = False
|
||||
saved = FakeBao.records.pop('patroni-pg-prod')
|
||||
play('consumers.yml', fail=True) # metadata 存在、data 已删除
|
||||
FakeBao.versions.clear()
|
||||
play('consumers.yml', fail=True) # 用户存在但 metadata 丢失
|
||||
assert FakeBao.writes == 1
|
||||
FakeBao.records['patroni-pg-prod'] = saved
|
||||
FakeBao.versions['patroni-pg-prod'] = 1
|
||||
ctl('snapshot', 'save', str(w/'snapshot.db'))
|
||||
snapshot = json.loads(run([BIN/'etcdutl', '--write-out=json', 'snapshot', 'status', w/'snapshot.db']).stdout)
|
||||
assert snapshot['totalKey'] > 0
|
||||
run([BIN/'etcdutl', 'snapshot', 'restore', w/'snapshot.db', '--data-dir='+str(w/'restored')])
|
||||
rss = []
|
||||
for process in processes:
|
||||
match = re.search(r'^VmRSS:\s+(\d+)', Path(f'/proc/{process.pid}/status').read_text(), re.M)
|
||||
rss.append(round(int(match[1]) / 1024, 1))
|
||||
processes[2].terminate();processes[2].wait(timeout=10)
|
||||
ctl('put', '/homelab/patroni/pg-prod/after-member-loss', 'ok', env=client_env)
|
||||
print('PASS: three-member mTLS, auth bootstrap/idempotence, Bao create-once/fail-closed, prefix isolation, gateway auth, snapshot/restore, one-member loss')
|
||||
print('Idle test member RSS MiB (not a production capacity result):', rss)
|
||||
finally:
|
||||
if server:
|
||||
server.shutdown();server.server_close()
|
||||
for process in processes:
|
||||
if process.poll() is None:
|
||||
process.terminate()
|
||||
try: process.wait(timeout=10)
|
||||
except subprocess.TimeoutExpired: process.kill();process.wait()
|
||||
for handle in handles:
|
||||
handle.close()
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -1,71 +0,0 @@
|
||||
"""续签调度失败关闭测试；不会连接真实 Bao 或运行 Ansible。"""
|
||||
import importlib.util
|
||||
import subprocess
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
from unittest.mock import patch, Mock
|
||||
|
||||
spec = importlib.util.spec_from_file_location('renew', Path(__file__).parents[1] / 'controller/renew.py')
|
||||
renew = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(renew)
|
||||
|
||||
|
||||
class RenewalTests(unittest.TestCase):
|
||||
def execute(self, results):
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
lock = Path(directory) / 'lock'
|
||||
success = Path(directory) / 'last-success'
|
||||
original_open = open
|
||||
response = Mock()
|
||||
response.__enter__ = Mock(return_value=response)
|
||||
response.__exit__ = Mock(return_value=False)
|
||||
with patch('builtins.open', side_effect=lambda *_: original_open(lock, 'a')), \
|
||||
patch.object(renew.subprocess, 'run', side_effect=results) as run, \
|
||||
patch.object(renew.urllib.request, 'urlopen', return_value=response) as revoke, \
|
||||
patch.object(renew, 'Path', return_value=success):
|
||||
rc = renew.main()
|
||||
return rc, run.call_args_list, revoke.call_count, success.exists()
|
||||
|
||||
def test_login_failure_never_changes_members(self):
|
||||
rc, calls, revokes, success = self.execute([subprocess.CompletedProcess([], 1, '', '')])
|
||||
self.assertEqual(rc, 1)
|
||||
self.assertEqual(len(calls), 1)
|
||||
self.assertEqual(revokes, 0)
|
||||
self.assertFalse(success)
|
||||
|
||||
def test_unhealthy_cluster_never_runs_site(self):
|
||||
rc, calls, revokes, success = self.execute([
|
||||
subprocess.CompletedProcess([], 0, 'test-only-token'),
|
||||
subprocess.CompletedProcess([], 2),
|
||||
])
|
||||
self.assertEqual(rc, 2)
|
||||
self.assertEqual(calls[1].args[0][-1], 'verify.yml')
|
||||
self.assertEqual(len(calls), 2)
|
||||
self.assertEqual(revokes, 1)
|
||||
self.assertFalse(success)
|
||||
|
||||
def test_success_checks_then_converges_and_revokes(self):
|
||||
rc, calls, revokes, success = self.execute([
|
||||
subprocess.CompletedProcess([], 0, 'test-only-token'),
|
||||
subprocess.CompletedProcess([], 0),
|
||||
subprocess.CompletedProcess([], 0),
|
||||
])
|
||||
self.assertEqual(rc, 0)
|
||||
self.assertEqual([c.args[0][-1] for c in calls[1:]], ['verify.yml', 'site.yml'])
|
||||
self.assertEqual(revokes, 1)
|
||||
self.assertTrue(success)
|
||||
|
||||
def test_site_failure_is_not_success(self):
|
||||
rc, calls, revokes, success = self.execute([
|
||||
subprocess.CompletedProcess([], 0, 'test-only-token'),
|
||||
subprocess.CompletedProcess([], 0),
|
||||
subprocess.CompletedProcess([], 2),
|
||||
])
|
||||
self.assertEqual(rc, 2)
|
||||
self.assertEqual(revokes, 1)
|
||||
self.assertFalse(success)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
@@ -1,29 +0,0 @@
|
||||
# laptop 内存预算
|
||||
|
||||
laptop 同时承载 Kubernetes、数据库和 libvirt VM。ARC 使用明确的 **4 GiB** 上限，
|
||||
为服务留下突发余量；不修改自动 ARC 下限、swap、业务内存或 VM I/O 策略。
|
||||
|
||||
在 laptop 上执行：
|
||||
|
||||
```bash
|
||||
ansible-playbook -i localhost, infrastructure/host-memory/laptop-arc.yml --check
|
||||
ansible-playbook -i localhost, infrastructure/host-memory/laptop-arc.yml
|
||||
```
|
||||
|
||||
play 在线写入 `/sys/module/zfs/parameters/zfs_arc_max`，并持久化到
|
||||
`/etc/modprobe.d/homelab-zfs-arc.conf`。配置变化时以低 CPU/I/O 优先级更新所有已安装
|
||||
内核的 initramfs。不重启、不卸载 ZFS。ARC 当前使用量不保证立即降到上限以下；
|
||||
`zfs_arc_max` 也不是 ZFS 全部内存的硬上限。
|
||||
|
||||
验证 `/proc/spl/kstat/zfs/arcstats` 的 `c_max=4294967296`，并在 Grafana 比较
|
||||
`node_zfs_arc_size`、`node_zfs_arc_c_max`、内存/I/O PSI、换页及数据库请求延迟。
|
||||
命中率下降或磁盘读压力上升时按实际负载重新评估预算，不通过重启或清缓存进行验收。
|
||||
|
||||
回滚时先从运行中的 `arcstats` 核实原自动上限，再在线写回该非零值。
|
||||
删除本 play 的 modprobe 文件并更新 initramfs，可恢复以后启动时的自动策略。
|
||||
不要依赖运行时写入 `0` 恢复自动计算；该行为受 OpenZFS 参数实现限制。
|
||||
|
||||
2026-09-26 故障关联：winadmin 自动更新期间虚拟磁盘写入与 ARC 增长、内存回收和
|
||||
Gitea 超时重合；限制 ARC 是内存预算措施，不代表已证明或解决全部存储延迟原因。
|
||||
排查证据与后续验收见
|
||||
[宿主机 ARC 预算](https://git.ddupan.top/panxiao81/homelab-wiki/src/branch/main/guides/laptop-arc-budget.md)。
|
||||
@@ -1,43 +0,0 @@
|
||||
---
|
||||
- name: 为 laptop 的 VM 和服务预留内存
|
||||
hosts: localhost
|
||||
connection: local
|
||||
become: true
|
||||
gather_facts: true
|
||||
vars:
|
||||
laptop_zfs_arc_max_bytes: 4294967296
|
||||
tasks:
|
||||
- name: 确认目标和 ARC 预算
|
||||
ansible.builtin.assert:
|
||||
that:
|
||||
- ansible_facts.hostname == 'laptop'
|
||||
- laptop_zfs_arc_max_bytes | int == 4294967296
|
||||
fail_msg: 此 play 仅用于 laptop 的 4 GiB ARC 预算。
|
||||
|
||||
- name: 读取运行中的 ARC 上限参数
|
||||
ansible.builtin.command: cat /sys/module/zfs/parameters/zfs_arc_max
|
||||
register: arc_max_before
|
||||
changed_when: false
|
||||
check_mode: false
|
||||
|
||||
- name: 持久化 ARC 上限
|
||||
ansible.builtin.copy:
|
||||
dest: /etc/modprobe.d/homelab-zfs-arc.conf
|
||||
owner: root
|
||||
group: root
|
||||
mode: '0644'
|
||||
content: |
|
||||
# Ansible managed: infrastructure/host-memory/laptop-arc.yml
|
||||
# 4 GiB ARC budget for laptop; retain the automatic minimum.
|
||||
options zfs zfs_arc_max={{ laptop_zfs_arc_max_bytes }}
|
||||
notify: 更新已安装内核的 initramfs
|
||||
|
||||
- name: 在线应用 ARC 上限
|
||||
ansible.builtin.shell: "printf '%s\\n' {{ laptop_zfs_arc_max_bytes | string | quote }} > /sys/module/zfs/parameters/zfs_arc_max"
|
||||
when: arc_max_before.stdout | int != laptop_zfs_arc_max_bytes | int
|
||||
changed_when: true
|
||||
|
||||
handlers:
|
||||
- name: 更新已安装内核的 initramfs
|
||||
ansible.builtin.command: nice -n 10 ionice -c 3 update-initramfs -u -k all
|
||||
changed_when: true
|
||||
@@ -1,72 +0,0 @@
|
||||
# kata-lab
|
||||
|
||||
`kata-lab` 是位于 Proxmox `pve2` 的可销毁验证环境，用来验证 nested KVM、k3s、
|
||||
Kata Containers，以及后续一 job 一 Kata Pod 的 Gitea Runner。它不是 dev/stg
|
||||
长期服务，也不承载持久数据。
|
||||
|
||||
> 状态：VMID 147 已于 2026-09-14 销毁，Terraform state 为空。目录保留首次验证结果
|
||||
> 与可复现配置；后续验证转向跨 PVE 节点的轻量 LXC worker 方案。
|
||||
|
||||
## Terraform bootstrap
|
||||
|
||||
1. 将 Proxmox API token 写入被 Git 忽略且权限为 `0600` 的
|
||||
`terraform/credentials.auto.tfvars`。
|
||||
2. 取得内部 CA，并仅为当前 Terraform 进程设置 Go TLS trust：
|
||||
|
||||
```bash
|
||||
curl -fsSL https://bao.ad.ddupan.top:8200/v1/pki/ca/pem \
|
||||
-o /tmp/kata-lab-ddupan-ca.pem
|
||||
export SSL_CERT_FILE=/tmp/kata-lab-ddupan-ca.pem
|
||||
```
|
||||
|
||||
3. 初始化并审阅 plan：
|
||||
|
||||
```bash
|
||||
cd infrastructure/kata-lab/terraform
|
||||
terraform init
|
||||
terraform plan
|
||||
```
|
||||
|
||||
Terraform 使用 `laptop:import/noble-server-cloudimg-amd64.qcow2`（40 GiB
|
||||
virtual size），在
|
||||
`pve-rg-hdd` 创建 VM root disk，并把 cloud-init snippet 放到 `laptop`
|
||||
datastore。VM root disk 设为 41 GiB，以容纳 PVE import 的块对齐增量。VM 使用
|
||||
DHCP、`cpu.type = host`、4 vCPU 和 4 GiB 内存；cloud-init
|
||||
安装 `qemu-guest-agent`、单节点 k3s，并记录 `/dev/kvm` 检查结果。
|
||||
|
||||
## Kata Containers
|
||||
|
||||
Ansible 使用 Kata Containers 4.1.0 官方 `kata-deploy` chart。配置明确选择 k3s，
|
||||
只安装 `qemu-runtime-rs`，不安装额外 snapshotter，并采用 `job` 模式，安装结束后
|
||||
不保留 privileged DaemonSet。默认 `RuntimeClass` 名为 `kata`。
|
||||
|
||||
Terraform 输出的 DHCP 地址变化时，先更新 `ansible/inventory/hosts.yml`，然后运行：
|
||||
|
||||
```bash
|
||||
cd infrastructure/kata-lab/ansible
|
||||
ansible-playbook kata.yml
|
||||
ansible-playbook verify.yml
|
||||
```
|
||||
|
||||
`verify.yml` 首先创建一个 `runtimeClassName: kata` 的短生命周期 Pod，并确认 Pod 内
|
||||
看到的 Kata guest kernel 与 k3s node 的宿主 kernel 不同。随后它在另一个 Kata Pod
|
||||
内启动 privileged `dockerd` sidecar，让普通 Docker client 容器通过共享网络命名空间
|
||||
的 `127.0.0.1:2375` 运行一个嵌套容器。这对应后续 Gitea Runner 的目标形态：runner
|
||||
直接执行 job，只在需要 Docker API 时连接同 Pod 的 daemon，而不把 Docker executor
|
||||
作为 job 的外层。Kata 下共享 volume 中的 Unix socket 文件虽然双方可见，但不能跨
|
||||
容器连接，因此不能在这里沿用普通 Pod 常见的 `/var/run/docker.sock` 方案；loopback
|
||||
端口没有通过 Service 暴露到 Pod 外。
|
||||
验证 Pod 会保留以供排查；再次运行时会先替换它们。
|
||||
|
||||
2026-09-13 的首次验证结果：Kata guest kernel 为 `6.18.35`，node host kernel 为
|
||||
`6.8.0-90-generic`；sidecar daemon 成功运行 `busybox:1.37` 嵌套容器。Docker 27.5.1
|
||||
在当前 Kata guest 中无法挂载 overlay2，自动回退到 `vfs`；功能验证通过，但正式用于
|
||||
CI 前需要解决或接受其镜像层复制与磁盘性能开销。
|
||||
|
||||
不要提交 `credentials.auto.tfvars`、Terraform state 或 saved plan。销毁前确认目标
|
||||
仍然是 VMID 147 / `kata-lab`：
|
||||
|
||||
```bash
|
||||
terraform plan -destroy
|
||||
terraform destroy
|
||||
```
|
||||
@@ -1,10 +0,0 @@
|
||||
[defaults]
|
||||
inventory = inventory/hosts.yml
|
||||
host_key_checking = True
|
||||
interpreter_python = auto_silent
|
||||
retry_files_enabled = False
|
||||
local_tmp = /tmp/ansible-kata-lab-local
|
||||
remote_tmp = /tmp/ansible-kata-lab-remote
|
||||
|
||||
[ssh_connection]
|
||||
pipelining = True
|
||||
@@ -1,8 +0,0 @@
|
||||
---
|
||||
all:
|
||||
hosts:
|
||||
kata-lab:
|
||||
ansible_host: 192.168.10.13
|
||||
ansible_user: ansible
|
||||
vars:
|
||||
ansible_become: true
|
||||
@@ -1,88 +0,0 @@
|
||||
---
|
||||
- name: Install Kata Containers in the disposable k3s lab
|
||||
hosts: kata-lab
|
||||
gather_facts: false
|
||||
vars:
|
||||
kata_version: "4.1.0"
|
||||
kata_chart_url: >-
|
||||
https://github.com/kata-containers/kata-containers/releases/download/{{ kata_version }}/kata-deploy-{{ kata_version }}.tgz
|
||||
tasks:
|
||||
- name: Check nested KVM is usable
|
||||
ansible.builtin.stat:
|
||||
path: /dev/kvm
|
||||
register: kata_kvm
|
||||
|
||||
- name: Require nested KVM
|
||||
ansible.builtin.assert:
|
||||
that:
|
||||
- kata_kvm.stat.exists
|
||||
- kata_kvm.stat.ischr
|
||||
fail_msg: /dev/kvm is unavailable; Kata QEMU cannot start efficiently
|
||||
|
||||
- name: Install Kata through the k3s Helm controller
|
||||
ansible.builtin.copy:
|
||||
dest: /var/lib/rancher/k3s/server/manifests/kata-deploy.yaml
|
||||
owner: root
|
||||
group: root
|
||||
mode: "0644"
|
||||
content: |
|
||||
apiVersion: helm.cattle.io/v1
|
||||
kind: HelmChart
|
||||
metadata:
|
||||
name: kata-deploy
|
||||
namespace: kube-system
|
||||
spec:
|
||||
chart: {{ kata_chart_url }}
|
||||
targetNamespace: kata-system
|
||||
createNamespace: true
|
||||
timeout: 15m
|
||||
failurePolicy: abort
|
||||
valuesContent: |-
|
||||
deploymentMode: job
|
||||
k8sDistribution: k3s
|
||||
snapshotter:
|
||||
setup: []
|
||||
shims:
|
||||
disableAll: true
|
||||
qemu-runtime-rs:
|
||||
enabled: true
|
||||
runtimeClasses:
|
||||
enabled: true
|
||||
createDefault: true
|
||||
node-feature-discovery:
|
||||
enabled: false
|
||||
|
||||
- name: Wait for the Helm installation job to appear
|
||||
ansible.builtin.command:
|
||||
cmd: k3s kubectl get job/helm-install-kata-deploy -n kube-system
|
||||
register: kata_helm_job
|
||||
retries: 60
|
||||
delay: 2
|
||||
until: kata_helm_job.rc == 0
|
||||
changed_when: false
|
||||
|
||||
- name: Wait for the Helm installation job to finish
|
||||
ansible.builtin.command:
|
||||
cmd: >-
|
||||
k3s kubectl wait --for=condition=complete
|
||||
job/helm-install-kata-deploy -n kube-system --timeout=20m
|
||||
changed_when: false
|
||||
|
||||
- name: Wait for Kata node installation jobs
|
||||
ansible.builtin.shell:
|
||||
cmd: |
|
||||
set -euo pipefail
|
||||
jobs=$(k3s kubectl get jobs -n kata-system -l app.kubernetes.io/instance=kata-deploy -o name)
|
||||
test -n "${jobs}"
|
||||
k3s kubectl wait --for=condition=complete -n kata-system $jobs --timeout=20m
|
||||
executable: /bin/bash
|
||||
changed_when: false
|
||||
|
||||
- name: Wait for the default Kata RuntimeClass
|
||||
ansible.builtin.command:
|
||||
cmd: k3s kubectl get runtimeclass kata
|
||||
register: kata_runtime_class
|
||||
retries: 30
|
||||
delay: 2
|
||||
until: kata_runtime_class.rc == 0
|
||||
changed_when: false
|
||||
@@ -1,125 +0,0 @@
|
||||
---
|
||||
- name: Verify Kata isolation with a disposable Pod
|
||||
hosts: kata-lab
|
||||
gather_facts: false
|
||||
tasks:
|
||||
- name: Remove an earlier smoke Pod
|
||||
ansible.builtin.command:
|
||||
cmd: k3s kubectl delete pod kata-smoke --ignore-not-found --wait=true
|
||||
changed_when: false
|
||||
|
||||
- name: Create the Kata smoke Pod
|
||||
ansible.builtin.shell:
|
||||
cmd: |
|
||||
set -o pipefail
|
||||
k3s kubectl apply -f - <<'EOF'
|
||||
apiVersion: v1
|
||||
kind: Pod
|
||||
metadata:
|
||||
name: kata-smoke
|
||||
spec:
|
||||
runtimeClassName: kata
|
||||
restartPolicy: Never
|
||||
containers:
|
||||
- name: smoke
|
||||
image: docker.io/library/busybox:1.37
|
||||
command:
|
||||
- sh
|
||||
- -c
|
||||
- 'printf "guest-kernel="; uname -r; test -c /dev/kvm && exit 1 || true'
|
||||
EOF
|
||||
executable: /bin/bash
|
||||
changed_when: true
|
||||
|
||||
- name: Wait for the Kata Pod to finish
|
||||
ansible.builtin.command:
|
||||
cmd: k3s kubectl wait --for=jsonpath='{.status.phase}'=Succeeded pod/kata-smoke --timeout=10m
|
||||
changed_when: false
|
||||
|
||||
- name: Read the guest kernel version
|
||||
ansible.builtin.command:
|
||||
cmd: k3s kubectl logs kata-smoke
|
||||
register: kata_smoke_log
|
||||
changed_when: false
|
||||
|
||||
- name: Read the host kernel version
|
||||
ansible.builtin.command:
|
||||
cmd: uname -r
|
||||
register: kata_host_kernel
|
||||
changed_when: false
|
||||
|
||||
- name: Require a distinct guest kernel
|
||||
ansible.builtin.assert:
|
||||
that:
|
||||
- kata_smoke_log.stdout is match('^guest-kernel=.+')
|
||||
- kata_smoke_log.stdout | regex_replace('^guest-kernel=', '') != kata_host_kernel.stdout
|
||||
success_msg: >-
|
||||
Kata guest {{ kata_smoke_log.stdout }} differs from host-kernel={{ kata_host_kernel.stdout }}
|
||||
|
||||
- name: Remove an earlier Docker sidecar smoke Pod
|
||||
ansible.builtin.command:
|
||||
cmd: k3s kubectl delete pod kata-docker-smoke --ignore-not-found --wait=true
|
||||
changed_when: false
|
||||
|
||||
- name: Create a Kata Pod with a Docker daemon sidecar
|
||||
ansible.builtin.shell:
|
||||
cmd: |
|
||||
set -o pipefail
|
||||
k3s kubectl apply -f - <<'EOF'
|
||||
apiVersion: v1
|
||||
kind: Pod
|
||||
metadata:
|
||||
name: kata-docker-smoke
|
||||
spec:
|
||||
runtimeClassName: kata
|
||||
restartPolicy: Never
|
||||
volumes:
|
||||
- name: docker-run
|
||||
emptyDir: {}
|
||||
containers:
|
||||
- name: dockerd
|
||||
image: docker.io/library/docker:27-dind
|
||||
securityContext:
|
||||
privileged: true
|
||||
env:
|
||||
- name: DOCKER_TLS_CERTDIR
|
||||
value: ""
|
||||
volumeMounts:
|
||||
- name: docker-run
|
||||
mountPath: /var/run
|
||||
- name: client
|
||||
image: docker.io/library/docker:27-cli
|
||||
env:
|
||||
- name: DOCKER_HOST
|
||||
value: tcp://127.0.0.1:2375
|
||||
volumeMounts:
|
||||
- name: docker-run
|
||||
mountPath: /var/run
|
||||
command:
|
||||
- sh
|
||||
- -c
|
||||
- |
|
||||
until docker info >/dev/null 2>&1; do sleep 1; done
|
||||
docker run --rm docker.io/library/busybox:1.37 echo nested-docker-ok
|
||||
EOF
|
||||
executable: /bin/bash
|
||||
changed_when: true
|
||||
|
||||
- name: Wait for the Docker client to finish
|
||||
ansible.builtin.command:
|
||||
cmd: >-
|
||||
k3s kubectl wait --for=jsonpath='{.status.containerStatuses[?(@.name=="client")].state.terminated.exitCode}'=0
|
||||
pod/kata-docker-smoke --timeout=10m
|
||||
changed_when: false
|
||||
|
||||
- name: Read the Docker client result
|
||||
ansible.builtin.command:
|
||||
cmd: k3s kubectl logs kata-docker-smoke -c client
|
||||
register: kata_docker_smoke_log
|
||||
changed_when: false
|
||||
|
||||
- name: Require Docker to run a nested container
|
||||
ansible.builtin.assert:
|
||||
that:
|
||||
- "'nested-docker-ok' in kata_docker_smoke_log.stdout"
|
||||
success_msg: Docker daemon sidecar completed a nested container inside the Kata VM
|
||||
@@ -1,24 +0,0 @@
|
||||
# This file is maintained automatically by "terraform init".
|
||||
# Manual edits may be lost in future updates.
|
||||
|
||||
provider "registry.terraform.io/bpg/proxmox" {
|
||||
version = "0.111.1"
|
||||
constraints = "0.111.1"
|
||||
hashes = [
|
||||
"h1:ML2D3UUZTM99yrll/EBXj7wBYMb8xmQgomqFNybEoxY=",
|
||||
"zh:18fb7c31a08dde6bffa1a4d4a211e604d6d17eec7092fd59331b3db3c6f3742c",
|
||||
"zh:1cd60761538289d4dd2a1086b3ae62a7b0bdd4b1a2f824e9a44e243413168dba",
|
||||
"zh:2eb76f6fc8299b6820ff678c8252332cc3366e226b5ae2e61748fd2449c1ed92",
|
||||
"zh:45e6f7ebd0bf48911d37060359a4f359b5743b3092e985295733990e406d0416",
|
||||
"zh:4aa8ba912eae37975d2e983394d173e595ca34fc76b5bf220b37d0e99d76e98c",
|
||||
"zh:58e0789923103a77d502a0a9fc3eb920625e8eb935ec2d4ac0d006aebd1d186c",
|
||||
"zh:6df8aa85fb8865915537e946c19b02538ad188018a629759c213c6f03730f642",
|
||||
"zh:6ed47bc00d0913a1d0880618fa1376115e9edab6b4a658c081061a7f0e4ca360",
|
||||
"zh:c5b10ff4f33df7e4c29e8f1127d49845b561b37b57517e844fb0954d7923d65e",
|
||||
"zh:d016510e14b738499f0db9d9b3aafe82fc6877fb4ab4e9f831fb68a8d70a1385",
|
||||
"zh:d941f394069bbf24351b363da1c64383f487067aaee0a84f9b96476d4912e212",
|
||||
"zh:ddf271dbc2632ae8ffa8de3972f243ee47d260cb2ac90aa784f2746d98e21a0f",
|
||||
"zh:ed0caa3501c42f611b7e9622c9b1df69fd85dc25a3cd88d3076381829688cd62",
|
||||
"zh:f26e0763dbe6a6b2195c94b44696f2110f7f55433dc142839be16b9697fa5597",
|
||||
]
|
||||
}
|
||||
@@ -1,34 +0,0 @@
|
||||
#cloud-config
|
||||
hostname: ${hostname}
|
||||
manage_etc_hosts: true
|
||||
timezone: Etc/UTC
|
||||
|
||||
users:
|
||||
- default
|
||||
- name: ansible
|
||||
groups: [adm, sudo]
|
||||
shell: /bin/bash
|
||||
lock_passwd: true
|
||||
sudo: ALL=(ALL) NOPASSWD:ALL
|
||||
ssh_authorized_keys:
|
||||
- ${ssh_public_key}
|
||||
|
||||
package_update: true
|
||||
package_upgrade: false
|
||||
packages:
|
||||
- ca-certificates
|
||||
- curl
|
||||
- jq
|
||||
- qemu-guest-agent
|
||||
|
||||
runcmd:
|
||||
- [systemctl, enable, --now, qemu-guest-agent]
|
||||
- [modprobe, kvm_amd]
|
||||
- [sh, -c, 'test -c /dev/kvm && echo available > /var/lib/cloud/nested-kvm.status || echo unavailable > /var/lib/cloud/nested-kvm.status']
|
||||
- [sh, -c, 'curl -sfL https://get.k3s.io -o /tmp/install-k3s.sh']
|
||||
- [chmod, '0755', /tmp/install-k3s.sh]
|
||||
- [sh, -c, 'INSTALL_K3S_VERSION="${k3s_version}" INSTALL_K3S_EXEC="server --disable=traefik --write-kubeconfig-mode=0644" /tmp/install-k3s.sh']
|
||||
- [sh, -c, 'kubectl wait --for=condition=Ready node/${hostname} --timeout=180s']
|
||||
- [touch, /var/lib/cloud/kata-lab-bootstrap.done]
|
||||
|
||||
final_message: "kata-lab bootstrap completed after $UPTIME seconds"
|
||||
@@ -1,3 +0,0 @@
|
||||
# Copy this file to credentials.auto.tfvars and replace the placeholder.
|
||||
# Format: user@realm!token-id=token-secret
|
||||
proxmox_api_token = "REPLACE_ME"
|
||||
@@ -1,86 +0,0 @@
|
||||
locals {
|
||||
ssh_public_key = trimspace(file(pathexpand(var.ssh_public_key_file)))
|
||||
cloud_init = templatefile("${path.module}/cloud-init.yaml.tftpl", {
|
||||
hostname = var.vm_name
|
||||
ssh_public_key = local.ssh_public_key
|
||||
k3s_version = var.k3s_version
|
||||
})
|
||||
}
|
||||
|
||||
data "proxmox_file" "ubuntu_noble" {
|
||||
content_type = "import"
|
||||
datastore_id = var.snippet_datastore_id
|
||||
node_name = var.node_name
|
||||
file_name = "noble-server-cloudimg-amd64.qcow2"
|
||||
}
|
||||
|
||||
resource "proxmox_virtual_environment_file" "cloud_init" {
|
||||
content_type = "snippets"
|
||||
datastore_id = var.snippet_datastore_id
|
||||
node_name = var.node_name
|
||||
|
||||
source_raw {
|
||||
data = local.cloud_init
|
||||
file_name = "${var.vm_name}-cloud-init.yaml"
|
||||
}
|
||||
}
|
||||
|
||||
resource "proxmox_virtual_environment_vm" "kata_lab" {
|
||||
name = var.vm_name
|
||||
description = "Disposable single-node k3s and Kata Containers validation environment."
|
||||
tags = ["terraform", "disposable", "kata"]
|
||||
|
||||
node_name = var.node_name
|
||||
vm_id = var.vm_id
|
||||
|
||||
started = true
|
||||
on_boot = false
|
||||
stop_on_destroy = true
|
||||
|
||||
agent {
|
||||
enabled = true
|
||||
timeout = "15m"
|
||||
}
|
||||
|
||||
cpu {
|
||||
cores = var.vm_cores
|
||||
type = "host"
|
||||
}
|
||||
|
||||
memory {
|
||||
dedicated = var.vm_memory_mb
|
||||
}
|
||||
|
||||
operating_system {
|
||||
type = "l26"
|
||||
}
|
||||
|
||||
scsi_hardware = "virtio-scsi-single"
|
||||
|
||||
disk {
|
||||
datastore_id = var.disk_datastore_id
|
||||
import_from = data.proxmox_file.ubuntu_noble.id
|
||||
interface = "scsi0"
|
||||
iothread = true
|
||||
discard = "on"
|
||||
size = var.vm_disk_gb
|
||||
}
|
||||
|
||||
initialization {
|
||||
datastore_id = var.disk_datastore_id
|
||||
user_data_file_id = proxmox_virtual_environment_file.cloud_init.id
|
||||
|
||||
ip_config {
|
||||
ipv4 {
|
||||
address = "dhcp"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
network_device {
|
||||
bridge = var.network_bridge
|
||||
model = "virtio"
|
||||
}
|
||||
|
||||
serial_device {}
|
||||
}
|
||||
@@ -1,12 +0,0 @@
|
||||
output "vm_id" {
|
||||
value = proxmox_virtual_environment_vm.kata_lab.vm_id
|
||||
}
|
||||
|
||||
output "vm_ipv4_addresses" {
|
||||
description = "QEMU guest agent 报告的地址；忽略 loopback 和 CNI 地址后再用于 SSH。"
|
||||
value = proxmox_virtual_environment_vm.kata_lab.ipv4_addresses
|
||||
}
|
||||
|
||||
output "ssh_user" {
|
||||
value = "ansible"
|
||||
}
|
||||
@@ -1,17 +0,0 @@
|
||||
provider "proxmox" {
|
||||
endpoint = var.proxmox_endpoint
|
||||
api_token = var.proxmox_api_token
|
||||
|
||||
# Snippet uploads use SSH. The provider deliberately does not read
|
||||
# ~/.ssh/config, so map the PVE node explicitly and use the current agent.
|
||||
ssh {
|
||||
agent = false
|
||||
username = "root"
|
||||
private_key = file(pathexpand(var.proxmox_ssh_private_key_file))
|
||||
|
||||
node {
|
||||
name = "pve2"
|
||||
address = "192.168.10.7"
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,83 +0,0 @@
|
||||
variable "proxmox_endpoint" {
|
||||
description = "Proxmox VE API endpoint，不包含 /api2/json。"
|
||||
type = string
|
||||
default = "https://pve1.ad.ddupan.top:8006/"
|
||||
}
|
||||
|
||||
variable "proxmox_api_token" {
|
||||
description = "Proxmox API token，格式为 user@realm!token-id=secret。"
|
||||
type = string
|
||||
sensitive = true
|
||||
}
|
||||
|
||||
variable "node_name" {
|
||||
description = "承载 disposable kata-lab VM 的 PVE 节点。"
|
||||
type = string
|
||||
default = "pve2"
|
||||
}
|
||||
|
||||
variable "vm_id" {
|
||||
description = "kata-lab VMID。"
|
||||
type = number
|
||||
default = 147
|
||||
}
|
||||
|
||||
variable "vm_name" {
|
||||
description = "kata-lab VM 名称和 guest hostname。"
|
||||
type = string
|
||||
default = "kata-lab"
|
||||
}
|
||||
|
||||
variable "vm_memory_mb" {
|
||||
description = "VM 固定内存；pve2 只有约 7.4 GiB 可见内存。"
|
||||
type = number
|
||||
default = 4096
|
||||
}
|
||||
|
||||
variable "vm_cores" {
|
||||
description = "VM vCPU 数量。"
|
||||
type = number
|
||||
default = 4
|
||||
}
|
||||
|
||||
variable "vm_disk_gb" {
|
||||
description = "VM root disk 大小。"
|
||||
type = number
|
||||
default = 41
|
||||
}
|
||||
|
||||
variable "disk_datastore_id" {
|
||||
description = "VM root disk 所在 datastore。"
|
||||
type = string
|
||||
default = "pve-rg-hdd"
|
||||
}
|
||||
|
||||
variable "snippet_datastore_id" {
|
||||
description = "启用 snippets 的共享 datastore。"
|
||||
type = string
|
||||
default = "laptop"
|
||||
}
|
||||
|
||||
variable "network_bridge" {
|
||||
description = "连接 kata-lab VM 的 PVE bridge。"
|
||||
type = string
|
||||
default = "vmbr0"
|
||||
}
|
||||
|
||||
variable "ssh_public_key_file" {
|
||||
description = "注入 cloud-init 用户的 SSH 公钥路径。"
|
||||
type = string
|
||||
default = "~/.ssh/id_ed25519.pub"
|
||||
}
|
||||
|
||||
variable "proxmox_ssh_private_key_file" {
|
||||
description = "provider 上传 snippet 时登录 PVE 节点使用的私钥路径。"
|
||||
type = string
|
||||
default = "~/.ssh/id_ed25519"
|
||||
}
|
||||
|
||||
variable "k3s_version" {
|
||||
description = "可选的固定 k3s 版本；空值使用 stable channel。"
|
||||
type = string
|
||||
default = ""
|
||||
}
|
||||
@@ -1,10 +0,0 @@
|
||||
terraform {
|
||||
required_version = ">= 1.10.0"
|
||||
|
||||
required_providers {
|
||||
proxmox = {
|
||||
source = "bpg/proxmox"
|
||||
version = "0.111.1"
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,180 +0,0 @@
|
||||
# kata-lxc-lab
|
||||
|
||||
在 `pve2` 上验证 privileged LXC 内运行 k3s、Kata/QEMU，并直接使用 PVE host 的
|
||||
`/dev/kvm`、`/dev/vhost-net` 与 `/dev/vhost-vsock`。这是可销毁 PoC，不承载持久数据。
|
||||
|
||||
Terraform 复用原 VM PoC 的 ignored credential 文件，不复制 token：
|
||||
|
||||
```bash
|
||||
cd infrastructure/kata-lxc-lab/terraform
|
||||
export SSL_CERT_FILE=/tmp/kata-lab-ddupan-ca.pem
|
||||
terraform init
|
||||
terraform plan \
|
||||
-var-file=../../kata-lab/terraform/credentials.auto.tfvars
|
||||
terraform apply \
|
||||
-var-file=../../kata-lab/terraform/credentials.auto.tfvars
|
||||
```
|
||||
|
||||
当前 PoC 使用 `pve2`、VMID 147、Ubuntu 24.04 LXC template 和 16 GiB
|
||||
`local-lvm` rootfs。容器为 privileged，并开启 nesting、keyctl、FUSE、mknod；
|
||||
Kata 所需的 KVM、vhost 与 `/dev/net/tun` 设备显式透传；另透传 `/dev/kmsg`，因为
|
||||
kubelet 启动时会打开该设备。
|
||||
PVE 9 的 `force_rw_sys=1` feature 用于开放 kubelet 所需的少量 `/proc/sys` 写操作；
|
||||
provider 0.111.1 尚未暴露该 feature，因此同样由 `pct` 设置。
|
||||
|
||||
k3s/Kata worker 还使用以下 privileged LXC 原生配置；PVE 9 使用 cgroup v2，因此设备
|
||||
规则采用 `lxc.cgroup2.devices.allow`：
|
||||
|
||||
```text
|
||||
lxc.apparmor.profile: unconfined
|
||||
lxc.cgroup2.devices.allow: a
|
||||
lxc.cap.drop:
|
||||
lxc.mount.auto: proc:rw sys:rw
|
||||
```
|
||||
|
||||
这些设置让容器内 kubelet、containerd 和 Kata shim 能管理 mount、cgroup、设备与
|
||||
共享宿主 sysctl。因此外层 LXC 不是安全边界；不可信 CI 的安全边界是内层 Kata VM。
|
||||
`/dev/kmsg` 当前直接透传，没有使用 `/dev/console` symlink fallback。
|
||||
|
||||
PVE 将 privileged LXC 的创建限制为 `root@pam`（API token 即使拥有 `PVEAdmin` 也
|
||||
缺少 root-only 的 `Sys.Modify`）。因此实际创建入口使用 root SSH 上的 Ansible/pct：
|
||||
|
||||
```bash
|
||||
cd infrastructure/kata-lxc-lab/ansible
|
||||
ansible-playbook create.yml
|
||||
ansible-playbook bootstrap.yml
|
||||
ansible-playbook kata.yml
|
||||
ansible-playbook verify.yml
|
||||
ansible-playbook verify-fuse.yml
|
||||
```
|
||||
|
||||
Terraform 文件保留用于记录 provider 权限边界与声明式目标；当前 token 执行 apply 会
|
||||
在创建前返回 403，不会产生资源或 state。
|
||||
|
||||
## 验证结果
|
||||
|
||||
当前管理地址为 `192.168.10.128`：2026-09-14 主 LAN DHCP 池调整后，从 `.11`
|
||||
续租迁移到 `.128`，并同步 Ansible inventory。它仍是动态租约，不是固定地址。
|
||||
|
||||
2026-09-14，pve2 VMID 147 / `192.168.10.11` 验证通过：
|
||||
|
||||
- LXC 内 k3s `v1.36.4+k3s1` node Ready；
|
||||
- Kata Containers 4.1.0 `qemu-runtime-rs` 安装成功；
|
||||
- Kata guest kernel `6.18.35`，外层共享的 PVE kernel `7.0.2-6-pve`；
|
||||
- Kata Pod 内的 Docker 27.5.1 daemon 成功运行 `busybox:1.37` 嵌套容器；
|
||||
- `/dev/net/tun` 是 Kata 创建 TAP/link 的必要设备，未透传时 runtime-rs 在
|
||||
`create link` 返回 `ENOENT`；
|
||||
- `fuse-overlayfs` smoke test 成功：Docker 报告
|
||||
`storage-driver=fuse-overlayfs`，并成功运行嵌套容器。
|
||||
|
||||
正式方案保持 Kata 默认 `emptydir_mode = "shared-fs"`，不向 LXC 暴露 host
|
||||
`/dev/loop-control` 或 `/dev/loopN`。dockerd 镜像需要包含 `fuse-overlayfs`；启动前在
|
||||
Kata guest 内创建临时设备节点 `mknod /dev/fuse c 10 229`，随后强制传入
|
||||
`--storage-driver=fuse-overlayfs`。该 `/dev/fuse` 属于内层 guest kernel，不是 PVE
|
||||
host 设备透传，并随 Kata Pod 一起销毁。
|
||||
|
||||
删除三个 smoke Pod 后，LXC cgroup 统计约 505 MB anonymous memory、1.34 GB file
|
||||
cache；后者主要来自刚拉取的 Kata/Docker images，可由宿主回收。也就是说常驻 worker
|
||||
的不可回收 working set 约 500 MB，8 GiB 配置是 cgroup 上限而非预分配。
|
||||
|
||||
## Gitea ephemeral runner PoC
|
||||
|
||||
`runner/` 定义一次性 Gitea runner Pod。整个 Pod 使用 `runtimeClassName: kata`；
|
||||
runner 通过 guest 内同 Pod 的 dockerd 创建 job container，不连接 LXC/PVE host 的
|
||||
Docker socket。dockerd 使用 zot 中预先发布的 `dind-fuse:29.7.1`，其
|
||||
`/var/lib/docker` 是随 Kata Pod 删除的 `emptyDir`。
|
||||
|
||||
runner 以 `GITEA_RUNNER_EPHEMERAL=1` 注册，只接收一个 `runs-on: kata-poc` job；接单
|
||||
后 Gitea 撤销其 runner credential，job 完成后进程与 Kata VM 一起退出。当前 PoC
|
||||
手动创建一个 runner 等待 `.gitea/workflows/kata-runner-smoke.yml`，尚未部署
|
||||
`workflow_job` webhook controller，因此不会保留常驻空闲 runner。
|
||||
|
||||
注册 token 只在运行时从现有 OpenBao/ESO 消费副本投递到 PoC 集群的临时 Secret，
|
||||
不写入此目录。两个 Pod image 使用 `imagePullPolicy: Never`，部署前从可信工作站将
|
||||
已验证镜像导入 PoC k3s containerd；这是当前 zot 拉取仍要求 SPIFFE 身份时的
|
||||
bootstrap 边界，不是正式镜像分发方案。
|
||||
|
||||
正式接入 zot 前需要把这个 cluster 注册到 SPIRE，并为 runner Pod 创建专用
|
||||
`ClusterSPIFFEID`。届时 runner 与 dockerd sidecar 在同一路径只读挂载 CSI Workload
|
||||
API socket，runner 的 `container.options` 再把该路径 bind-mount 到 job container。
|
||||
zot 只对最终 SPIFFE ID 的 `panxiao81/dind-fuse` 仓库授予
|
||||
`read/create/update`；禁止授予 namespace 或整个 trust domain 写权限。
|
||||
|
||||
2026-09-14 已完成真实 Gitea job 验收：Actions run `242` / job `445` 使用
|
||||
`alpine:3.22`，10 秒成功；runner container 退出码为 `0`，原生 dockerd sidecar 随后
|
||||
由 kubelet 终止，Pod 最终为 `Succeeded`，没有保留空闲 Kata VM。guest 内核为
|
||||
`6.18.35`，dockerd `29.7.1` 使用 `fuse-overlayfs`。
|
||||
|
||||
首次尝试暴露了两个部署陷阱：`COPY` 默认保留源文件 mode，入口脚本原为 `0664`，
|
||||
必须在 Dockerfile 使用 `COPY --chmod=0755`；只配置 `ephemeral-storage: 20Gi` limit
|
||||
会形成同值 request，使 12 GiB PoC 节点无法调度。当前 manifest 显式 request 1 GiB、
|
||||
limit 8 GiB。完整 `ubuntu-latest` job image 也不适合这个小 rootfs，基础生命周期验收
|
||||
改用 Alpine；正式 worker 必须扩容本地 image/cache 空间或使用更精简的 job image。
|
||||
|
||||
## Cloud Hypervisor 内存记账验证
|
||||
|
||||
2026-09-17 在 pve2 的一次性 VMID 148 中安装 k3s v1.36.4+k3s1 与 Kata 4.1.0，
|
||||
只启用 `clh-runtime-rs`，验证 Cloud Hypervisor 在 privileged LXC 内不会重现早期
|
||||
microVM PoC 的 private memfd 双重记账问题。测试完成后已删除 VMID 148。
|
||||
|
||||
Cloud Hypervisor `/api/v1/vm.info` 报告：
|
||||
|
||||
```json
|
||||
{"size":3254779904,"shared":true,"hugepages":false,"prefault":false,"thp":true}
|
||||
```
|
||||
|
||||
Kata guest 在 memory-backed `emptyDir` 中写入 1 GiB 后：
|
||||
|
||||
- VMM `Pss_Shmem` 从约 247 MiB 增至约 1269 MiB；
|
||||
- VMM `Pss_Anon` 写入前后均约 1.2 MiB；
|
||||
- 外层 LXC `shmem` 增加约 1 GiB，`anon` 基本不变；
|
||||
- LXC `memory.events` 的 `max`、`oom` 与 `oom_kill` 均为 0；
|
||||
- 删除 Pod 后 VMM 退出，LXC `shmem` 回落到约 1.6 MiB。
|
||||
|
||||
因此 Kata 的 Cloud Hypervisor handler 已使用 shared memfd，等价于独立 launcher 的
|
||||
`--memory shared=on` 修复；后续 OpenSandbox/Kata 方案无需为 guest RAM 预留近似双倍
|
||||
的 LXC cgroup 内存。部署验收仍应检查 `vm.info.config.memory.shared=true`，防止上游
|
||||
配置或 runtime handler 变化造成回归。
|
||||
|
||||
测试期间还发现 pve2 `local-lvm` thin pool 已达到 100%；临时 rootfs 必须放到
|
||||
`pve-rg-hdd`，不能根据容器内部 `df` 的剩余空间判断 thin pool 是否可写。
|
||||
|
||||
## Block-backed emptyDir 与 CI 构建验证
|
||||
|
||||
2026-09-17 又在 pve2 的一次性 VMID 148 中验证 Kata 4.1.0
|
||||
`clh-runtime-rs` 的以下 drop-in：
|
||||
|
||||
```toml
|
||||
[runtime]
|
||||
emptydir_mode = "block-plain"
|
||||
```
|
||||
|
||||
结果如下：
|
||||
|
||||
- 普通 Kubernetes `emptyDir` 被创建为稀疏 `disk.img`，由 Cloud Hypervisor 直接作为
|
||||
block device 热插拔；guest 内 `/var/lib/docker` 是 `/dev/vdb` 上的 ext4。
|
||||
- Docker 29.1.5 成功强制使用原生 `overlay2`，不再需要 `fuse-overlayfs`；整个过程
|
||||
没有使用 PVE host 或 LXC 的 loop device。
|
||||
- `emptyDir.sizeLimit: 8Gi` 没有决定虚拟磁盘容量。由于 kubelet 所在文件系统约
|
||||
12 GiB，guest 看到的 ext4 约 11.7 GiB；这与 Kata 已知的 block-backed emptyDir
|
||||
限制一致。
|
||||
- 包含 256 MiB payload、2000 个小文件和一个额外复制层的冷 BuildKit 构建耗时
|
||||
46 秒；同一 LXC、同一 DRBD HDD 存储上的 runc 对照为 34 秒，Kata 约慢 35%。
|
||||
- 在 overlay2 容器层内写入并 fsync 512 MiB：Kata 为 12 秒（40.5 MB/s），runc 为
|
||||
10 秒（50.3 MB/s）；创建一万个小文件两者均约 2 秒。
|
||||
- kind v0.27.0 / Kubernetes v1.32.2 首次启动失败是因为 Kata guest 内的 dockerd
|
||||
容器没有 `/dev/kmsg`，不是 block storage 或 cgroup 故障。在 privileged dockerd
|
||||
启动前执行 `mknod /dev/kmsg c 1 11` 后，kind 创建成功：总计 122 秒，进入等待阶段
|
||||
后 23 秒 control-plane Ready，随后可正常删除。
|
||||
- 测试时稀疏 backing file 逻辑大小约 12 GiB，构建和 kind 后实际占用约 1.8 GiB。
|
||||
删除 Pod 后 backing file、Cloud Hypervisor 进程和 kind 容器全部消失；host/LXC
|
||||
均无 loop device 残留，LXC 没有 OOM 事件。
|
||||
|
||||
因此 `block-plain` 能解决 virtio-fs 不能作为 overlayfs upperdir 的问题，并满足
|
||||
BuildKit 与 kind 的功能要求。正式 CI RuntimeClass 必须使用独立的 block-backed
|
||||
配置，并在 dockerd bootstrap 中创建 guest `/dev/kmsg`。它仍比同机 runc 构建慢约
|
||||
20–35%，且 `emptyDir.sizeLimit` 不能可靠限制虚拟磁盘容量。PoC 曾据此建议为 kubelet
|
||||
目录提供独立文件系统；正式 sandbox 集群最终没有采用该设计：独立 volume 增加了
|
||||
DRBD 与 LXC 挂载生命周期复杂度，却没有隔离 containerd 等其他节点数据。正式节点
|
||||
改用单一、容量明确的 rootfs，并以磁盘占用监控、Pod 删除后的 backing-file 回收检查
|
||||
和实际构建基准作为上线门槛。
|
||||
@@ -1,10 +0,0 @@
|
||||
[defaults]
|
||||
inventory = inventory/hosts.yml
|
||||
host_key_checking = True
|
||||
interpreter_python = auto_silent
|
||||
retry_files_enabled = False
|
||||
local_tmp = /tmp/ansible-kata-lxc-local
|
||||
remote_tmp = /tmp/ansible-kata-lxc-remote
|
||||
|
||||
[ssh_connection]
|
||||
pipelining = True
|
||||
@@ -1,58 +0,0 @@
|
||||
---
|
||||
- name: Bootstrap k3s in the Kata LXC worker
|
||||
hosts: pve2
|
||||
gather_facts: false
|
||||
vars:
|
||||
kata_lxc_id: 147
|
||||
tasks:
|
||||
- name: Check base packages in LXC
|
||||
ansible.builtin.command:
|
||||
cmd: >-
|
||||
pct exec {{ kata_lxc_id }} -- dpkg-query -W
|
||||
ca-certificates curl jq openssh-server
|
||||
register: kata_lxc_packages
|
||||
changed_when: false
|
||||
failed_when: false
|
||||
|
||||
- name: Install base packages and SSH server in LXC
|
||||
ansible.builtin.command:
|
||||
cmd: >-
|
||||
pct exec {{ kata_lxc_id }} -- bash -lc
|
||||
'apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y
|
||||
ca-certificates curl jq openssh-server'
|
||||
when: kata_lxc_packages.rc != 0
|
||||
|
||||
- name: Install k3s in LXC
|
||||
ansible.builtin.shell:
|
||||
cmd: |
|
||||
set -euo pipefail
|
||||
pct exec {{ kata_lxc_id }} -- bash -lc '
|
||||
if ! command -v k3s >/dev/null; then
|
||||
curl -sfL https://get.k3s.io -o /tmp/install-k3s.sh
|
||||
chmod 0755 /tmp/install-k3s.sh
|
||||
INSTALL_K3S_EXEC="server --disable=traefik --write-kubeconfig-mode=0644" /tmp/install-k3s.sh
|
||||
fi
|
||||
'
|
||||
executable: /bin/bash
|
||||
register: kata_lxc_k3s_install
|
||||
changed_when: "'No change detected' not in kata_lxc_k3s_install.stdout"
|
||||
|
||||
- name: Wait for k3s node readiness
|
||||
ansible.builtin.command:
|
||||
cmd: >-
|
||||
pct exec {{ kata_lxc_id }} -- /usr/local/bin/k3s kubectl wait
|
||||
--for=condition=Ready node/kata-lxc-lab --timeout=180s
|
||||
changed_when: false
|
||||
|
||||
- name: Check LXC virtualization and devices
|
||||
ansible.builtin.command:
|
||||
cmd: >-
|
||||
pct exec {{ kata_lxc_id }} -- bash -lc
|
||||
'systemd-detect-virt; ls -l /dev/kvm /dev/vhost-net /dev/vhost-vsock;
|
||||
/usr/local/bin/k3s kubectl get node -o wide'
|
||||
register: kata_lxc_ready
|
||||
changed_when: false
|
||||
|
||||
- name: Report k3s readiness
|
||||
ansible.builtin.debug:
|
||||
var: kata_lxc_ready.stdout_lines
|
||||
@@ -1,137 +0,0 @@
|
||||
---
|
||||
- name: Create the disposable Kata LXC worker
|
||||
hosts: pve2
|
||||
gather_facts: false
|
||||
vars:
|
||||
kata_lxc_id: 147
|
||||
kata_lxc_template: laptop:vztmpl/ubuntu-24.04-standard_24.04-2_amd64.tar.zst
|
||||
tasks:
|
||||
- name: Check whether the LXC already exists
|
||||
ansible.builtin.stat:
|
||||
path: /etc/pve/lxc/{{ kata_lxc_id }}.conf
|
||||
register: kata_lxc_config
|
||||
|
||||
- name: Create privileged LXC with Kata host devices
|
||||
ansible.builtin.command:
|
||||
argv:
|
||||
- pct
|
||||
- create
|
||||
- "{{ kata_lxc_id }}"
|
||||
- "{{ kata_lxc_template }}"
|
||||
- --hostname
|
||||
- kata-lxc-lab
|
||||
- --ostype
|
||||
- ubuntu
|
||||
- --rootfs
|
||||
- local-lvm:12
|
||||
- --cores
|
||||
- "4"
|
||||
- --memory
|
||||
- "8192"
|
||||
- --swap
|
||||
- "0"
|
||||
- --net0
|
||||
- name=eth0,bridge=vmbr0,ip=dhcp
|
||||
- --features
|
||||
- nesting=1,keyctl=1,fuse=1,mknod=1,force_rw_sys=1
|
||||
- --unprivileged
|
||||
- "0"
|
||||
- --onboot
|
||||
- "0"
|
||||
- --ssh-public-keys
|
||||
- /root/.ssh/authorized_keys
|
||||
- --dev0
|
||||
- path=/dev/kvm,mode=0660
|
||||
- --dev1
|
||||
- path=/dev/vhost-net,mode=0660
|
||||
- --dev2
|
||||
- path=/dev/vhost-vsock,mode=0660
|
||||
- --dev3
|
||||
- path=/dev/kmsg,mode=0660
|
||||
- --dev4
|
||||
- path=/dev/net/tun,mode=0666
|
||||
- --tags
|
||||
- disposable;kata;lxc;ansible
|
||||
when: not kata_lxc_config.stat.exists
|
||||
|
||||
- name: Read LXC status
|
||||
ansible.builtin.command:
|
||||
cmd: pct status {{ kata_lxc_id }}
|
||||
register: kata_lxc_status
|
||||
changed_when: false
|
||||
|
||||
- name: Read current LXC configuration
|
||||
ansible.builtin.command:
|
||||
cmd: pct config {{ kata_lxc_id }}
|
||||
register: kata_lxc_current_config
|
||||
changed_when: false
|
||||
|
||||
- name: Ensure kubelet kernel log device is present
|
||||
ansible.builtin.command:
|
||||
cmd: pct set {{ kata_lxc_id }} --dev3 path=/dev/kmsg,mode=0660
|
||||
register: kata_lxc_kmsg
|
||||
when: "'/dev/kmsg' not in kata_lxc_current_config.stdout"
|
||||
|
||||
- name: Enable writable sysctls required by kubelet
|
||||
ansible.builtin.command:
|
||||
cmd: >-
|
||||
pct set {{ kata_lxc_id }} --features
|
||||
nesting=1,keyctl=1,fuse=1,mknod=1,force_rw_sys=1
|
||||
register: kata_lxc_rw_sys
|
||||
when: "'force_rw_sys=1' not in kata_lxc_current_config.stdout"
|
||||
|
||||
- name: Ensure TUN device required by Kata networking is present
|
||||
ansible.builtin.command:
|
||||
cmd: pct set {{ kata_lxc_id }} --dev4 path=/dev/net/tun,mode=0666
|
||||
register: kata_lxc_tun
|
||||
when: "'/dev/net/tun' not in kata_lxc_current_config.stdout"
|
||||
|
||||
- name: Configure privileged nested-runtime LXC directives
|
||||
ansible.builtin.lineinfile:
|
||||
path: /etc/pve/lxc/{{ kata_lxc_id }}.conf
|
||||
regexp: "^{{ item.key | regex_escape }}:"
|
||||
line: "{{ item.key }}: {{ item.value }}"
|
||||
loop:
|
||||
- key: lxc.apparmor.profile
|
||||
value: unconfined
|
||||
- key: lxc.cgroup2.devices.allow
|
||||
value: a
|
||||
- key: lxc.cap.drop
|
||||
value: ""
|
||||
- key: lxc.mount.auto
|
||||
value: proc:rw sys:rw
|
||||
register: kata_lxc_raw_config
|
||||
|
||||
- name: Restart LXC after device configuration change
|
||||
ansible.builtin.command:
|
||||
cmd: pct reboot {{ kata_lxc_id }}
|
||||
when: >-
|
||||
kata_lxc_kmsg.changed or kata_lxc_rw_sys.changed or kata_lxc_tun.changed or
|
||||
kata_lxc_raw_config.changed
|
||||
|
||||
- name: Start LXC
|
||||
ansible.builtin.command:
|
||||
cmd: pct start {{ kata_lxc_id }}
|
||||
when: "'status: stopped' in kata_lxc_status.stdout"
|
||||
|
||||
- name: Wait for systemd in LXC
|
||||
ansible.builtin.command:
|
||||
cmd: pct exec {{ kata_lxc_id }} -- systemctl is-system-running --wait
|
||||
register: kata_lxc_systemd
|
||||
retries: 30
|
||||
delay: 2
|
||||
until: kata_lxc_systemd.rc in [0, 1]
|
||||
changed_when: false
|
||||
failed_when: kata_lxc_systemd.rc not in [0, 1]
|
||||
|
||||
- name: Show LXC address and Kata devices
|
||||
ansible.builtin.command:
|
||||
cmd: >-
|
||||
pct exec {{ kata_lxc_id }} -- sh -c
|
||||
'hostname -I; ls -l /dev/kvm /dev/vhost-net /dev/vhost-vsock /dev/net/tun'
|
||||
register: kata_lxc_facts
|
||||
changed_when: false
|
||||
|
||||
- name: Report LXC address and devices
|
||||
ansible.builtin.debug:
|
||||
var: kata_lxc_facts.stdout_lines
|
||||
@@ -1,9 +0,0 @@
|
||||
---
|
||||
all:
|
||||
hosts:
|
||||
pve2:
|
||||
ansible_host: 192.168.10.7
|
||||
ansible_user: root
|
||||
kata-lab:
|
||||
ansible_host: 192.168.10.128
|
||||
ansible_user: root
|
||||
@@ -1,3 +0,0 @@
|
||||
---
|
||||
# Reuse the proven Kata 4.1.0 installation playbook against the LXC inventory.
|
||||
- import_playbook: ../../kata-lab/ansible/kata.yml
|
||||
@@ -1,73 +0,0 @@
|
||||
---
|
||||
- name: Verify fuse-overlayfs for Docker inside Kata
|
||||
hosts: kata-lab
|
||||
gather_facts: false
|
||||
tasks:
|
||||
- name: Remove an earlier fuse-overlayfs smoke Pod
|
||||
ansible.builtin.command:
|
||||
cmd: k3s kubectl delete pod kata-fuse-smoke --ignore-not-found --wait=true
|
||||
changed_when: false
|
||||
|
||||
- name: Create Kata Docker Pod using fuse-overlayfs
|
||||
ansible.builtin.shell:
|
||||
cmd: |
|
||||
set -o pipefail
|
||||
k3s kubectl apply -f - <<'EOF'
|
||||
apiVersion: v1
|
||||
kind: Pod
|
||||
metadata:
|
||||
name: kata-fuse-smoke
|
||||
spec:
|
||||
runtimeClassName: kata
|
||||
restartPolicy: Never
|
||||
containers:
|
||||
- name: dockerd
|
||||
image: docker.io/library/docker:27-dind
|
||||
securityContext:
|
||||
privileged: true
|
||||
env:
|
||||
- name: DOCKER_TLS_CERTDIR
|
||||
value: ""
|
||||
command:
|
||||
- sh
|
||||
- -c
|
||||
- |
|
||||
apk add --no-cache fuse-overlayfs
|
||||
test -e /dev/fuse || mknod /dev/fuse c 10 229
|
||||
chmod 0666 /dev/fuse
|
||||
exec dockerd-entrypoint.sh --storage-driver=fuse-overlayfs
|
||||
- name: client
|
||||
image: docker.io/library/docker:27-cli
|
||||
env:
|
||||
- name: DOCKER_HOST
|
||||
value: tcp://127.0.0.1:2375
|
||||
command:
|
||||
- sh
|
||||
- -c
|
||||
- |
|
||||
until docker info >/dev/null 2>&1; do sleep 1; done
|
||||
docker info --format 'storage-driver={{ "{{" }}.Driver{{ "}}" }}'
|
||||
docker run --rm docker.io/library/busybox:1.37 echo fuse-nested-docker-ok
|
||||
EOF
|
||||
executable: /bin/bash
|
||||
changed_when: true
|
||||
|
||||
- name: Wait for the Docker client to finish
|
||||
ansible.builtin.command:
|
||||
cmd: >-
|
||||
k3s kubectl wait --for=jsonpath='{.status.containerStatuses[?(@.name=="client")].state.terminated.exitCode}'=0
|
||||
pod/kata-fuse-smoke --timeout=10m
|
||||
changed_when: false
|
||||
|
||||
- name: Read Docker client result
|
||||
ansible.builtin.command:
|
||||
cmd: k3s kubectl logs kata-fuse-smoke -c client
|
||||
register: kata_fuse_smoke_log
|
||||
changed_when: false
|
||||
|
||||
- name: Require fuse-overlayfs and nested Docker
|
||||
ansible.builtin.assert:
|
||||
that:
|
||||
- "'storage-driver=fuse-overlayfs' in kata_fuse_smoke_log.stdout"
|
||||
- "'fuse-nested-docker-ok' in kata_fuse_smoke_log.stdout"
|
||||
success_msg: Docker used fuse-overlayfs and completed a nested container inside Kata
|
||||
@@ -1,3 +0,0 @@
|
||||
---
|
||||
# Reuse the guest-kernel and Docker sidecar smoke tests against the LXC worker.
|
||||
- import_playbook: ../../kata-lab/ansible/verify.yml
|
||||
@@ -1,35 +0,0 @@
|
||||
apiVersion: v1
|
||||
kind: Namespace
|
||||
metadata:
|
||||
name: gitea-actions
|
||||
labels:
|
||||
pod-security.kubernetes.io/enforce: privileged
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: ConfigMap
|
||||
metadata:
|
||||
name: kata-ephemeral-runner-config
|
||||
namespace: gitea-actions
|
||||
data:
|
||||
config.yaml: |
|
||||
log:
|
||||
level: info
|
||||
runner:
|
||||
file: /runner-state/.runner
|
||||
capacity: 1
|
||||
envs:
|
||||
DOCKER_HOST: tcp://host.docker.internal:2375
|
||||
timeout: 3h
|
||||
shutdown_timeout: 5m
|
||||
labels:
|
||||
- kata-poc:docker://docker.io/library/alpine:3.22
|
||||
cache:
|
||||
enabled: false
|
||||
container:
|
||||
network: ""
|
||||
options: --add-host=host.docker.internal:host-gateway
|
||||
docker_host: tcp://127.0.0.1:2375
|
||||
require_docker: true
|
||||
docker_timeout: 5m
|
||||
force_pull: true
|
||||
bind_workdir: false
|
||||
@@ -1,111 +0,0 @@
|
||||
apiVersion: v1
|
||||
kind: Pod
|
||||
metadata:
|
||||
generateName: kata-ephemeral-runner-
|
||||
namespace: gitea-actions
|
||||
labels:
|
||||
app.kubernetes.io/name: kata-ephemeral-runner
|
||||
spec:
|
||||
runtimeClassName: kata
|
||||
restartPolicy: Never
|
||||
terminationGracePeriodSeconds: 330
|
||||
automountServiceAccountToken: false
|
||||
initContainers:
|
||||
# Native sidecar: startupProbe gates the runner, and kubelet terminates this
|
||||
# container after the ephemeral runner exits so the Kata sandbox can end.
|
||||
- name: docker
|
||||
image: zot.ad.ddupan.top/panxiao81/dind-fuse:29.7.1
|
||||
imagePullPolicy: Never
|
||||
restartPolicy: Always
|
||||
env:
|
||||
- name: DOCKER_TLS_CERTDIR
|
||||
value: ""
|
||||
- name: DIND_LOOPBACK_SIZE
|
||||
value: 6G
|
||||
- name: DIND_LOOPBACK_FILE
|
||||
value: /loopback/docker.img
|
||||
args:
|
||||
- --host=tcp://0.0.0.0:2375
|
||||
- --tls=false
|
||||
- --mtu=1450
|
||||
- --default-network-opt=bridge=com.docker.network.driver.mtu=1450
|
||||
securityContext:
|
||||
privileged: true
|
||||
resources:
|
||||
requests:
|
||||
cpu: 250m
|
||||
memory: 512Mi
|
||||
ephemeral-storage: 1Gi
|
||||
limits:
|
||||
cpu: "4"
|
||||
memory: 6Gi
|
||||
ephemeral-storage: 8Gi
|
||||
volumeMounts:
|
||||
- name: docker-data
|
||||
mountPath: /var/lib/docker
|
||||
- name: loopback-data
|
||||
mountPath: /loopback
|
||||
startupProbe:
|
||||
tcpSocket:
|
||||
port: 2375
|
||||
periodSeconds: 2
|
||||
failureThreshold: 150
|
||||
readinessProbe:
|
||||
tcpSocket:
|
||||
port: 2375
|
||||
periodSeconds: 5
|
||||
containers:
|
||||
- name: runner
|
||||
image: docker.io/gitea/runner:2.3.0
|
||||
imagePullPolicy: Never
|
||||
env:
|
||||
- name: CONFIG_FILE
|
||||
value: /config/config.yaml
|
||||
- name: DOCKER_HOST
|
||||
value: tcp://127.0.0.1:2375
|
||||
- name: GITEA_INSTANCE_URL
|
||||
value: https://git.ddupan.top
|
||||
- name: GITEA_RUNNER_EPHEMERAL
|
||||
value: "1"
|
||||
- name: GITEA_RUNNER_NAME
|
||||
valueFrom:
|
||||
fieldRef:
|
||||
fieldPath: metadata.name
|
||||
- name: GITEA_RUNNER_LABELS
|
||||
value: kata-poc:docker://docker.io/library/alpine:3.22
|
||||
- name: GITEA_RUNNER_REGISTRATION_TOKEN_FILE
|
||||
value: /registration/token
|
||||
resources:
|
||||
requests:
|
||||
cpu: 100m
|
||||
memory: 128Mi
|
||||
limits:
|
||||
cpu: "1"
|
||||
memory: 1Gi
|
||||
volumeMounts:
|
||||
- name: config
|
||||
mountPath: /config
|
||||
readOnly: true
|
||||
- name: registration
|
||||
mountPath: /registration
|
||||
readOnly: true
|
||||
- name: runner-state
|
||||
mountPath: /runner-state
|
||||
volumes:
|
||||
- name: config
|
||||
configMap:
|
||||
name: kata-ephemeral-runner-config
|
||||
- name: registration
|
||||
secret:
|
||||
secretName: gitea-runner-registration
|
||||
- name: runner-state
|
||||
emptyDir:
|
||||
medium: Memory
|
||||
sizeLimit: 16Mi
|
||||
- name: docker-data
|
||||
emptyDir:
|
||||
sizeLimit: 8Gi
|
||||
- name: loopback-data
|
||||
emptyDir:
|
||||
medium: Memory
|
||||
sizeLimit: 6Gi
|
||||
@@ -1,24 +0,0 @@
|
||||
# This file is maintained automatically by "terraform init".
|
||||
# Manual edits may be lost in future updates.
|
||||
|
||||
provider "registry.terraform.io/bpg/proxmox" {
|
||||
version = "0.111.1"
|
||||
constraints = "0.111.1"
|
||||
hashes = [
|
||||
"h1:ML2D3UUZTM99yrll/EBXj7wBYMb8xmQgomqFNybEoxY=",
|
||||
"zh:18fb7c31a08dde6bffa1a4d4a211e604d6d17eec7092fd59331b3db3c6f3742c",
|
||||
"zh:1cd60761538289d4dd2a1086b3ae62a7b0bdd4b1a2f824e9a44e243413168dba",
|
||||
"zh:2eb76f6fc8299b6820ff678c8252332cc3366e226b5ae2e61748fd2449c1ed92",
|
||||
"zh:45e6f7ebd0bf48911d37060359a4f359b5743b3092e985295733990e406d0416",
|
||||
"zh:4aa8ba912eae37975d2e983394d173e595ca34fc76b5bf220b37d0e99d76e98c",
|
||||
"zh:58e0789923103a77d502a0a9fc3eb920625e8eb935ec2d4ac0d006aebd1d186c",
|
||||
"zh:6df8aa85fb8865915537e946c19b02538ad188018a629759c213c6f03730f642",
|
||||
"zh:6ed47bc00d0913a1d0880618fa1376115e9edab6b4a658c081061a7f0e4ca360",
|
||||
"zh:c5b10ff4f33df7e4c29e8f1127d49845b561b37b57517e844fb0954d7923d65e",
|
||||
"zh:d016510e14b738499f0db9d9b3aafe82fc6877fb4ab4e9f831fb68a8d70a1385",
|
||||
"zh:d941f394069bbf24351b363da1c64383f487067aaee0a84f9b96476d4912e212",
|
||||
"zh:ddf271dbc2632ae8ffa8de3972f243ee47d260cb2ac90aa784f2746d98e21a0f",
|
||||
"zh:ed0caa3501c42f611b7e9622c9b1df69fd85dc25a3cd88d3076381829688cd62",
|
||||
"zh:f26e0763dbe6a6b2195c94b44696f2110f7f55433dc142839be16b9697fa5597",
|
||||
]
|
||||
}
|
||||
@@ -1,95 +0,0 @@
|
||||
locals {
|
||||
ssh_public_key = trimspace(file(pathexpand(var.ssh_public_key_file)))
|
||||
}
|
||||
|
||||
data "proxmox_file" "ubuntu_noble" {
|
||||
content_type = "vztmpl"
|
||||
datastore_id = var.template_datastore
|
||||
node_name = var.proxmox_node
|
||||
file_name = var.template_file_name
|
||||
}
|
||||
|
||||
resource "proxmox_virtual_environment_container" "kata_lxc_lab" {
|
||||
node_name = var.proxmox_node
|
||||
vm_id = var.container_id
|
||||
|
||||
description = "Disposable privileged LXC for validating k3s and Kata with direct host KVM access."
|
||||
unprivileged = false
|
||||
started = true
|
||||
start_on_boot = false
|
||||
tags = ["disposable", "kata", "lxc", "terraform"]
|
||||
|
||||
cpu {
|
||||
cores = 4
|
||||
}
|
||||
|
||||
memory {
|
||||
dedicated = 8192
|
||||
swap = 0
|
||||
}
|
||||
|
||||
disk {
|
||||
datastore_id = var.root_datastore
|
||||
size = 16
|
||||
}
|
||||
|
||||
features {
|
||||
fuse = true
|
||||
keyctl = true
|
||||
mknod = true
|
||||
nesting = true
|
||||
}
|
||||
|
||||
device_passthrough {
|
||||
path = "/dev/kvm"
|
||||
mode = "0660"
|
||||
}
|
||||
|
||||
device_passthrough {
|
||||
path = "/dev/vhost-net"
|
||||
mode = "0660"
|
||||
}
|
||||
|
||||
device_passthrough {
|
||||
path = "/dev/vhost-vsock"
|
||||
mode = "0660"
|
||||
}
|
||||
|
||||
device_passthrough {
|
||||
path = "/dev/kmsg"
|
||||
mode = "0660"
|
||||
}
|
||||
|
||||
device_passthrough {
|
||||
path = "/dev/net/tun"
|
||||
mode = "0666"
|
||||
}
|
||||
|
||||
initialization {
|
||||
hostname = var.container_name
|
||||
|
||||
ip_config {
|
||||
ipv4 {
|
||||
address = "dhcp"
|
||||
}
|
||||
}
|
||||
|
||||
user_account {
|
||||
keys = [local.ssh_public_key]
|
||||
}
|
||||
}
|
||||
|
||||
network_interface {
|
||||
name = "eth0"
|
||||
bridge = "vmbr0"
|
||||
}
|
||||
|
||||
operating_system {
|
||||
template_file_id = data.proxmox_file.ubuntu_noble.id
|
||||
type = "ubuntu"
|
||||
}
|
||||
|
||||
wait_for_ip {
|
||||
ipv4 = true
|
||||
}
|
||||
}
|
||||
@@ -1,7 +0,0 @@
|
||||
output "container_id" {
|
||||
value = proxmox_virtual_environment_container.kata_lxc_lab.vm_id
|
||||
}
|
||||
|
||||
output "ipv4" {
|
||||
value = proxmox_virtual_environment_container.kata_lxc_lab.ipv4
|
||||
}
|
||||
@@ -1,15 +0,0 @@
|
||||
provider "proxmox" {
|
||||
endpoint = var.proxmox_endpoint
|
||||
api_token = var.proxmox_api_token
|
||||
|
||||
ssh {
|
||||
agent = false
|
||||
username = "root"
|
||||
private_key = file(pathexpand(var.proxmox_ssh_private_key_file))
|
||||
|
||||
node {
|
||||
name = var.proxmox_node
|
||||
address = var.proxmox_node_address
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,54 +0,0 @@
|
||||
variable "proxmox_endpoint" {
|
||||
type = string
|
||||
default = "https://pve1.ad.ddupan.top:8006"
|
||||
}
|
||||
|
||||
variable "proxmox_api_token" {
|
||||
type = string
|
||||
sensitive = true
|
||||
}
|
||||
|
||||
variable "proxmox_node" {
|
||||
type = string
|
||||
default = "pve2"
|
||||
}
|
||||
|
||||
variable "proxmox_node_address" {
|
||||
type = string
|
||||
default = "192.168.10.7"
|
||||
}
|
||||
|
||||
variable "proxmox_ssh_private_key_file" {
|
||||
type = string
|
||||
default = "~/.ssh/id_ed25519"
|
||||
}
|
||||
|
||||
variable "ssh_public_key_file" {
|
||||
type = string
|
||||
default = "~/.ssh/id_ed25519.pub"
|
||||
}
|
||||
|
||||
variable "container_id" {
|
||||
type = number
|
||||
default = 147
|
||||
}
|
||||
|
||||
variable "container_name" {
|
||||
type = string
|
||||
default = "kata-lxc-lab"
|
||||
}
|
||||
|
||||
variable "template_datastore" {
|
||||
type = string
|
||||
default = "laptop"
|
||||
}
|
||||
|
||||
variable "template_file_name" {
|
||||
type = string
|
||||
default = "ubuntu-24.04-standard_24.04-2_amd64.tar.zst"
|
||||
}
|
||||
|
||||
variable "root_datastore" {
|
||||
type = string
|
||||
default = "local-lvm"
|
||||
}
|
||||
@@ -1,10 +0,0 @@
|
||||
terraform {
|
||||
required_version = ">= 1.10"
|
||||
|
||||
required_providers {
|
||||
proxmox = {
|
||||
source = "bpg/proxmox"
|
||||
version = "0.111.1"
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,53 +0,0 @@
|
||||
# Gitea kind microVM runner
|
||||
|
||||
`kind-microvm` 是专门运行 kind / nested Kubernetes CI 的 runner label。每个 job
|
||||
独占一台 Cloud Hypervisor VM；guest 内直接运行 Docker 与 ephemeral Gitea Runner，
|
||||
不经过 Kata。VM 完成一个 job 后关机，临时 COW 磁盘随即删除。
|
||||
|
||||
集群内的 `controller.py` 消费 Gitea `workflow_job` webhook。仅当
|
||||
`action=queued` 且 `workflow_job.labels` 包含 `kind-microvm` 时，向 NATS JetStream
|
||||
的 `ci.runner.kind-microvm` subject 发布消息。`workflow_job.id` 写入
|
||||
`Nats-Msg-Id`，重复 webhook 不会生成第二个任务。
|
||||
|
||||
pve2 LXC 内的 `worker.py` 使用 durable pull consumer 领取消息。领取后启动一台
|
||||
microVM，运行期间每分钟发送 `InProgress`；VM/ephemeral runner 正常退出才 ACK，
|
||||
启动失败则 NAK 并延迟重试。当前 `RUNNER_CAPACITY=1` 只是 pve2 的资源配置；以后可
|
||||
提高单 worker capacity，或在其他宿主机增加共享同一 durable consumer 的 worker，
|
||||
而无需修改 webhook/controller。相同 runner 类型必须共享 durable；WorkQueue stream
|
||||
不允许多个 consumer 使用重叠的 subject filter。
|
||||
|
||||
worker 的凭据位于：
|
||||
|
||||
```text
|
||||
/etc/microvm-runner/registration-token
|
||||
/etc/microvm-runner/nats-password
|
||||
```
|
||||
|
||||
二者必须为 root-only 文件，不能写入镜像、cloud-init seed 或仓库。worker 为每次
|
||||
启动生成一次性 nonce；guest 只能从 TAP 网段请求一次 registration token，请求后
|
||||
立即从 worker 内存删除。runner 注册成功后删除 guest 内的 token 文件，再启动
|
||||
daemon。这样 job 无法从 seed disk 或 Docker inspect 取回可复用 registration token。
|
||||
|
||||
NATS stream 使用 `WorkQueuePolicy`，ACK 后立即删除消息；同时限制为 24 小时、
|
||||
10000 条和 256 MiB，异常积压也不会无限增长。NATS 平台配置见
|
||||
`../../platform/nats/`。
|
||||
|
||||
2026-09-16 PoC 已验证：Cloud Hypervisor v52.0、Ubuntu 24.04、4 vCPU、3 GiB RAM、
|
||||
10 GiB COW disk 中，Docker 29.1.3 成功创建 kind v0.27.0 / Kubernetes v1.32.2
|
||||
集群，并运行 `busybox:1.37` smoke Pod。冷启动约 20 秒。pve2 资源只允许一台：
|
||||
VMM cgroup（guest RAM 与 page cache）峰值约 6.2 GiB。
|
||||
|
||||
正式启用前还需要安装 launcher、制作不含凭据的基础镜像、部署 controller，并在
|
||||
Gitea 配置只发送 `workflow_job` 的 instance/organization webhook。workflow 使用：
|
||||
|
||||
```yaml
|
||||
runs-on: kind-microvm
|
||||
```
|
||||
|
||||
当前 pve2 配置：
|
||||
|
||||
```ini
|
||||
RUNNER_CAPACITY=1
|
||||
NATS_DURABLE=kind-microvm
|
||||
RUNNER_MAX_INFLIGHT=64
|
||||
```
|
||||
@@ -1,113 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Gitea workflow_job webhook to NATS JetStream producer."""
|
||||
|
||||
import hashlib
|
||||
import hmac
|
||||
import json
|
||||
import os
|
||||
import ssl
|
||||
from pathlib import Path
|
||||
|
||||
import nats
|
||||
from aiohttp import web
|
||||
from nats.js.api import DiscardPolicy, RetentionPolicy, StorageType, StreamConfig
|
||||
from nats.js.errors import NotFoundError
|
||||
|
||||
LABEL = os.environ.get("RUNNER_LABEL", "kind-microvm")
|
||||
SUBJECT = os.environ.get("NATS_SUBJECT", f"ci.runner.{LABEL}")
|
||||
STREAM = os.environ.get("NATS_STREAM", "CI_RUNNER")
|
||||
NATS_URL = os.environ.get("NATS_URL", "tls://nats.nats.svc.cluster.local:4222")
|
||||
NATS_USER = os.environ.get("NATS_USER", "ci-producer")
|
||||
NATS_PASSWORD_FILE = Path(os.environ.get("NATS_PASSWORD_FILE", "/run/secrets/nats/password"))
|
||||
NATS_CA_FILE = os.environ.get("NATS_CA_FILE", "/etc/ssl/certs/ca-certificates.crt")
|
||||
WEBHOOK_SECRET_FILE = Path(os.environ.get("WEBHOOK_SECRET_FILE", "/run/secrets/gitea/webhook-secret"))
|
||||
|
||||
|
||||
def accepts(payload: object) -> tuple[bool, str | None]:
|
||||
if not isinstance(payload, dict) or payload.get("action") != "queued":
|
||||
return False, None
|
||||
job = payload.get("workflow_job")
|
||||
if not isinstance(job, dict) or LABEL not in job.get("labels", []):
|
||||
return False, None
|
||||
job_id = job.get("id")
|
||||
if not isinstance(job_id, int):
|
||||
return False, None
|
||||
return True, str(job_id)
|
||||
|
||||
|
||||
def valid_signature(body: bytes, signature: str) -> bool:
|
||||
expected = hmac.new(WEBHOOK_SECRET_FILE.read_bytes().strip(), body, hashlib.sha256).hexdigest()
|
||||
return hmac.compare_digest(signature, expected)
|
||||
|
||||
|
||||
async def ensure_stream(js: object) -> None:
|
||||
config = StreamConfig(
|
||||
name=STREAM,
|
||||
subjects=["ci.runner.*"],
|
||||
retention=RetentionPolicy.WORK_QUEUE,
|
||||
storage=StorageType.FILE,
|
||||
discard=DiscardPolicy.OLD,
|
||||
max_age=24 * 60 * 60,
|
||||
max_msgs=10_000,
|
||||
max_bytes=256 * 1024 * 1024,
|
||||
duplicate_window=24 * 60 * 60,
|
||||
)
|
||||
try:
|
||||
await js.stream_info(STREAM)
|
||||
except NotFoundError:
|
||||
await js.add_stream(config=config)
|
||||
else:
|
||||
await js.update_stream(config=config)
|
||||
|
||||
|
||||
async def webhook(request: web.Request) -> web.Response:
|
||||
body = await request.read()
|
||||
if not valid_signature(body, request.headers.get("X-Gitea-Signature", "")):
|
||||
raise web.HTTPUnauthorized()
|
||||
try:
|
||||
payload = json.loads(body)
|
||||
except json.JSONDecodeError as error:
|
||||
raise web.HTTPBadRequest(text="invalid JSON\n") from error
|
||||
accepted, job_id = accepts(payload)
|
||||
if not accepted:
|
||||
return web.Response(status=204)
|
||||
await request.app["js"].publish(
|
||||
SUBJECT,
|
||||
body,
|
||||
headers={"Nats-Msg-Id": f"gitea-workflow-job-{job_id}"},
|
||||
)
|
||||
return web.Response(status=202, text="queued\n")
|
||||
|
||||
|
||||
async def health(request: web.Request) -> web.Response:
|
||||
return web.Response(text="ok\n" if request.app["nc"].is_connected else "disconnected\n",
|
||||
status=200 if request.app["nc"].is_connected else 503)
|
||||
|
||||
|
||||
async def nats_context(app: web.Application):
|
||||
tls = ssl.create_default_context(cafile=NATS_CA_FILE)
|
||||
nc = await nats.connect(
|
||||
NATS_URL,
|
||||
user=NATS_USER,
|
||||
password=NATS_PASSWORD_FILE.read_text().strip(),
|
||||
tls=tls,
|
||||
name="microvm-runner-controller",
|
||||
)
|
||||
app["nc"] = nc
|
||||
app["js"] = nc.jetstream()
|
||||
await ensure_stream(app["js"])
|
||||
yield
|
||||
await nc.drain()
|
||||
|
||||
|
||||
def create_app() -> web.Application:
|
||||
app = web.Application(client_max_size=1024 * 1024)
|
||||
app.cleanup_ctx.append(nats_context)
|
||||
app.router.add_post("/webhook", webhook)
|
||||
app.router.add_get("/healthz", health)
|
||||
return app
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
web.run_app(create_app(), host=os.environ.get("LISTEN", "0.0.0.0"),
|
||||
port=int(os.environ.get("PORT", "8787")))
|
||||
@@ -1,19 +0,0 @@
|
||||
[Unit]
|
||||
Description=Gitea kind microVM runner worker
|
||||
After=network-online.target
|
||||
Wants=network-online.target
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
EnvironmentFile=-/etc/microvm-runner/worker.env
|
||||
ExecStart=/opt/microvm-runner/venv/bin/python /opt/microvm-runner/worker.py
|
||||
Restart=on-failure
|
||||
RestartSec=5s
|
||||
SupplementaryGroups=kvm
|
||||
PrivateTmp=yes
|
||||
ProtectHome=yes
|
||||
ProtectSystem=strict
|
||||
ReadWritePaths=/var/lib/microvm-runner /run/microvm-runner
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
@@ -1,2 +0,0 @@
|
||||
aiohttp==3.12.15
|
||||
nats-py==2.11.0
|
||||
@@ -1,138 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Capacity-bounded JetStream consumer that launches one ephemeral VM per job."""
|
||||
|
||||
import asyncio
|
||||
import os
|
||||
import secrets
|
||||
import ssl
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
|
||||
import nats
|
||||
from aiohttp import web
|
||||
from nats.errors import TimeoutError
|
||||
from nats.js.api import AckPolicy, ConsumerConfig
|
||||
|
||||
CAPACITY = int(os.environ.get("RUNNER_CAPACITY", "1"))
|
||||
SUBJECT = os.environ.get("NATS_SUBJECT", "ci.runner.kind-microvm")
|
||||
STREAM = os.environ.get("NATS_STREAM", "CI_RUNNER")
|
||||
DURABLE = os.environ.get("NATS_DURABLE", "kind-microvm")
|
||||
MAX_INFLIGHT = int(os.environ.get("RUNNER_MAX_INFLIGHT", "64"))
|
||||
NATS_URL = os.environ.get("NATS_URL", "tls://nats.ad.ddupan.top:4222")
|
||||
NATS_USER = os.environ.get("NATS_USER", "ci-worker")
|
||||
NATS_PASSWORD_FILE = Path(os.environ.get("NATS_PASSWORD_FILE", "/etc/microvm-runner/nats-password"))
|
||||
NATS_CA_FILE = os.environ.get("NATS_CA_FILE", "/etc/ssl/certs/ca-certificates.crt")
|
||||
REGISTRATION_TOKEN_FILE = Path(os.environ.get("REGISTRATION_TOKEN_FILE", "/etc/microvm-runner/registration-token"))
|
||||
LAUNCHER = os.environ.get("LAUNCHER", "/usr/local/libexec/microvm-runner-launch")
|
||||
TOKEN_LISTEN = os.environ.get("TOKEN_LISTEN", "172.30.0.1")
|
||||
TOKEN_PORT = int(os.environ.get("TOKEN_PORT", "8787"))
|
||||
|
||||
tokens: dict[str, bytes] = {}
|
||||
token_lock = asyncio.Lock()
|
||||
|
||||
|
||||
async def token(request: web.Request) -> web.Response:
|
||||
nonce = request.match_info["nonce"]
|
||||
async with token_lock:
|
||||
value = tokens.pop(nonce, None)
|
||||
if value is None:
|
||||
raise web.HTTPNotFound()
|
||||
return web.Response(body=value, headers={"Cache-Control": "no-store"})
|
||||
|
||||
|
||||
async def heartbeat(message: object, stop: asyncio.Event) -> None:
|
||||
while True:
|
||||
try:
|
||||
await asyncio.wait_for(stop.wait(), timeout=60)
|
||||
return
|
||||
except asyncio.TimeoutError:
|
||||
await message.in_progress()
|
||||
|
||||
|
||||
async def run_one(message: object) -> None:
|
||||
instance_id = str(uuid.uuid4())
|
||||
nonce = secrets.token_urlsafe(32)
|
||||
async with token_lock:
|
||||
tokens[nonce] = REGISTRATION_TOKEN_FILE.read_bytes().strip()
|
||||
stop = asyncio.Event()
|
||||
pulse = asyncio.create_task(heartbeat(message, stop))
|
||||
try:
|
||||
process = await asyncio.create_subprocess_exec(LAUNCHER, instance_id, nonce)
|
||||
return_code = await process.wait()
|
||||
if return_code == 0:
|
||||
await message.ack()
|
||||
else:
|
||||
await message.nak(delay=30)
|
||||
except Exception:
|
||||
await message.nak(delay=30)
|
||||
raise
|
||||
finally:
|
||||
stop.set()
|
||||
await pulse
|
||||
async with token_lock:
|
||||
tokens.pop(nonce, None)
|
||||
|
||||
|
||||
async def consume() -> None:
|
||||
if CAPACITY < 1:
|
||||
raise ValueError("RUNNER_CAPACITY must be at least 1")
|
||||
tls = ssl.create_default_context(cafile=NATS_CA_FILE)
|
||||
nc = await nats.connect(
|
||||
NATS_URL,
|
||||
user=NATS_USER,
|
||||
password=NATS_PASSWORD_FILE.read_text().strip(),
|
||||
tls=tls,
|
||||
name=DURABLE,
|
||||
)
|
||||
js = nc.jetstream()
|
||||
subscription = await js.pull_subscribe(
|
||||
SUBJECT,
|
||||
durable=DURABLE,
|
||||
stream=STREAM,
|
||||
config=ConsumerConfig(
|
||||
durable_name=DURABLE,
|
||||
filter_subject=SUBJECT,
|
||||
ack_policy=AckPolicy.EXPLICIT,
|
||||
ack_wait=5 * 60,
|
||||
# This durable consumer is shared by every host of this runner type.
|
||||
# Local CAPACITY controls each host; this is only a global safety cap.
|
||||
max_ack_pending=MAX_INFLIGHT,
|
||||
max_deliver=5,
|
||||
),
|
||||
)
|
||||
active: set[asyncio.Task[None]] = set()
|
||||
try:
|
||||
while True:
|
||||
active = {task for task in active if not task.done()}
|
||||
free = CAPACITY - len(active)
|
||||
if free == 0:
|
||||
await asyncio.wait(active, return_when=asyncio.FIRST_COMPLETED)
|
||||
continue
|
||||
try:
|
||||
messages = await subscription.fetch(batch=free, timeout=5)
|
||||
except TimeoutError:
|
||||
continue
|
||||
for message in messages:
|
||||
task = asyncio.create_task(run_one(message))
|
||||
task.add_done_callback(lambda done: done.exception())
|
||||
active.add(task)
|
||||
finally:
|
||||
if active:
|
||||
await asyncio.gather(*active, return_exceptions=True)
|
||||
await nc.drain()
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
app = web.Application()
|
||||
app.router.add_get("/token/{nonce}", token)
|
||||
runner = web.AppRunner(app)
|
||||
await runner.setup()
|
||||
await web.TCPSite(runner, TOKEN_LISTEN, TOKEN_PORT).start()
|
||||
try:
|
||||
await consume()
|
||||
finally:
|
||||
await runner.cleanup()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
@@ -1,57 +0,0 @@
|
||||
# OpenBao 监控运维
|
||||
|
||||
受鉴权 `/v1/sys/metrics` 通过 vmagent 内的 Bao Agent 使用 Kubernetes 身份采集。
|
||||
`metrics` policy 仅允许读取该路径；policy 和 role 由 `terraform/monitoring.tf` 管理。
|
||||
|
||||
Bao VM 的 Ubuntu `prometheus-node-exporter` 软件包监听 `192.168.10.8:9100`，
|
||||
由 `ansible/roles/openbao_monitoring` 管理。仅供内网监控，不配置公共入口；
|
||||
现场 UFW 未启用，故当前边界是 LAN 地址绑定，并非逐来源访问控制。
|
||||
标准主机指标复用 `job=node-exporter` 的主机告警，额外标记 `node=bao1`。
|
||||
|
||||
首次部署前需要 bootstrap 已创建 `/etc/openbao/snapshot.token` 和快照目录。
|
||||
只更新监控与快照运行文件（不会重启 Bao 或轮换 token）：
|
||||
|
||||
```bash
|
||||
cd ansible
|
||||
ansible-playbook monitor-openbao.yml --check
|
||||
ansible-playbook monitor-openbao.yml
|
||||
# 在 bao1 上执行一次真实快照，建立成功基线：
|
||||
sudo systemctl start openbao-snapshot.service
|
||||
```
|
||||
|
||||
快照脚本在 `/var/lib/prometheus/node-exporter/` 原子更新两个 `.prom` 文件：
|
||||
|
||||
- `openbao_snapshot_result.prom`：最近一次运行结果（0/1）及完成时间。
|
||||
- `openbao_snapshot_success.prom`：最后一次成功生成完整本地快照的时间。
|
||||
|
||||
失败保留上次成功时间，先完成快照再删除超出保留数量的文件。
|
||||
指标文件 root 写、exporter 只读；不包含 token、路径标签或快照内容。
|
||||
首次成功前成功指标缺失，不用既有文件 mtime 冒充已验证成功。
|
||||
Bao sealed 或 token 失效时，独立 exporter 仍能报告失败及超时。
|
||||
|
||||
`PrometheusRule/openbao` 包含：采集不可用（3 分钟）、内部健康异常（3 分钟）、
|
||||
健康 gauge 缺失（5 分钟）、快照失败（5 分钟）、快照超过 36 小时或无成功记录
|
||||
（持续 15 分钟）、快照监控缺失/损坏（5 分钟）。内部健康规则按当前**单节点**设计，
|
||||
以后增加 standby 必须先调整 active 判定。快照指标仅证明本地任务成功，不能证明异地备份或可恢复性。
|
||||
|
||||
检查与恢复入口：
|
||||
|
||||
```bash
|
||||
sudo systemctl status openbao-snapshot.timer openbao-snapshot.service prometheus-node-exporter
|
||||
sudo journalctl -u openbao-snapshot.service -n 50 --no-pager
|
||||
curl -fsS http://192.168.10.8:9100/metrics | grep '^openbao_snapshot_'
|
||||
```
|
||||
|
||||
token 续期失败按[维护 runbook](https://git.ddupan.top/panxiao81/homelab-wiki/src/branch/main/guides/openbao-monitoring-maintenance.md)
|
||||
受控轮换；不要通过反复重启 Bao 排障。回滚 exporter 或快照脚本无需重启 Bao。
|
||||
回滚时同步撤回相应 ServiceMonitor/规则，否则缺指标告警会继续触发。
|
||||
|
||||
本地验证：
|
||||
|
||||
```bash
|
||||
python3 -m unittest discover -s infrastructure/openbao/ansible/tests -v
|
||||
# 需 PyYAML、Jinja2 和 promtool；已用 promtool 3.5.0 验证。
|
||||
PATH=/path/to/promtool:$PATH bash platform/observability/metrics/tests/check-rules.sh
|
||||
```
|
||||
|
||||
textfile 的原子发布方式遵循 [node_exporter 文档](https://github.com/prometheus/node_exporter#textfile-collector)。
|
||||
@@ -1,13 +0,0 @@
|
||||
---
|
||||
# 只部署 exporter、快照脚本和 timer，不改 Bao 配置、不重启 Bao、不创建 token。
|
||||
# 前提：bootstrap 已安装有效的 /etc/openbao/snapshot.token。
|
||||
- name: 补齐 OpenBao 主机和快照监控
|
||||
hosts: openbao
|
||||
become: true
|
||||
roles:
|
||||
- openbao_monitoring
|
||||
tasks:
|
||||
- name: 更新快照运行文件
|
||||
ansible.builtin.include_role:
|
||||
name: openbao_bootstrap
|
||||
tasks_from: snapshot-runtime
|
||||
@@ -47,6 +47,3 @@ openbao_transit_mount_path: "transit/"
|
||||
openbao_manage_firewall: false
|
||||
openbao_allowed_cidrs:
|
||||
- "100.64.0.0/10"
|
||||
|
||||
# 30 秒采集间隔下保留五分钟，指标端点继续要求鉴权。
|
||||
openbao_prometheus_retention_time: "5m"
|
||||
|
||||
@@ -25,9 +25,3 @@ seal "transit" {
|
||||
mount_path = "{{ openbao_transit_mount_path }}"
|
||||
}
|
||||
{% endif %}
|
||||
|
||||
# 显式固定 Prometheus 行为；应用此配置会经 handler 重启，须人工解封。
|
||||
telemetry {
|
||||
prometheus_retention_time = "{{ openbao_prometheus_retention_time }}"
|
||||
disable_hostname = true
|
||||
}
|
||||
|
||||
@@ -96,8 +96,3 @@ openbao_snapshot_oncalendar: "*-*-* 02:00:00"
|
||||
# 3. terraform apply # mounts, roles, policies
|
||||
# 4. ansible-playbook bootstrap-openbao.yml # CA material, OIDC secret, snapshots
|
||||
openbao_config_managed_by_terraform: true
|
||||
|
||||
# 仅在维护窗口显式启用，用于替换已失效的快照 token；不读取/打印 token。
|
||||
openbao_snapshot_rotate_token: false
|
||||
|
||||
openbao_snapshot_metrics_dir: /var/lib/prometheus/node-exporter
|
||||
|
||||
@@ -1,34 +0,0 @@
|
||||
---
|
||||
- name: Ensure the snapshot output directory exists
|
||||
ansible.builtin.file:
|
||||
path: "{{ openbao_snapshot_dir }}"
|
||||
state: directory
|
||||
owner: root
|
||||
group: root
|
||||
mode: "0700"
|
||||
|
||||
- name: Install the snapshot script
|
||||
ansible.builtin.template:
|
||||
src: bao-snapshot.sh.j2
|
||||
dest: /usr/local/bin/bao-snapshot.sh
|
||||
owner: root
|
||||
group: root
|
||||
mode: "0755"
|
||||
|
||||
- name: Install the snapshot systemd service + timer
|
||||
ansible.builtin.template:
|
||||
src: "{{ item }}.j2"
|
||||
dest: "/etc/systemd/system/{{ item }}"
|
||||
owner: root
|
||||
group: root
|
||||
mode: "0644"
|
||||
loop:
|
||||
- openbao-snapshot.service
|
||||
- openbao-snapshot.timer
|
||||
|
||||
- name: Enable and start the snapshot timer
|
||||
ansible.builtin.systemd:
|
||||
name: openbao-snapshot.timer
|
||||
state: started
|
||||
enabled: true
|
||||
daemon_reload: true
|
||||
@@ -17,11 +17,11 @@
|
||||
path: /etc/openbao/snapshot.token
|
||||
register: snap_tok_stat
|
||||
|
||||
- name: Create or explicitly rotate the periodic snapshot token
|
||||
- name: Create a periodic snapshot token (once)
|
||||
ansible.builtin.command: "bao token create -policy=snapshot -period=768h -orphan -field=token"
|
||||
environment: "{{ openbao_cli_env }}"
|
||||
register: snap_tok_new
|
||||
when: (not snap_tok_stat.stat.exists) or (openbao_snapshot_rotate_token | bool)
|
||||
when: not snap_tok_stat.stat.exists
|
||||
changed_when: snap_tok_new.rc == 0
|
||||
no_log: true
|
||||
|
||||
@@ -32,12 +32,39 @@
|
||||
owner: root
|
||||
group: root
|
||||
mode: "0600"
|
||||
when: (not snap_tok_stat.stat.exists) or (openbao_snapshot_rotate_token | bool)
|
||||
when: not snap_tok_stat.stat.exists
|
||||
no_log: true
|
||||
|
||||
- name: 安装独立的主机与快照指标 exporter
|
||||
ansible.builtin.include_role:
|
||||
name: openbao_monitoring
|
||||
- name: Ensure the snapshot output directory exists
|
||||
ansible.builtin.file:
|
||||
path: "{{ openbao_snapshot_dir }}"
|
||||
state: directory
|
||||
owner: root
|
||||
group: root
|
||||
mode: "0700"
|
||||
|
||||
- name: 部署快照脚本和定时器
|
||||
ansible.builtin.import_tasks: snapshot-runtime.yml
|
||||
- name: Install the snapshot script
|
||||
ansible.builtin.template:
|
||||
src: bao-snapshot.sh.j2
|
||||
dest: /usr/local/bin/bao-snapshot.sh
|
||||
owner: root
|
||||
group: root
|
||||
mode: "0755"
|
||||
|
||||
- name: Install the snapshot systemd service + timer
|
||||
ansible.builtin.template:
|
||||
src: "{{ item }}.j2"
|
||||
dest: "/etc/systemd/system/{{ item }}"
|
||||
owner: root
|
||||
group: root
|
||||
mode: "0644"
|
||||
loop:
|
||||
- openbao-snapshot.service
|
||||
- openbao-snapshot.timer
|
||||
|
||||
- name: Enable and start the snapshot timer
|
||||
ansible.builtin.systemd:
|
||||
name: openbao-snapshot.timer
|
||||
state: started
|
||||
enabled: true
|
||||
daemon_reload: true
|
||||
|
||||
+6
-44
@@ -1,58 +1,20 @@
|
||||
#!/usr/bin/env bash
|
||||
# {{ ansible_managed }}
|
||||
# 本机快照不等于异地备份。结果通过独立 node_exporter 上报。
|
||||
# Take a Raft snapshot and prune old ones. Ship {{ openbao_snapshot_dir }} off-box
|
||||
# separately (rsync/restic/scp) — a snapshot on the same host is not a backup.
|
||||
set -euo pipefail
|
||||
umask 077
|
||||
|
||||
dir="{{ openbao_snapshot_dir }}"
|
||||
metrics_dir="{{ openbao_snapshot_metrics_dir }}"
|
||||
# systemd oneshot 不会并发；flock 同时阻止手动执行与 timer 竞争。
|
||||
exec 9>"${dir}/.snapshot.lock"
|
||||
flock -n 9 || exit 0
|
||||
partial=""
|
||||
metrics_tmp=""
|
||||
finish() {
|
||||
rc=$?
|
||||
trap - EXIT
|
||||
[ -z "$partial" ] || rm -f -- "$partial"
|
||||
# success 文件只在实际快照完成后更新；失败不能抹掉上次成功时间。
|
||||
metrics_tmp="$(mktemp "${metrics_dir}/.openbao-result.XXXXXX")" || exit 1
|
||||
{
|
||||
echo '# HELP openbao_snapshot_last_run_success Whether the last snapshot run succeeded.'
|
||||
echo '# TYPE openbao_snapshot_last_run_success gauge'
|
||||
if [ "$rc" -eq 0 ]; then echo 'openbao_snapshot_last_run_success 1'; else echo 'openbao_snapshot_last_run_success 0'; fi
|
||||
echo '# HELP openbao_snapshot_last_run_timestamp_seconds Completion time of the last snapshot attempt.'
|
||||
echo '# TYPE openbao_snapshot_last_run_timestamp_seconds gauge'
|
||||
echo "openbao_snapshot_last_run_timestamp_seconds $(date +%s)"
|
||||
} > "$metrics_tmp"
|
||||
chmod 644 "$metrics_tmp"
|
||||
mv -f -- "$metrics_tmp" "${metrics_dir}/openbao_snapshot_result.prom"
|
||||
exit "$rc"
|
||||
}
|
||||
trap finish EXIT
|
||||
|
||||
export BAO_ADDR="{{ openbao_addr }}"
|
||||
export BAO_CACERT="{{ openbao_tls_dir }}/cert.pem"
|
||||
BAO_TOKEN="$(cat /etc/openbao/snapshot.token)"
|
||||
export BAO_TOKEN
|
||||
bao token renew >/dev/null
|
||||
|
||||
dir="{{ openbao_snapshot_dir }}"
|
||||
stamp="$(date +%Y%m%d-%H%M%S)"
|
||||
out="${dir}/openbao-${stamp}.snap"
|
||||
partial="${out}.partial"
|
||||
bao operator raft snapshot save "$partial"
|
||||
chmod 600 "$partial"
|
||||
mv -- "$partial" "$out"
|
||||
partial=""
|
||||
|
||||
metrics_tmp="$(mktemp "${metrics_dir}/.openbao-success.XXXXXX")"
|
||||
{
|
||||
echo '# HELP openbao_snapshot_last_success_timestamp_seconds Completion time of the last successful local Raft snapshot.'
|
||||
echo '# TYPE openbao_snapshot_last_success_timestamp_seconds gauge'
|
||||
echo "openbao_snapshot_last_success_timestamp_seconds $(date +%s)"
|
||||
} > "$metrics_tmp"
|
||||
chmod 644 "$metrics_tmp"
|
||||
mv -f -- "$metrics_tmp" "${metrics_dir}/openbao_snapshot_success.prom"
|
||||
bao operator raft snapshot save "${out}"
|
||||
chmod 600 "${out}"
|
||||
|
||||
# 只有产生完整快照后才做保留清理。
|
||||
# Retention: keep the newest {{ openbao_snapshot_keep }}.
|
||||
ls -1t "${dir}"/openbao-*.snap 2>/dev/null | tail -n +{{ openbao_snapshot_keep + 1 }} | xargs -r rm -f
|
||||
|
||||
@@ -1,3 +0,0 @@
|
||||
---
|
||||
openbao_metrics_listen_address: "{{ ansible_host }}:9100"
|
||||
openbao_snapshot_metrics_dir: /var/lib/prometheus/node-exporter
|
||||
@@ -1,6 +0,0 @@
|
||||
---
|
||||
- name: restart openbao node exporter
|
||||
ansible.builtin.systemd:
|
||||
name: prometheus-node-exporter
|
||||
state: restarted
|
||||
when: not ansible_check_mode
|
||||
@@ -1,35 +0,0 @@
|
||||
---
|
||||
- name: 安装 Ubuntu node_exporter 软件包
|
||||
ansible.builtin.apt:
|
||||
name: prometheus-node-exporter
|
||||
state: present
|
||||
update_cache: true
|
||||
cache_valid_time: 3600
|
||||
install_recommends: false
|
||||
policy_rc_d: 101
|
||||
|
||||
- name: 创建只允许 root 写入的 textfile 目录
|
||||
ansible.builtin.file:
|
||||
path: "{{ openbao_snapshot_metrics_dir }}"
|
||||
state: directory
|
||||
owner: root
|
||||
group: root
|
||||
mode: "0755"
|
||||
|
||||
- name: 将 exporter 绑定到内网地址并启用 textfile
|
||||
ansible.builtin.copy:
|
||||
dest: /etc/default/prometheus-node-exporter
|
||||
owner: root
|
||||
group: root
|
||||
mode: "0644"
|
||||
content: |
|
||||
# Ansible managed; no Bao credentials needed.
|
||||
ARGS="--web.listen-address={{ openbao_metrics_listen_address }} --collector.textfile.directory={{ openbao_snapshot_metrics_dir }}"
|
||||
notify: restart openbao node exporter
|
||||
|
||||
- name: 启用主机指标服务
|
||||
ansible.builtin.systemd:
|
||||
name: prometheus-node-exporter
|
||||
enabled: true
|
||||
state: started
|
||||
when: not ansible_check_mode
|
||||
@@ -1,81 +0,0 @@
|
||||
"""用假 bao 验证失败不会推进成功时间或删除已有快照。"""
|
||||
import os
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import tempfile
|
||||
import unittest
|
||||
from jinja2 import Template
|
||||
|
||||
TEMPLATE = Path(__file__).resolve().parents[1] / 'roles/openbao_bootstrap/templates/bao-snapshot.sh.j2'
|
||||
|
||||
|
||||
class SnapshotTest(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.tmp = tempfile.TemporaryDirectory()
|
||||
self.addCleanup(self.tmp.cleanup)
|
||||
self.root = Path(self.tmp.name)
|
||||
self.snap = self.root / 'snapshots'
|
||||
self.metrics = self.root / 'metrics'
|
||||
self.snap.mkdir()
|
||||
self.metrics.mkdir()
|
||||
self.token = self.root / 'token'
|
||||
self.token.write_text('test-only')
|
||||
self.script = self.root / 'snapshot.sh'
|
||||
self.script.write_text(Template(TEMPLATE.read_text()).render(
|
||||
ansible_managed='test', openbao_snapshot_dir=str(self.snap),
|
||||
openbao_snapshot_metrics_dir=str(self.metrics), openbao_addr='https://invalid',
|
||||
openbao_tls_dir='/unused', openbao_snapshot_keep=2,
|
||||
).replace('/etc/openbao/snapshot.token', str(self.token)))
|
||||
bao = self.root / 'bao'
|
||||
bao.write_text('''#!/usr/bin/env bash
|
||||
if [ "$1" = token ]; then
|
||||
[ "${FAIL_AT:-}" != renew ]; exit $?
|
||||
fi
|
||||
printf snapshot > "$5"
|
||||
[ "${FAIL_AT:-}" != save ]
|
||||
''')
|
||||
bao.chmod(0o755)
|
||||
for n in range(3):
|
||||
p = self.snap / f'openbao-old{n}.snap'
|
||||
p.write_text('old snapshot')
|
||||
os.utime(p, (100+n, 100+n))
|
||||
self.success = self.metrics / 'openbao_snapshot_success.prom'
|
||||
self.success.write_text('openbao_snapshot_last_success_timestamp_seconds 123\n')
|
||||
|
||||
def run_snapshot(self, failure=''):
|
||||
return subprocess.run(['bash', str(self.script)], env=dict(os.environ,
|
||||
PATH=str(self.root)+':'+os.environ['PATH'], FAIL_AT=failure), capture_output=True)
|
||||
|
||||
def test_renew_failure_preserves_success_and_snapshots(self):
|
||||
self.assertNotEqual(self.run_snapshot('renew').returncode, 0)
|
||||
self.check_failure()
|
||||
|
||||
def test_partial_snapshot_is_removed(self):
|
||||
self.assertNotEqual(self.run_snapshot('save').returncode, 0)
|
||||
self.check_failure()
|
||||
self.assertEqual(list(self.snap.glob('*.partial')), [])
|
||||
|
||||
def test_missing_token_is_reported(self):
|
||||
self.token.unlink()
|
||||
self.assertNotEqual(self.run_snapshot().returncode, 0)
|
||||
self.check_failure()
|
||||
|
||||
def check_failure(self):
|
||||
self.assertEqual(self.success.read_text(), 'openbao_snapshot_last_success_timestamp_seconds 123\n')
|
||||
self.assertIn('openbao_snapshot_last_run_success 0', (self.metrics/'openbao_snapshot_result.prom').read_text())
|
||||
self.assertEqual(len(list(self.snap.glob('*.snap'))), 3)
|
||||
|
||||
def test_success_retention_and_permissions(self):
|
||||
p = self.run_snapshot()
|
||||
self.assertEqual(p.returncode, 0, p.stderr)
|
||||
self.assertNotIn('seconds 123\n', self.success.read_text())
|
||||
self.assertIn('openbao_snapshot_last_run_success 1', (self.metrics/'openbao_snapshot_result.prom').read_text())
|
||||
snapshots = list(self.snap.glob('*.snap'))
|
||||
self.assertEqual(len(snapshots), 2)
|
||||
new = next(p for p in snapshots if 'old' not in p.name)
|
||||
self.assertEqual(new.stat().st_mode & 0o777, 0o600)
|
||||
self.assertEqual(self.success.stat().st_mode & 0o777, 0o644)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
@@ -68,14 +68,3 @@ import {
|
||||
to = vault_pki_secret_backend_config_urls.this
|
||||
id = "pki/config/urls"
|
||||
}
|
||||
|
||||
# 本次维护预检先通过同名 API 配置验证，后续 Terraform 纳入既有主 state。
|
||||
import {
|
||||
to = vault_policy.metrics
|
||||
id = "metrics"
|
||||
}
|
||||
|
||||
import {
|
||||
to = vault_kubernetes_auth_backend_role.metrics
|
||||
id = "auth/kubernetes/role/metrics"
|
||||
}
|
||||
|
||||
@@ -1,15 +0,0 @@
|
||||
# 指标只读身份；vmagent 的 Bao Agent sidecar 使用 Pod SA 自动登录与续期。
|
||||
resource "vault_policy" "metrics" {
|
||||
name = "metrics"
|
||||
policy = file("${path.module}/policies/metrics.hcl")
|
||||
}
|
||||
|
||||
resource "vault_kubernetes_auth_backend_role" "metrics" {
|
||||
backend = "kubernetes"
|
||||
role_name = "metrics"
|
||||
bound_service_account_names = ["vmagent-main"]
|
||||
bound_service_account_namespaces = ["monitoring"]
|
||||
token_policies = [vault_policy.metrics.name]
|
||||
token_ttl = 900
|
||||
token_max_ttl = 3600
|
||||
}
|
||||
@@ -1,4 +0,0 @@
|
||||
# 不含秘密读取或管理权限；默认 policy 只用于自身 token 续期等通用能力。
|
||||
path "sys/metrics" {
|
||||
capabilities = ["read"]
|
||||
}
|
||||
@@ -1 +0,0 @@
|
||||
__pycache__/
|
||||
@@ -1,79 +0,0 @@
|
||||
# 共享数据服务维护监控
|
||||
|
||||
为 laptop、etcd-pve1（10.60.0.20）、etcd-pve2（10.60.0.21）补齐备份、磁盘证书、
|
||||
续签检查与 dev SQL 的指标。复用现有 VictoriaMetrics/Alertmanager；不部署新监控栈。
|
||||
实现由 [PR #166](https://git.ddupan.top/panxiao81/homelab-infra/pulls/166) 发布。
|
||||
|
||||
2026-09-27：三个主机端已部署并验证，Ansible 重跑均 `changed=0`；vmagent Pod 能访问三个
|
||||
内网 `:9109/metrics`。PR #166 已合并，Flux observability Ready，应用版本 `589c34e`；
|
||||
PrometheusRule 与 VMStaticScrape 已进入 inventory，converter 生成的 VMRule 为 operational。
|
||||
三个抓取目标均 up=1，八条规则均 health=ok、inactive，告警已启用。
|
||||
|
||||
## 采集和阈值
|
||||
|
||||
每分钟执行 `homelab-data-collect.service`，读取明确清单并原子替换
|
||||
`/var/lib/homelab-data-monitoring/health.prom`。每项检查失败输出零值及 `check_success=0`，
|
||||
不沿用旧成功值；完整采集时间用于发现任务停止或文件缺失。端点不暴露密码、token、私钥。
|
||||
|
||||
| 项目 | 证据 | 告警 |
|
||||
|---|---|---|
|
||||
| etcd 快照 | 每成员已完成 `.db` 文件 mtime，排除 `.partial` | 超过 36 小时 warning |
|
||||
| PG full 备份 | 仓库本地 pgBackRest JSON 中无错误 full 的完成时间 | 超过 36 小时 warning |
|
||||
| 证书 | 显式清单包含 server/peer/admin/gateway/PG etcd client/CA；取完整 PEM 链最早到期时间 | 不足 10 天 warning；不足 3 天或无法读取 critical |
|
||||
| 续签检查 | laptop 两个 renew service 的成功退出时间及 timer active | 失败、停用或超过 36 小时 warning |
|
||||
| dev | 本地 socket、pgdev peer 身份，只读 SQL 检查非 recovery 且非只读 | 连续异常 3 分钟 warning |
|
||||
| 采集链路 | exporter up、单项检查、textfile 错误、完整采集时间 | 失败或心跳超过 3 分钟后持续 3 分钟 warning |
|
||||
|
||||
证书在不足 14 天时由原自动续签任务处理，10 天阈值给重试留出余量；critical 不重复触发 warning。
|
||||
36 小时适用于现有每日快照/full/续签检查计划。dev 未新增备份，异地备份仍后置。
|
||||
|
||||
node_exporter 当前版本为 1.9.1；下载包与官方 `sha256sums.txt` 均由
|
||||
`data_monitor_exporter_version` 生成，Ansible 按包名匹配摘要，升级只改版本变量；保留上游 LICENSE。
|
||||
仅启用 textfile，使用 DynamicUser，绑定各主机内网地址的 9109，与现有 9100 不冲突。
|
||||
采集程序以 root 运行，以便读取快照元数据和切换为 pgdev/pgbackup；systemd 限制文件系统写入
|
||||
到 textfile 目录，限制执行时间与内存。无需 Bao 登录；数据库和 etcd 不因部署监控重启。
|
||||
采集并非常驻进程；laptop 上 exporter 当前 MemoryCurrent 约 2.6 MiB，仅作现场开销参考。
|
||||
|
||||
## 部署和验证
|
||||
|
||||
在 PR 分支或合并后的完整 checkout 中执行：
|
||||
|
||||
```bash
|
||||
cd infrastructure/shared-data-monitoring/ansible
|
||||
ansible-playbook site.yml
|
||||
```
|
||||
|
||||
本机需要免交互 sudo，远程使用已有 root SSH；配置不包含秘密。当前实现仅支持 Linux amd64，
|
||||
依赖现有 Python 3、OpenSSL、systemd、PG 客户端和 pgBackRest。下载 exporter 需要访问 GitHub。
|
||||
|
||||
只读查看结果：
|
||||
|
||||
```bash
|
||||
curl --fail http://192.168.10.127:9109/metrics
|
||||
sudo systemctl status homelab-data-collect.timer homelab-data-exporter.service
|
||||
sudo journalctl -u homelab-data-collect.service --since '2 hours ago'
|
||||
```
|
||||
|
||||
主机端 `check_success` 应全为 1、`dev_sql_ready=1`，`node_textfile_scrape_error=0`。
|
||||
日常检查 `up{job="shared-data-maintenance"}` 三个目标及 `shared-data-maintenance` 规则组。
|
||||
告警声明使用 `monitoring.coreos.com/v1` 的 `PrometheusRule`，由现有 converter 生成 VMRule；
|
||||
静态主机抓取使用 VMStaticScrape。Kubernetes 资源由现有 observability Kustomization 纳管；
|
||||
主机端由此 Ansible 独立维护。
|
||||
|
||||
2026-09-27 验证：6 项采集测试、12 个新增 promtool 场景及既有规则回归通过；Ansible lint
|
||||
零失败零告警；Kustomize 渲染通过；三主机实际采集和 vmagent 网络路径通过。未停止业务来测试告警。
|
||||
|
||||
## 故障入口与边界
|
||||
|
||||
- 采集停止：先查 exporter 和 collect timer/service，再看 `.prom` 的 collection timestamp。
|
||||
- 备份陈旧：etcd 查 `homelab-etcd-snapshot`；PG 仓库查 `homelab-postgresql-backup`。
|
||||
不以 touch 文件消除告警。快照年龄依赖完成文件 mtime，手工复制会改变证据，不能冒充新备份。
|
||||
- 证书或续签异常：查 laptop 的 `homelab-etcd-renew` / `homelab-postgresql-renew`；
|
||||
修复 Bao/SPIRE/网络或受限身份后重跑原续签入口，不删除现有证书或重置数据库。
|
||||
- dev SQL 失败：查 `homelab-postgresql-dev`、磁盘与 socket，再核对本地 peer 认证。
|
||||
|
||||
备份年龄不能证明恢复能力或 WAL 连续性；定期隔离恢复演练仍必要。证书指标读取磁盘文件，
|
||||
不证明运行进程已重载该证书。dev 检查不验证外部 DNS/TLS 路径、不执行写入。
|
||||
SQL 级指标见 [共享 PG](../shared-postgresql/README.md#sql-级指标2026-10-01-主机端上线)；异地备份及应用迁移不在本次范围。
|
||||
|
||||
跨服务状态见 [共享 PG wiki](https://git.ddupan.top/panxiao81/homelab-wiki/src/branch/main/services/shared-postgresql.md)。
|
||||
@@ -1,5 +0,0 @@
|
||||
[defaults]
|
||||
inventory = inventory/hosts.yml
|
||||
host_key_checking = True
|
||||
retry_files_enabled = False
|
||||
interpreter_python = auto_silent
|
||||
@@ -1,20 +0,0 @@
|
||||
---
|
||||
all:
|
||||
children:
|
||||
shared_data_monitoring:
|
||||
hosts:
|
||||
laptop:
|
||||
ansible_connection: local
|
||||
ansible_user: panxiao81
|
||||
ansible_host: 192.168.10.127
|
||||
data_monitor_dev: true
|
||||
data_monitor_pg: true
|
||||
data_monitor_renewals: [homelab-etcd-renew, homelab-postgresql-renew]
|
||||
etcd-pve1:
|
||||
ansible_host: 10.60.0.20
|
||||
ansible_user: root
|
||||
data_monitor_pg: true
|
||||
etcd-pve2:
|
||||
ansible_host: 10.60.0.21
|
||||
ansible_user: root
|
||||
data_monitor_repository: true
|
||||
@@ -1,146 +0,0 @@
|
||||
---
|
||||
- name: 采集共享数据服务维护指标
|
||||
hosts: shared_data_monitoring
|
||||
become: true
|
||||
gather_facts: false
|
||||
vars:
|
||||
data_monitor_root: /opt/homelab-data-monitoring
|
||||
data_monitor_exporter_version: 1.9.1
|
||||
data_monitor_download_base: https://github.com/prometheus/node_exporter/releases/download
|
||||
data_monitor_archive: "node_exporter-{{ data_monitor_exporter_version }}.linux-amd64"
|
||||
tasks:
|
||||
- name: 检查仅支持 amd64
|
||||
ansible.builtin.command: uname -m
|
||||
register: data_monitor_arch
|
||||
changed_when: false
|
||||
- name: 拒绝错误架构
|
||||
ansible.builtin.assert:
|
||||
that: data_monitor_arch.stdout == 'x86_64'
|
||||
- name: 创建 root 管理目录
|
||||
ansible.builtin.file:
|
||||
path: "{{ item }}"
|
||||
state: directory
|
||||
owner: root
|
||||
group: root
|
||||
mode: '0755'
|
||||
loop:
|
||||
- "{{ data_monitor_root }}"
|
||||
- /etc/homelab-data-monitoring
|
||||
- /var/lib/homelab-data-monitoring
|
||||
- name: 下载 exporter 并校验同版本官方摘要
|
||||
ansible.builtin.get_url:
|
||||
url: >-
|
||||
{{ data_monitor_download_base }}/v{{ data_monitor_exporter_version }}/{{ data_monitor_archive }}.tar.gz
|
||||
dest: "{{ data_monitor_root }}/node_exporter.tar.gz"
|
||||
checksum: >-
|
||||
sha256:{{ data_monitor_download_base }}/v{{ data_monitor_exporter_version }}/sha256sums.txt
|
||||
owner: root
|
||||
group: root
|
||||
mode: '0644'
|
||||
- name: 解压 exporter（保留上游许可证）
|
||||
ansible.builtin.unarchive:
|
||||
src: "{{ data_monitor_root }}/node_exporter.tar.gz"
|
||||
dest: "{{ data_monitor_root }}"
|
||||
remote_src: true
|
||||
creates: "{{ data_monitor_root }}/node_exporter-{{ data_monitor_exporter_version }}.linux-amd64/node_exporter"
|
||||
- name: 安装只读采集程序
|
||||
ansible.builtin.copy:
|
||||
src: ../files/collect.py
|
||||
dest: "{{ data_monitor_root }}/collect.py"
|
||||
owner: root
|
||||
group: root
|
||||
mode: '0755'
|
||||
- name: 配置明确的证书与检查清单
|
||||
ansible.builtin.template:
|
||||
src: config.json.j2
|
||||
dest: /etc/homelab-data-monitoring/config.json
|
||||
owner: root
|
||||
group: root
|
||||
mode: '0644'
|
||||
- name: 安装采集 service
|
||||
ansible.builtin.copy:
|
||||
dest: /etc/systemd/system/homelab-data-collect.service
|
||||
mode: '0644'
|
||||
content: |
|
||||
[Unit]
|
||||
Description=Read-only shared data maintenance checks
|
||||
[Service]
|
||||
Type=oneshot
|
||||
ExecStart=/usr/bin/python3 {{ data_monitor_root }}/collect.py \
|
||||
--config /etc/homelab-data-monitoring/config.json \
|
||||
--output /var/lib/homelab-data-monitoring/health.prom
|
||||
Environment=LC_ALL=C
|
||||
Environment=PGCONNECT_TIMEOUT=5
|
||||
Environment="PGOPTIONS=-c statement_timeout=5000"
|
||||
TimeoutStartSec=50
|
||||
UMask=0022
|
||||
ProtectSystem=strict
|
||||
ReadWritePaths=/var/lib/homelab-data-monitoring
|
||||
PrivateTmp=true
|
||||
MemoryMax=96M
|
||||
Nice=10
|
||||
- name: 安装每分钟采集 timer
|
||||
ansible.builtin.copy:
|
||||
dest: /etc/systemd/system/homelab-data-collect.timer
|
||||
mode: '0644'
|
||||
content: |
|
||||
[Unit]
|
||||
Description=Collect shared data maintenance metrics every minute
|
||||
[Timer]
|
||||
OnBootSec=30s
|
||||
OnUnitActiveSec=60s
|
||||
AccuracySec=5s
|
||||
[Install]
|
||||
WantedBy=timers.target
|
||||
- name: 安装仅内网 textfile exporter
|
||||
ansible.builtin.copy:
|
||||
dest: /etc/systemd/system/homelab-data-exporter.service
|
||||
mode: '0644'
|
||||
content: |
|
||||
[Unit]
|
||||
Description=Shared data textfile metrics
|
||||
After=network-online.target
|
||||
Wants=network-online.target
|
||||
[Service]
|
||||
DynamicUser=true
|
||||
ExecStart={{ data_monitor_root }}/{{ data_monitor_archive }}/node_exporter \
|
||||
--web.listen-address={{ ansible_host }}:9109 \
|
||||
--collector.disable-defaults --collector.textfile \
|
||||
--collector.textfile.directory=/var/lib/homelab-data-monitoring --web.disable-exporter-metrics
|
||||
Restart=on-failure
|
||||
NoNewPrivileges=true
|
||||
ProtectSystem=strict
|
||||
ProtectHome=true
|
||||
PrivateTmp=true
|
||||
MemoryMax=64M
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
notify: Restart shared data exporter
|
||||
- name: 重载 unit 声明
|
||||
ansible.builtin.systemd_service:
|
||||
daemon_reload: true
|
||||
- name: 首次只读采集并检查程序能够完成
|
||||
ansible.builtin.command:
|
||||
argv:
|
||||
- /usr/bin/python3
|
||||
- "{{ data_monitor_root }}/collect.py"
|
||||
- --config
|
||||
- /etc/homelab-data-monitoring/config.json
|
||||
- --output
|
||||
- /var/lib/homelab-data-monitoring/health.prom
|
||||
environment:
|
||||
LC_ALL: C
|
||||
PGCONNECT_TIMEOUT: '5'
|
||||
PGOPTIONS: '-c statement_timeout=5000'
|
||||
changed_when: false
|
||||
- name: 启用监控服务
|
||||
ansible.builtin.systemd_service:
|
||||
name: "{{ item }}"
|
||||
enabled: true
|
||||
state: started
|
||||
loop: [homelab-data-collect.timer, homelab-data-exporter.service]
|
||||
handlers:
|
||||
- name: Restart shared data exporter
|
||||
ansible.builtin.systemd_service:
|
||||
name: homelab-data-exporter.service
|
||||
state: restarted
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user