Compare commits
5
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
  panxiao81andClaude Opus 5.5
|
31f455d51d
|
||
|
panxiao81andClaude Opus 5.5
|
9f7b1b98dd
|
||
|
panxiao81andClaude Opus 5.5
|
094b38b4b5
|
||
|
panxiao81andClaude Opus 5.5
|
5dcfcad098
|
||
|
panxiao81
|
d45e0b94e8
|
@@ -1,32 +0,0 @@
|
||||
---
|
||||
name: dind-fuse-image
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
push:
|
||||
branches: [main]
|
||||
paths:
|
||||
- platform/gitea-runner/images/dind-fuse/**
|
||||
- .gitea/workflows/dind-fuse-image.yml
|
||||
|
||||
jobs:
|
||||
publish:
|
||||
runs-on: [self-hosted, pod]
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- name: Log in to the Gitea container registry
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
registry: git.ddupan.top
|
||||
username: ${{ gitea.actor }}
|
||||
password: ${{ secrets.REGISTRY_TOKEN }}
|
||||
|
||||
- name: Build and publish
|
||||
uses: docker/build-push-action@v6
|
||||
with:
|
||||
context: platform/gitea-runner/images/dind-fuse
|
||||
push: true
|
||||
tags: |
|
||||
git.ddupan.top/panxiao81/dind-fuse:29.7.1
|
||||
git.ddupan.top/panxiao81/dind-fuse:latest
|
||||
@@ -1,24 +0,0 @@
|
||||
---
|
||||
name: kata-runner-smoke
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
push:
|
||||
branches:
|
||||
- poc/kata-runner-smoke
|
||||
paths:
|
||||
- .gitea/workflows/kata-runner-smoke.yml
|
||||
|
||||
jobs:
|
||||
smoke:
|
||||
runs-on: [self-hosted, pod]
|
||||
steps:
|
||||
- name: Verify isolated job environment
|
||||
shell: sh
|
||||
run: |
|
||||
set -eu
|
||||
uname -a
|
||||
grep -q '^ID=alpine$' /etc/os-release
|
||||
test -f /.dockerenv
|
||||
test ! -e /dev/kvm
|
||||
printf 'kata ephemeral runner job ok\n'
|
||||
@@ -1,63 +0,0 @@
|
||||
---
|
||||
name: kind-on-kata-smoke
|
||||
|
||||
on:
|
||||
push:
|
||||
branches:
|
||||
- poc/kind-on-kata
|
||||
paths:
|
||||
- .gitea/workflows/kind-on-kata-smoke.yml
|
||||
workflow_dispatch:
|
||||
|
||||
jobs:
|
||||
smoke:
|
||||
runs-on: [self-hosted, pod]
|
||||
steps:
|
||||
- name: Create nested kind cluster
|
||||
shell: sh
|
||||
env:
|
||||
KIND_VERSION: v0.27.0
|
||||
KIND_NODE_IMAGE: kindest/node:v1.32.2@sha256:f226345927d7e348497136874b6d207e0b32cc52154ad8323129352923a3142f
|
||||
run: |
|
||||
set -eu
|
||||
apk add --no-cache ca-certificates curl docker-cli
|
||||
curl --retry 5 --retry-all-errors --connect-timeout 15 -fsSLo /tmp/kind \
|
||||
"https://kind.sigs.k8s.io/dl/${KIND_VERSION}/kind-linux-amd64"
|
||||
curl --retry 5 --retry-all-errors --connect-timeout 15 -fsSLo /tmp/kind.sha256sum \
|
||||
"https://kind.sigs.k8s.io/dl/${KIND_VERSION}/kind-linux-amd64.sha256sum"
|
||||
expected="$(awk '{print $1}' /tmp/kind.sha256sum)"
|
||||
printf '%s %s\n' "$expected" /tmp/kind | sha256sum -c -
|
||||
install -m 0755 /tmp/kind /usr/local/bin/kind
|
||||
docker info --format 'kernel={{.KernelVersion}} driver={{.Driver}}'
|
||||
test "$(docker info --format '{{.Driver}}')" = overlay2
|
||||
|
||||
cleanup() {
|
||||
kind delete cluster --name nested >/dev/null 2>&1 || true
|
||||
}
|
||||
trap cleanup EXIT
|
||||
|
||||
cat >/tmp/kind-config.yaml <<'EOF'
|
||||
kind: Cluster
|
||||
apiVersion: kind.x-k8s.io/v1alpha4
|
||||
name: nested
|
||||
nodes:
|
||||
- role: control-plane
|
||||
extraMounts:
|
||||
- hostPath: /dev/kmsg
|
||||
containerPath: /dev/kmsg
|
||||
EOF
|
||||
kind create cluster -v 9 --retain --config /tmp/kind-config.yaml --image "$KIND_NODE_IMAGE" --wait 5m
|
||||
docker exec nested-control-plane kubectl \
|
||||
--kubeconfig=/etc/kubernetes/admin.conf wait \
|
||||
--for=condition=Ready node/nested-control-plane --timeout=2m
|
||||
docker exec nested-control-plane kubectl \
|
||||
--kubeconfig=/etc/kubernetes/admin.conf run smoke \
|
||||
--image=docker.io/library/busybox:1.37 --restart=Never \
|
||||
--command -- sh -c 'echo kind-on-kata-ok'
|
||||
docker exec nested-control-plane kubectl \
|
||||
--kubeconfig=/etc/kubernetes/admin.conf wait \
|
||||
--for=jsonpath='{.status.phase}'=Succeeded pod/smoke --timeout=2m
|
||||
test "$(docker exec nested-control-plane kubectl \
|
||||
--kubeconfig=/etc/kubernetes/admin.conf logs smoke)" = kind-on-kata-ok
|
||||
kind delete cluster --name nested
|
||||
trap - EXIT
|
||||
@@ -0,0 +1,30 @@
|
||||
# 独立增量声明：全量 values.yaml 尚未覆盖所有线上客户端，不能用它覆盖 release。
|
||||
authorization_policies:
|
||||
incus_admin:
|
||||
default_policy: deny
|
||||
rules:
|
||||
- policy: two_factor
|
||||
subject: user:panxiao81
|
||||
clients:
|
||||
- client_id: incus
|
||||
client_name: Incus
|
||||
public: true
|
||||
authorization_policy: incus_admin
|
||||
require_pkce: true
|
||||
pkce_challenge_method: S256
|
||||
redirect_uris:
|
||||
- https://incus.ad.ddupan.top/oidc/callback
|
||||
audience:
|
||||
- https://incus.ad.ddupan.top
|
||||
scopes:
|
||||
- openid
|
||||
- offline_access
|
||||
response_types:
|
||||
- code
|
||||
grant_types:
|
||||
- authorization_code
|
||||
- refresh_token
|
||||
- urn:ietf:params:oauth:grant-type:device_code
|
||||
access_token_signed_response_alg: RS256
|
||||
userinfo_signed_response_alg: none
|
||||
token_endpoint_auth_method: none
|
||||
@@ -18,13 +18,3 @@ OCI digest 固定镜像。
|
||||
|
||||
Flux 等待 ExternalSecret 和 Deployment 就绪；任何前置缺失都会使该
|
||||
Kustomization 保持 NotReady，而不会回退到明文 Secret。
|
||||
|
||||
## 镜像更新
|
||||
|
||||
`panxiao81/backstage` 的 CI 在 main push 后只推送 `:latest`。
|
||||
`platform/flux-image-automation` 跟踪 `latest` 背后的 digest，由 `flux-bot`
|
||||
直接提交到本仓库 main，不经过 PR。`deployment.yaml` 中 image 行的
|
||||
`$imagepolicy` 注释是 setter 标记，删除后自动更新会静默停止。
|
||||
|
||||
旧 digest 失去 tag 后会被 zot GC 回收（24h），因此不要把 Deployment 手工固定到
|
||||
一个已不是 `latest` 的 digest 并长期保留。
|
||||
|
||||
@@ -27,7 +27,7 @@ spec:
|
||||
type: RuntimeDefault
|
||||
containers:
|
||||
- name: backstage
|
||||
image: zot.ad.ddupan.top/panxiao81/backstage:latest@sha256:da55be5c2f5c8b33de2d87ee0ef02edeacf52a349c8b19a5fac9be4152b2723d # {"$imagepolicy": "flux-system:backstage"}
|
||||
image: zot.ad.ddupan.top/panxiao81/backstage@sha256:008a3ccf832c9ee061ef03766022789a7539bd9d001a658af2e9f3ff59a9a5cc
|
||||
imagePullPolicy: IfNotPresent
|
||||
env:
|
||||
- name: BACKSTAGE_BASE_URL
|
||||
|
||||
+6
-61
@@ -1,69 +1,14 @@
|
||||
# Hydra 与独立 IAM 登录接入
|
||||
# Hydra 与 OIDC Login/Consent PoC
|
||||
|
||||
本目录提供独立 Hydra 签发服务。当前分支将实验性 Hydra 登录入口接至 Spring `iam-login`
|
||||
开发实例，由其执行 AD 密码、WebAuthn 和授权确认；不改变 issuer、Gitea 登录源或数据库。
|
||||
旧 Go OIDC 适配器继续部署以便回退，Gitea 的直接 Authelia 登录源也保留。
|
||||
该切换已于 2026-10-01 经 PR #168 合并并由 Flux 应用；下面原有 Go/Authelia 说明保留为回退路径资料。
|
||||
本目录提供独立 Hydra 签发服务，以及一个薄的 **OIDC 上游适配器**。当前上游配置为
|
||||
Authelia；适配器不连接 LDAP，也不管理用户目录。Samba AD、密码和 MFA 继续由现有
|
||||
Authelia 链路负责。第一轮只接入人类和 Gitea，不实现 agent 动态授权。
|
||||
|
||||
```text
|
||||
Gitea → Hydra → iam-login（Tailscale 开发实例）→ Samba AD + WebAuthn
|
||||
Gitea → Hydra → OIDC Login/Consent → Authelia → Samba AD
|
||||
← OIDC ← 经验证的上游身份 ← OIDC callback
|
||||
```
|
||||
|
||||
## Spring 开发路径验收
|
||||
|
||||
源码版本：[iam-login 5f49a7c](https://git.ddupan.top/panxiao81/iam-login/commit/5f49a7c)。
|
||||
开发 UI 是 `https://laptop.tail7e769.ts.net:18082`，已有 Passkey RP ID 保持不变。
|
||||
Hydra 回调为 `/oauth2/start`、`/oauth2/consent`、`/oauth2/logout`；默认注销回到 `/signin`。
|
||||
浏览器需要 Tailscale 连通性；开发服务暂由本地 JVM 容器运行，并非生产 Native 部署。
|
||||
|
||||
开发服务通过 `kubectl -n hydra port-forward --address 127.0.0.1 service/hydra-admin 18445:4445`
|
||||
访问私有 Admin API。转发需在 Hydra 滚动更新后重连；验收运行器会循环重建该通道。
|
||||
该通道与本地容器是临时验收依赖，必须保持运行；不修改 NetworkPolicy、
|
||||
不新增 Admin HTTPRoute。临时运行配置与旧版回退备份仅保存在受限本地运行目录，不提交秘密。
|
||||
|
||||
验收前已只读核对目标 Gitea 账号的现有 Hydra 外部关联，并将旧 sub 显式绑定到 AD
|
||||
稳定主体；不按邮箱重建关联、不修改 Gitea 账号或仓库权限。客户端管理仅允许有效 MFA 且
|
||||
直接属于 AD Domain Admins 的用户；这是验收期明确指定的粗粒度管理组。
|
||||
|
||||
### 客户端管理 `/console`
|
||||
|
||||
iam-login 的客户端管理 API 只接受 Hydra 签发、带 `iam.clients.manage` scope 的 access token；
|
||||
`/console` 是调用该 API 的前端，在 Hydra 中登记为普通 **public** 客户端 `iam-admin-ui`。
|
||||
该 scope 只在 consent 时发给 `iam-admin-ui` 且直接属于管理组的用户，规则在 iam-login。
|
||||
|
||||
浏览器直接向 Hydra 换取 token，因此 `hydra.yaml` 为 public 端口开启 CORS，只允许开发实例
|
||||
origin；不放行 cookie 凭据。Gitea 等服务端客户端不受影响。
|
||||
|
||||
`iam-admin-ui` 无 secret，与 `gitea` 一样经 Admin 带外登记（它是 public 客户端，iam-login API
|
||||
看不到也删不掉它，恢复同样走此通道）：
|
||||
|
||||
```sh
|
||||
curl -fsS -X POST http://127.0.0.1:18445/admin/clients -H 'Content-Type: application/json' -d '{
|
||||
"client_id": "iam-admin-ui", "client_name": "IAM 管理",
|
||||
"redirect_uris": ["https://laptop.tail7e769.ts.net:18082/console/callback"],
|
||||
"grant_types": ["authorization_code"], "response_types": ["code"],
|
||||
"scope": "openid iam.clients.manage", "token_endpoint_auth_method": "none",
|
||||
"subject_type": "public", "metadata": {"iam_login_enabled": true}}'
|
||||
```
|
||||
|
||||
开发实例另需 `iam.clients.admin-ui-client-id=iam-admin-ui`；未配置时 `/console` 不提供，API
|
||||
无法获得可用 token（fail closed）。
|
||||
|
||||
从 Gitea `/user/oauth2/hydra` 发起，应进入新密码/Passkey 页，确认授权后返回原账号，核对
|
||||
仓库权限。当前 Gitea client 未登记 front/back-channel 或 post-logout 回调，不能声称 Gitea 会话会
|
||||
随 IAM 注销。应用的注销协议兼容性需单独验收。旧 `authelia` 登录源始终保留。
|
||||
|
||||
开发配置关闭 Boot 默认 LDAP health indicator：它未配置目录连接，不对应同用户 bind 的
|
||||
AD 仓储。总健康检查仍验证数据库，真实 AD 认证由本次人类登录验收，不将 readiness 当作
|
||||
AD 凭据有效性的证明。
|
||||
|
||||
Hydra 的 login/consent URL 是全局配置，本次影响全部经 Hydra 的登录请求（当前接入 Gitea），
|
||||
不影响直接走 Authelia 的应用。不要在无法访问 Tailscale 开发实例时合并此切换。
|
||||
验收失败时 revert 本次回调变更，恢复旧 Go 的 `/login`、`/consent`；不删除客户端、签名密钥、
|
||||
数据库、用户关联或 Passkey。最终 `auth.ddupan.top` 同源部署另开 PR，不在此次切换范围内。
|
||||
|
||||
## 原 Go/Authelia 路径与回退资料
|
||||
|
||||
目标入口：
|
||||
|
||||
- `https://hydra.ad.ddupan.top`：Hydra 公共 OAuth2/OIDC endpoint。
|
||||
|
||||
+2
-18
@@ -1,20 +1,6 @@
|
||||
serve:
|
||||
public:
|
||||
port: 4444
|
||||
# The iam-login /console admin UI is a public OIDC client that exchanges its code from the
|
||||
# browser, so only that origin may call the public endpoints cross-origin. Server-side
|
||||
# clients such as Gitea are unaffected by CORS.
|
||||
cors:
|
||||
enabled: true
|
||||
allowed_origins:
|
||||
- https://laptop.tail7e769.ts.net:18082
|
||||
allowed_methods:
|
||||
- GET
|
||||
- POST
|
||||
allowed_headers:
|
||||
- Authorization
|
||||
- Content-Type
|
||||
allow_credentials: false
|
||||
admin:
|
||||
port: 4445
|
||||
tls:
|
||||
@@ -26,10 +12,8 @@ urls:
|
||||
self:
|
||||
issuer: https://hydra.ad.ddupan.top
|
||||
public: https://hydra.ad.ddupan.top
|
||||
login: https://laptop.tail7e769.ts.net:18082/oauth2/start
|
||||
consent: https://laptop.tail7e769.ts.net:18082/oauth2/consent
|
||||
logout: https://laptop.tail7e769.ts.net:18082/oauth2/logout
|
||||
post_logout_redirect: https://laptop.tail7e769.ts.net:18082/signin
|
||||
login: https://hydra-login.ad.ddupan.top/login
|
||||
consent: https://hydra-login.ad.ddupan.top/consent
|
||||
ttl:
|
||||
access_token: 15m
|
||||
id_token: 15m
|
||||
|
||||
@@ -1,16 +0,0 @@
|
||||
apiVersion: kustomize.toolkit.fluxcd.io/v1
|
||||
kind: Kustomization
|
||||
metadata:
|
||||
name: flux-image-automation
|
||||
namespace: flux-system
|
||||
spec:
|
||||
dependsOn:
|
||||
- name: external-secrets
|
||||
interval: 10m
|
||||
path: ./platform/flux-image-automation
|
||||
prune: true
|
||||
sourceRef:
|
||||
kind: GitRepository
|
||||
name: flux-system
|
||||
timeout: 5m
|
||||
wait: true
|
||||
File diff suppressed because it is too large
Load Diff
@@ -8,7 +8,6 @@ resources:
|
||||
- apps/external-secrets.yaml
|
||||
- apps/gitea.yaml
|
||||
- apps/backstage.yaml
|
||||
- apps/flux-image-automation.yaml
|
||||
- apps/http-echo.yaml
|
||||
- apps/openebs.yaml
|
||||
- apps/nats.yaml
|
||||
|
||||
@@ -6,6 +6,11 @@ homelab_dns:
|
||||
# Samba remains authoritative for the AD zone. Only these explicitly listed
|
||||
# RRsets are reconciled; Samba-generated AD/Kerberos records are untouched.
|
||||
records:
|
||||
- { zone: ad.ddupan.top, name: incus, type: A, values: [192.168.10.127] }
|
||||
# Static, outside the NEC IX pool (.128-.250); infrastructure/incus/terraform reads
|
||||
# these same values into each container's cloud-init network-config.
|
||||
- { zone: ad.ddupan.top, name: ayatori-dev, type: A, values: [192.168.10.11] }
|
||||
- { zone: ad.ddupan.top, name: ayatori-prod, type: A, values: [192.168.10.12] }
|
||||
- { zone: ad.ddupan.top, name: bao, type: A, values: [192.168.10.8] }
|
||||
- { zone: ad.ddupan.top, name: pve1, type: A, values: [192.168.10.4] }
|
||||
- { zone: ad.ddupan.top, name: pve2, type: A, values: [192.168.10.7] }
|
||||
|
||||
@@ -0,0 +1,147 @@
|
||||
# laptop Incus
|
||||
|
||||
Ansible 管理 Ubuntu 24.04 amd64 上的 Zabbly stable APT 源、公钥、固定包版本和
|
||||
Incus 本地服务。当前版本为 `7.5.1`，完整 Debian 版本见
|
||||
`ansible/group_vars/incus_hosts.yml`。使用系统包，不使用 snap。
|
||||
|
||||
```bash
|
||||
ANSIBLE_LOCAL_TEMP=/tmp/ansible-incus ansible-playbook -i infrastructure/incus/ansible/inventory/hosts.yml infrastructure/incus/ansible/site.yml --syntax-check
|
||||
ANSIBLE_LOCAL_TEMP=/tmp/ansible-incus ansible-playbook -i infrastructure/incus/ansible/inventory/hosts.yml infrastructure/incus/ansible/site.yml --check --diff
|
||||
ANSIBLE_LOCAL_TEMP=/tmp/ansible-incus ansible-playbook -i infrastructure/incus/ansible/inventory/hosts.yml infrastructure/incus/ansible/site.yml
|
||||
```
|
||||
|
||||
执行者需有免密 sudo。首次机器没有包索引时，check 模式不能验证待安装包的可用性；
|
||||
实际执行会刷新该源并校验包签名。重跑应 `changed=0`。
|
||||
|
||||
## 管理边界
|
||||
|
||||
- `site.yml` 管安装；`access.yml` 管 Web UI/OIDC 入口。不执行 `incus admin init`，不创建存储池、profile、实例或受管网络。
|
||||
- 保留 libvirt VM、k3s、现有 bridge、路由和防火墙配置；不引入 OVN/SDN/LB。
|
||||
- HTTPS listener 为 `192.168.10.127:8443`，域名入口经现有 Envoy；不新增 `incus-admin` 成员。本地 `sudo incus` 保留为恢复入口。
|
||||
- etcd 和共享 PostgreSQL 由各自目录管理；此目录不迁移它们。
|
||||
- 不自动删除旧实例或存储；历史 `data/incus` 已由维护者另行授权删除，非本 playbook 行为。
|
||||
|
||||
## 版本与公钥维护
|
||||
|
||||
公钥来自 Zabbly，主指纹为 `4EFC590696CB15B87C73A3AD82CC8797C838DCFD`，
|
||||
与[上游安装说明](https://github.com/zabbly/incus)核对后随配置保存。
|
||||
更新密钥时先核对上游指纹。版本由 APT preferences 固定；升级需修改变量，
|
||||
审阅 `--check --diff` 后执行，不自动降级。
|
||||
上游 stable 源未承诺永久保留旧构建，长期离线重建需另行保存包及依赖。
|
||||
|
||||
## 验证与故障入口
|
||||
|
||||
```bash
|
||||
sudo incus version
|
||||
sudo incus list local:
|
||||
sudo incus storage list local:
|
||||
sudo incus network list local:
|
||||
systemctl status incus.service incus.socket
|
||||
sudo journalctl -u incus.service -n 80 --no-pager
|
||||
```
|
||||
|
||||
网络列表可能展示宿主已有的非受管接口，不代表 Incus 创建了网络。
|
||||
安装验收不等于实例运行验收；存储、实例备份恢复方案留待资源初始化时确定。
|
||||
共享知识入口：`homelab-wiki/services/incus.md`（随本次变更同步）。
|
||||
|
||||
## Web UI 与 Authelia
|
||||
|
||||
入口：`https://incus.ad.ddupan.top`，选择 **Login with SSO**，使用 `panxiao81`
|
||||
并完成 MFA。Authelia 专属 policy 默认 deny，仅此账号可取得 Incus token。
|
||||
该客户端登录者具有 Incus 完整管理权限，不根据当前 AD 组放权。
|
||||
|
||||
CLI 可使用 `incus remote add laptop https://incus.ad.ddupan.top --auth-type=oidc`，
|
||||
按设备码流程在浏览器中完成同一登录；token 由客户端保存在本机，不写入 Git。
|
||||
|
||||
声明分工：
|
||||
|
||||
- `apps/authelia/clients/incus.yaml`：单用户 MFA 准入、public client、PKCE S256、
|
||||
签名 access token、唯一 audience、浏览器回调和 device/refresh grants。
|
||||
- `ansible/group_vars/incus_hosts.yml`：Incus OIDC issuer/client/audience/scopes 和 listener。
|
||||
- `ansible/templates/gateway.yaml.j2`：独立 namespace、Service/EndpointSlice、HTTPRoute、
|
||||
BackendTLSPolicy。入口复用现有通配符证书，后端以 Incus 的公共 server.crt 验证
|
||||
`laptop` SAN，不跳过 TLS 验证。公共证书由 playbook 读取，不复制私钥。
|
||||
- `infrastructure/dns/records.yml`：唯一 DNS 声明；`ansible/dns.yml` 只协调 Incus 与两个 Ayatori 容器的记录。
|
||||
|
||||
入口资源由此 Ansible playbook 管理，尚未交由 Flux。共享 Gateway 的后端 TLS
|
||||
兼容配置由 Flux 管理：EnvoyProxy `eg` 允许 TLS 1.2–1.3，以连接要求 TLS 1.3
|
||||
的 Incus；见 [PR #164](https://git.ddupan.top/panxiao81/homelab-infra/pulls/164)。
|
||||
此补丁不升级网关、不改变前端证书或其他路由。
|
||||
Authelia 当前不是 Flux 受管 release。协调脚本先读取 live Helm values，
|
||||
仅合并 Incus client/policy，固定 chart 0.11.6 渲染后部署；保留 Hydra/Backstage 等
|
||||
现有配置和秘密，禁止以滞后的 `apps/authelia/values.yaml` 全量覆盖线上 release。
|
||||
临时 values 存在 0700 目录、0600 文件中并自动清理，部署前检查 release revision，
|
||||
避免覆盖准备期间的并发 Helm 变更；仍应避免同时升级该 release。
|
||||
|
||||
```bash
|
||||
# 先 site.yml 安装 UI，再接入；均先 --check --diff 再移除这两个参数执行。
|
||||
ANSIBLE_LOCAL_TEMP=/tmp/ansible-incus ansible-playbook -i infrastructure/incus/ansible/inventory/hosts.yml infrastructure/incus/ansible/access.yml --check --diff
|
||||
|
||||
# DNS 使用既有 Samba inventory/collection，仅修改 Incus 的 RRset。
|
||||
cd infrastructure/samba-ad/ansible
|
||||
ANSIBLE_CONFIG=ansible.cfg ANSIBLE_COLLECTIONS_PATH=collections ANSIBLE_LOCAL_TEMP=/tmp/ansible-incus-dns ansible-playbook -i inventory/hosts.yml ../../incus/ansible/dns.yml --check --diff
|
||||
```
|
||||
|
||||
执行接入需要本机 Python3/PyYAML、Helm、kubectl 及现有集群管理凭据；DNS 另需
|
||||
既有 Samba Ansible vault 与 SSH 权限。首次入口 namespace 不存在时 check 使用
|
||||
客户端 dry-run；存在后使用 kubectl diff。后续实际重跑要求 `changed=0`。
|
||||
|
||||
### 故障与迁移
|
||||
|
||||
- Web 入口和 OIDC 依赖 k3s/Envoy/Authelia；它们不可用时使用宿主 `sudo incus`，
|
||||
Incus daemon 与实例生命周期不依赖网页登录。
|
||||
- `8443` 直连仍由 Incus 原生认证保护，但只注册域名 443 的 OIDC callback，
|
||||
浏览器登录应从正式域名进入。网关不透传客户端 TLS 证书；远程主要使用 OIDC。
|
||||
- 更换宿主 Incus 证书后重跑 `access.yml`，同步网关公共信任证书。
|
||||
- 未来切换 IdM：更新 issuer/client/audience/scopes 和新 provider 的准入策略，
|
||||
验证浏览器及 CLI 后再停用 Authelia client；本次不引入永久组模型。
|
||||
- 移除用户准入不会立即撤销已签发的 JWT；紧急撤权需同时处理现有 token/会话。
|
||||
- 回退入口时先撤 HTTPRoute，再关闭 `core.https_address`；保留本地管理和现有实例数据。
|
||||
|
||||
验证包括 DNS、TLS、route conditions、匿名 API 拒绝、PKCE/回调和设备码授权发起。
|
||||
2026-09-25 维护者确认 `panxiao81` 完成 MFA 并成功返回 Incus 控制台。
|
||||
安装、接入、DNS playbook 重跑均 `changed=0`；后续基础容器验收见下节。
|
||||
未验证 Web 实例控制台或 Ayatori 应用数据路径。
|
||||
|
||||
## Ayatori 基础容器
|
||||
|
||||
`terraform/` 只管理 `ayatori-dev`、`ayatori-prod` 两个 Ubuntu 24.04 非特权 LXC，
|
||||
以及专属 ZFS 存储池 `ayatori`（`data/incus-ayatori`）。每个容器上限 2 CPU、
|
||||
2 GiB 内存、20 GiB rootfs，禁用容器 swap，自动启动。没有安装 Ayatori、k0s、
|
||||
kube-apiserver、controller 或数据库；准备 Prod 空容器不代表生产服务已上线。
|
||||
|
||||
网络接入既有 `br0`，固定 MAC 分别为 `02:16:3e:aa:00:01`、`02:16:3e:aa:00:02`。
|
||||
Terraform 通过 `cloud-init.network-config` 配置静态地址：Dev `192.168.10.11`、
|
||||
Prod `192.168.10.12`（`ayatori-{dev,prod}.ad.ddupan.top`）。地址只在
|
||||
`infrastructure/dns/records.yml` 声明一次，Terraform 与 AD DNS 均从此读取；须位于
|
||||
NEC IX DHCP 池（`.128–.250`）之外。⚠ 镜像模板只在创建/复制时渲染 cloud-init seed，
|
||||
对已有实例改地址不会生效（重启、`cloud-init clean` 都不行），只能重建实例。
|
||||
|
||||
SSH 全部由 cloud-init 完成：安装 openssh-server，创建 `panxiao81`（`ansible/files/panxiao81.pub`
|
||||
公钥、免密 sudo），装包后再写 `/etc/ssh/sshd_config.d/60-homelab.conf` 关闭密码与 root 登录。
|
||||
不使用 `ssh_pwauth`：它在装包前写出残缺的 `sshd_config`，包自带的 `UsePAM yes` 落不下来，
|
||||
锁定密码的账号会被拒（2026-09-25 曾发生）。宿主也可使用 `incus exec local:ayatori-dev -- bash`。
|
||||
provider 通过本地 Unix socket 操作，执行账号须有 socket 权限。
|
||||
|
||||
```bash
|
||||
terraform -chdir=infrastructure/incus/terraform init
|
||||
terraform -chdir=infrastructure/incus/terraform plan -out=containers.tfplan
|
||||
terraform -chdir=infrastructure/incus/terraform apply -parallelism=1 containers.tfplan
|
||||
terraform -chdir=infrastructure/incus/terraform output containers
|
||||
terraform -chdir=infrastructure/incus/terraform plan -detailed-exitcode
|
||||
```
|
||||
|
||||
provider 固定 1.2.0 并保留 lockfile。镜像跟随 `images:ubuntu/24.04/cloud` 最新构建，
|
||||
只在创建时使用（`ignore_changes`）；不固定指纹，因为上游会下架旧构建，2026-10-01
|
||||
重建时固定指纹已无法获取。
|
||||
cloud-init 用户设置主要在首次启动执行，修改声明不能替代后续用户/密钥轮换流程。
|
||||
|
||||
当前 state 位于 `terraform/terraform.tfstate`（Git 忽略），是本地 backend；
|
||||
现有 Bao 会话无法读取远端 tfstate 凭据，因此尚未启用远端 backend。
|
||||
后续迁移须使用 `terraform init -migrate-state` 保留资源归属，不创建第二份独立 state。
|
||||
实例和池启用 `prevent_destroy`；删除需显式审阅，不能通过移走整个资源块绕过保护。
|
||||
当前为空系统，丢失后可重建；承载持久数据前须补离机备份与恢复验收。
|
||||
|
||||
首次从同一远端并行创建实例时，Incus 7.5.1 曾出现 simplestreams 缓存目录
|
||||
`mkdir ... file exists` 竞争；按上面的串行 apply 执行。失败后先重新 plan，
|
||||
保留已成功创建的实例，不清理或销毁其资源。
|
||||
@@ -0,0 +1,49 @@
|
||||
---
|
||||
- name: 接入 Incus Web UI 与 Authelia
|
||||
hosts: incus_hosts
|
||||
become: true
|
||||
gather_facts: false
|
||||
tasks:
|
||||
- name: 增量协调 Authelia 的 Incus 客户端
|
||||
become: false
|
||||
ansible.builtin.command:
|
||||
argv: "{{ ['python3', playbook_dir ~ '/../scripts/reconcile_authelia.py', '--desired', playbook_dir ~ '/../../../apps/authelia/clients/incus.yaml'] + (['--check'] if ansible_check_mode else []) }}"
|
||||
register: incus_authelia
|
||||
changed_when: "'changed=true' in incus_authelia.stdout"
|
||||
check_mode: false
|
||||
|
||||
- name: 查询当前 Incus 服务配置
|
||||
ansible.builtin.command: incus query /1.0
|
||||
register: incus_server
|
||||
changed_when: false
|
||||
check_mode: false
|
||||
no_log: true
|
||||
|
||||
- name: 协调 OIDC 与 LAN HTTPS listener
|
||||
ansible.builtin.command:
|
||||
argv: [incus, config, set, "{{ item.key }}={{ item.value }}"]
|
||||
loop: "{{ incus_server_config | dict2items }}"
|
||||
when: (incus_server.stdout | from_json).config.get(item.key, '') != item.value
|
||||
changed_when: true
|
||||
|
||||
- name: 读取 Incus 公共证书用于网关后端验证
|
||||
ansible.builtin.slurp:
|
||||
src: /var/lib/incus/server.crt
|
||||
register: incus_backend_certificate
|
||||
|
||||
- name: 检查入口 namespace 是否已存在
|
||||
become: false
|
||||
ansible.builtin.command: kubectl get namespace incus --ignore-not-found -o name
|
||||
register: incus_namespace
|
||||
changed_when: false
|
||||
check_mode: false
|
||||
|
||||
- name: 渲染并预览或应用专属入口资源
|
||||
become: false
|
||||
ansible.builtin.command:
|
||||
argv: "{{ (['kubectl', 'apply', '--dry-run=client', '-f', '-'] if incus_namespace.stdout == '' else ['kubectl', 'diff', '-f', '-']) if ansible_check_mode else ['kubectl', 'apply', '-f', '-'] }}"
|
||||
stdin: "{{ lookup('template', 'gateway.yaml.j2') }}"
|
||||
register: incus_gateway
|
||||
check_mode: false
|
||||
changed_when: "(incus_gateway.rc == 1 or incus_namespace.stdout == '') if ansible_check_mode else ('created' in incus_gateway.stdout or 'configured' in incus_gateway.stdout)"
|
||||
failed_when: "incus_gateway.rc not in ([0, 1] if ansible_check_mode else [0])"
|
||||
@@ -0,0 +1,29 @@
|
||||
---
|
||||
- name: 仅协调 Incus 与 Ayatori 容器的 AD DNS 记录
|
||||
hosts: samba_dc
|
||||
become: true
|
||||
gather_facts: false
|
||||
vars:
|
||||
incus_dns_names: [incus, ayatori-dev, ayatori-prod]
|
||||
vars_files:
|
||||
- ../../dns/records.yml
|
||||
- ../../samba-ad/ansible/group_vars/all/vars.yml
|
||||
tasks:
|
||||
- name: 从共享清单选择 Incus 相关 RRset
|
||||
ansible.builtin.set_fact:
|
||||
incus_dns_records: "{{ homelab_dns.samba.records | selectattr('name', 'in', incus_dns_names) | selectattr('zone', 'equalto', 'ad.ddupan.top') | list }}"
|
||||
- name: 确认每个名称恰有一个受管 A 记录
|
||||
ansible.builtin.assert:
|
||||
that:
|
||||
- incus_dns_records | length == incus_dns_names | length
|
||||
- incus_dns_records | map(attribute='type') | unique == ['A']
|
||||
- name: 协调 A 记录
|
||||
ddupan.homelab.samba_dns_record:
|
||||
server: "{{ samba_ad_dc_ip }}"
|
||||
zone: "{{ item.zone }}"
|
||||
name: "{{ item.name }}"
|
||||
type: "{{ item.type }}"
|
||||
values: "{{ item['values'] }}"
|
||||
state: present
|
||||
exact: true
|
||||
loop: "{{ incus_dns_records }}"
|
||||
@@ -0,0 +1 @@
|
||||
ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIOLvzIxZhVRd9wEFWR/uCOx7b4HQEdPDiZd8LCN7Hics panxiao81@laptop
|
||||
@@ -0,0 +1,41 @@
|
||||
-----BEGIN PGP PUBLIC KEY BLOCK-----
|
||||
|
||||
mQGNBGTlYcIBDACYQoVXVyQ6Y3Of14GwEaiv/RstQ8jWnH441OtvDbD/VVT8yF0P
|
||||
pUfypWjQS8aq0g32Qgb9H9+b8UAAKojA2W0szjJFlmmSq19YDMMmNC4AnfeZlKYM
|
||||
61Zonna7fPaXmlsTlSiUeo/PGvmAXrkFURC9S8FbhZdWEcUpf9vcKAoEzV8qGA4J
|
||||
xbKlj8EOjSkdq3OQ1hHjP8gynbbzMhZQwjbnWqoiPj35ed9EMn+0QcX+GmynGq6T
|
||||
hBXdRdeQjZC6rmXzNF2opCyxqx3BJ0C7hUtpHegmeoH34wnJHCqGYkEKFAjlRLoW
|
||||
tOzHY9J7OFvB6U7ENtnquj7lg2VQK+hti3uiHW+oide06QgjVw2irucCblQzphgo
|
||||
iX5QJs7tgFFDsA9Ee0DZP6cu83hNFdDcXEZBc9MT5Iu0Ijvj7Oeym3DJpkCuIWgk
|
||||
SeP56sp7333zrg73Ua7YZsZHRayAe/4YdNUua+90P4GD12TpTtJa4iRWRd7bis6m
|
||||
tSkKRj7kxyTsxpEAEQEAAbQmWmFiYmx5IEtlcm5lbCBCdWlsZHMgPGluZm9AemFi
|
||||
Ymx5LmNvbT6JAdQEEwEKAD4CGwMFCwkIBwIGFQoJCAsCBBYCAwECHgECF4AWIQRO
|
||||
/FkGlssVuHxzo62CzIeXyDjc/QUCaKN/OgUJDSQe+AAKCRCCzIeXyDjc/dSYC/47
|
||||
EJPEuRtZCdRFsYVeecQ9CFYcD01DQdS1pfYaK7mgW582aluc1TWAE4J6P8FcCweC
|
||||
tWLC1bY7613ZGCVmoRTHWEOaKYG+NGaR5YRXVkZXcLCmV1KbJ/tkWQD4qIkvuVah
|
||||
Q5J42itFXZ0kz6bs6Wkd6+C2RHL6VtvtVXfVlQtdBni72TgseM01U8WHW6tnweJf
|
||||
XKDXAws8UEc6wQeD4Ik0OCTWbrwQMyDTBn+NTx4Apc2t5QGFi5ehmPbnq0jhF1FB
|
||||
b1gaEmFZLXz/zkDFkj52k/qEPj8099+0sAxld8oQPKWacmGzhBjYzKKHuEQO4Z8t
|
||||
XVlgzCnNlNmWCnkm4AKgTzmKAIgMoA6tUfWBzDy20VZ2J+8dcL52vIJJa30knnLN
|
||||
g3qmqtFTRFQBMl9hC11JOI7qvPmQlt38m6YBEOHBq4QUsuqqVJkQPAtJeROcDbNF
|
||||
aqobwhP5bSsIDMYygTn50LBZtl9LGmLRY4YyZAiVRviXNh5r6lEqDBtjsdnI/Z65
|
||||
AY0EZOVhwgEMAMIztf6WlRsweysb0tzktYE5E/GxIK1lwcD10Jzq3ovJJPa2Tg2t
|
||||
J6ZBmMQfwU4OYO8lJxlgm7t6MYh41ZZaRhySCtbJiAXqK08LP9Gc1iWLRvKuMzli
|
||||
NFSiFDFGT1D6kwucVfL/THxvZlQ559kK+LB4iXEKXz37r+MCX1K9uiv0wn63Vm0K
|
||||
gD3HDgfXWYJcNyXXfJBe3/T5AhuSBOQcpa7Ow5n8zJ+OYg3FFKWHDBTSSZHpbJFr
|
||||
ArMIGARz5/f+EVj9XGY4W/+ZJlxNh8FzrTLeRArmCWqKLPRG/KF36dTY7MDpOzlw
|
||||
vu7frv+cgiXHZ2NfPrkH8oOl4L+ufze5KBGcN0QwFDcuwCkv/7Ft9Ta7gVaIBsK7
|
||||
12oHInUJ6EkBovxpuaLlHlP8IfmZLZbbHzR2gR0e6IhLtrzd7urB+gXUtp6+wCL+
|
||||
kWD14TTJhSQ+SFU8ajvUah7/1m2bxdjZNp9pzOPGkr/jEjCM0CpZiCY62SeIJqVc
|
||||
4/ID9NYLAGmSIwARAQABiQG8BBgBCgAmAhsMFiEETvxZBpbLFbh8c6OtgsyHl8g4
|
||||
3P0FAmijf0cFCQ0kHwUACgkQgsyHl8g43P00BgwAhdg/Vh0zJOCvee9hyf+Wd68F
|
||||
oWz5LUlNGrCsbyNrk27RCR6hM4Td25kLCU03C/aq8a/qiWWgUHho6LpA1t9OsBde
|
||||
59i1wR5Ca6XZAkjBIftlEzuHhg67Dm4mTVSRdTNT/WIhyv5T7Y/ba+TOq7VW8M3D
|
||||
fqwuJSKQ//MUzOcE0pjfH1WI9uFJH+arQBGXD+425lPA/6symWpHm9PHmHwIcd6N
|
||||
Bdc7fjNVRFUjat/auXfcvrDn36PP9w84seBtyeLS20pQtpnL06al6GKOY3rrWPMx
|
||||
4h7fpyURuhQH6nygS/Cxkpf38Zo+EIMajf+19vLhTr+x8HyMfe42GVpEVP5WL43f
|
||||
UcSxG6+cdTm7Yr+PICs4idy62E2y1AGOS5ePHsX4FOAsUquZD5dqhqV/A7Mb+ypk
|
||||
fIqxG8sZAXYIaMrYcDA4ZS7CbuKcSmy0nUws+o7gwSeYLyApBLea/F/ywctODhxh
|
||||
ZBqN6R8SuRc5NWWPDcSdr1myXY2YpB0AVEV8zGtF
|
||||
=tHYp
|
||||
-----END PGP PUBLIC KEY BLOCK-----
|
||||
@@ -0,0 +1,16 @@
|
||||
---
|
||||
# 显式升级版本，避免例行重跑意外升级虚拟化服务。
|
||||
incus_package_version: '1:7.5.1-ubuntu24.04-202609250207'
|
||||
incus_packages:
|
||||
- incus
|
||||
- incus-base
|
||||
- incus-client
|
||||
- incus-ui-canonical
|
||||
|
||||
# 先完成 IdP 准入限制，再配置 OIDC，最后开放 listener。
|
||||
incus_server_config:
|
||||
oidc.issuer: https://auth.ddupan.top
|
||||
oidc.client.id: incus
|
||||
oidc.audience: https://incus.ad.ddupan.top
|
||||
oidc.scopes: openid,offline_access
|
||||
core.https_address: 192.168.10.127:8443
|
||||
@@ -0,0 +1,7 @@
|
||||
all:
|
||||
children:
|
||||
incus_hosts:
|
||||
hosts:
|
||||
laptop:
|
||||
ansible_connection: local
|
||||
ansible_python_interpreter: /usr/bin/python3
|
||||
@@ -0,0 +1,114 @@
|
||||
---
|
||||
- name: 安装 laptop 的 Incus 基础服务
|
||||
hosts: incus_hosts
|
||||
become: true
|
||||
gather_facts: true
|
||||
tasks:
|
||||
- name: 限定已验证的平台
|
||||
ansible.builtin.assert:
|
||||
that:
|
||||
- ansible_facts['distribution'] == 'Ubuntu'
|
||||
- ansible_facts['distribution_release'] == 'noble'
|
||||
- ansible_facts['architecture'] == 'x86_64'
|
||||
fail_msg: 当前包版本只针对 Ubuntu 24.04 amd64 验证。
|
||||
|
||||
- name: 创建 APT 公钥目录
|
||||
ansible.builtin.file:
|
||||
path: /etc/apt/keyrings
|
||||
state: directory
|
||||
owner: root
|
||||
group: root
|
||||
mode: '0755'
|
||||
|
||||
- name: 安装已核对指纹的 Zabbly 公钥
|
||||
ansible.builtin.copy:
|
||||
src: zabbly.asc
|
||||
dest: /etc/apt/keyrings/zabbly.asc
|
||||
owner: root
|
||||
group: root
|
||||
mode: '0644'
|
||||
register: incus_key
|
||||
|
||||
- name: 声明 Zabbly stable 软件源
|
||||
ansible.builtin.copy:
|
||||
content: |
|
||||
Enabled: yes
|
||||
Types: deb
|
||||
URIs: https://pkgs.zabbly.com/incus/stable
|
||||
Suites: noble
|
||||
Components: main
|
||||
Architectures: amd64
|
||||
Signed-By: /etc/apt/keyrings/zabbly.asc
|
||||
dest: /etc/apt/sources.list.d/zabbly-incus-stable.sources
|
||||
owner: root
|
||||
group: root
|
||||
mode: '0644'
|
||||
register: incus_source
|
||||
|
||||
# 防止系统自动更新绕开版本声明；显式改版本后重跑才升级。
|
||||
- name: 固定 Incus 包版本
|
||||
ansible.builtin.copy:
|
||||
content: |
|
||||
Package: {{ incus_packages | join(' ') }}
|
||||
Pin: version {{ incus_package_version }}
|
||||
Pin-Priority: 1000
|
||||
dest: /etc/apt/preferences.d/incus
|
||||
owner: root
|
||||
group: root
|
||||
mode: '0644'
|
||||
|
||||
# 只刷新本组件源，避免其他服务仓库故障阻塞安装。
|
||||
- name: 刷新 Incus 包索引
|
||||
ansible.builtin.command:
|
||||
argv:
|
||||
- apt-get
|
||||
- update
|
||||
- -o
|
||||
- Dir::Etc::sourcelist=sources.list.d/zabbly-incus-stable.sources
|
||||
- -o
|
||||
- Dir::Etc::sourceparts=-
|
||||
- -o
|
||||
- APT::Get::List-Cleanup=0
|
||||
- -o
|
||||
- APT::Update::Error-Mode=any
|
||||
changed_when: false
|
||||
register: incus_refresh
|
||||
retries: 3
|
||||
delay: 5
|
||||
until: incus_refresh.rc == 0
|
||||
when: not ansible_check_mode
|
||||
|
||||
- name: 安装固定版本且禁止移除既有包
|
||||
ansible.builtin.apt:
|
||||
name: "{{ incus_packages | map('regex_replace', '$', '=' ~ incus_package_version) | list }}"
|
||||
state: present
|
||||
install_recommends: false
|
||||
fail_on_autoremove: true
|
||||
lock_timeout: 120
|
||||
environment:
|
||||
# 不让 needrestart 顺带重启 k3s、libvirt 等无关服务。
|
||||
NEEDRESTART_MODE: l
|
||||
register: incus_install
|
||||
retries: 3
|
||||
delay: 5
|
||||
until: incus_install is succeeded
|
||||
# check 模式不会创建新源，APT 无法解析只在新源存在的版本。
|
||||
when: not (ansible_check_mode and (incus_source.changed or incus_key.changed))
|
||||
|
||||
- name: 提示首次 check 的包验证边界
|
||||
ansible.builtin.debug:
|
||||
msg: 软件源或公钥尚待写入，本次仅预览仓库配置；安装后须重跑 check 和幂等验证。
|
||||
when: ansible_check_mode and (incus_source.changed or incus_key.changed)
|
||||
|
||||
- name: 启用 Incus 本地 socket
|
||||
ansible.builtin.systemd_service:
|
||||
name: incus.socket
|
||||
enabled: true
|
||||
state: started
|
||||
when: not ansible_check_mode
|
||||
|
||||
- name: 启动 Incus 服务
|
||||
ansible.builtin.systemd_service:
|
||||
name: incus.service
|
||||
state: started
|
||||
when: not ansible_check_mode
|
||||
@@ -0,0 +1,109 @@
|
||||
apiVersion: v1
|
||||
kind: Namespace
|
||||
metadata:
|
||||
name: incus
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: incus
|
||||
namespace: incus
|
||||
spec:
|
||||
ports:
|
||||
- name: https
|
||||
port: 8443
|
||||
targetPort: 8443
|
||||
---
|
||||
apiVersion: discovery.k8s.io/v1
|
||||
kind: EndpointSlice
|
||||
metadata:
|
||||
name: incus-laptop
|
||||
namespace: incus
|
||||
labels:
|
||||
kubernetes.io/service-name: incus
|
||||
endpointslice.kubernetes.io/managed-by: homelab-ansible
|
||||
addressType: IPv4
|
||||
ports:
|
||||
- name: https
|
||||
protocol: TCP
|
||||
port: 8443
|
||||
endpoints:
|
||||
- addresses: [192.168.10.127]
|
||||
conditions:
|
||||
ready: true
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: ConfigMap
|
||||
metadata:
|
||||
name: incus-backend-ca
|
||||
namespace: incus
|
||||
data:
|
||||
ca.crt: |
|
||||
{{ incus_backend_certificate.content | b64decode | indent(4, true) }}
|
||||
---
|
||||
apiVersion: gateway.networking.k8s.io/v1alpha3
|
||||
kind: BackendTLSPolicy
|
||||
metadata:
|
||||
name: incus
|
||||
namespace: incus
|
||||
spec:
|
||||
targetRefs:
|
||||
- group: ''
|
||||
kind: Service
|
||||
name: incus
|
||||
validation:
|
||||
hostname: laptop
|
||||
caCertificateRefs:
|
||||
- group: ''
|
||||
kind: ConfigMap
|
||||
name: incus-backend-ca
|
||||
---
|
||||
apiVersion: gateway.networking.k8s.io/v1
|
||||
kind: HTTPRoute
|
||||
metadata:
|
||||
name: incus
|
||||
namespace: incus
|
||||
spec:
|
||||
parentRefs:
|
||||
- group: gateway.networking.k8s.io
|
||||
kind: Gateway
|
||||
name: eg
|
||||
namespace: envoy-gateway-system
|
||||
sectionName: https
|
||||
hostnames: [incus.ad.ddupan.top]
|
||||
rules:
|
||||
- matches:
|
||||
- path:
|
||||
type: PathPrefix
|
||||
value: /
|
||||
backendRefs:
|
||||
- group: ''
|
||||
kind: Service
|
||||
name: incus
|
||||
port: 8443
|
||||
weight: 1
|
||||
---
|
||||
apiVersion: gateway.networking.k8s.io/v1
|
||||
kind: HTTPRoute
|
||||
metadata:
|
||||
name: incus-http
|
||||
namespace: incus
|
||||
spec:
|
||||
parentRefs:
|
||||
- group: gateway.networking.k8s.io
|
||||
kind: Gateway
|
||||
name: eg
|
||||
namespace: envoy-gateway-system
|
||||
sectionName: http
|
||||
hostnames: [incus.ad.ddupan.top]
|
||||
rules:
|
||||
- matches:
|
||||
- path:
|
||||
type: PathPrefix
|
||||
value: /
|
||||
filters:
|
||||
- type: RequestRedirect
|
||||
requestRedirect:
|
||||
scheme: https
|
||||
port: 443
|
||||
statusCode: 301
|
||||
@@ -0,0 +1,74 @@
|
||||
#!/usr/bin/env python3
|
||||
"""只协调 Incus client/policy，保留线上其他客户端、秘密和 claims 配置。"""
|
||||
import argparse
|
||||
import copy
|
||||
import json
|
||||
import subprocess
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
|
||||
import yaml
|
||||
|
||||
|
||||
def merge(values, desired):
|
||||
result = copy.deepcopy(values)
|
||||
oidc = result['configMap']['identity_providers']['oidc']
|
||||
clients = oidc.setdefault('clients', [])
|
||||
for client in desired['clients']:
|
||||
matches = [i for i, old in enumerate(clients) if old['client_id'] == client['client_id']]
|
||||
if len(matches) > 1:
|
||||
raise ValueError('发现重复 client_id，拒绝自动覆盖')
|
||||
if matches:
|
||||
clients[matches[0]] = copy.deepcopy(client)
|
||||
else:
|
||||
clients.append(copy.deepcopy(client))
|
||||
oidc.setdefault('authorization_policies', {}).update(desired['authorization_policies'])
|
||||
return result
|
||||
|
||||
|
||||
def run(args):
|
||||
# Helm 输出可能含秘密：只在内存处理，错误不回显正文。
|
||||
result = subprocess.run(args, capture_output=True, text=True)
|
||||
if result.returncode:
|
||||
raise RuntimeError(f'{args[0]} {args[1]} 失败（退出码 {result.returncode}）；未输出可能含秘密的响应')
|
||||
return result.stdout
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument('--desired', required=True)
|
||||
parser.add_argument('--check', action='store_true')
|
||||
args = parser.parse_args()
|
||||
release = json.loads(run(['helm', 'list', '-n', 'authelia', '-o', 'json']))
|
||||
current = next(x for x in release if x['name'] == 'authelia')
|
||||
if current['chart'] != 'authelia-0.11.6' or current['status'] != 'deployed':
|
||||
raise RuntimeError('仅验证 authelia-0.11.6 且 release 必须 deployed；请先审阅版本变化')
|
||||
before = json.loads(run(['helm', 'get', 'values', 'authelia', '-n', 'authelia', '-o', 'json']))
|
||||
desired = yaml.safe_load(Path(args.desired).read_text())
|
||||
after = merge(before, desired)
|
||||
if before == after:
|
||||
print('changed=false: Incus client/policy 已收敛')
|
||||
return
|
||||
|
||||
# 目录 0700，文件 0600，退出即清理；既有秘密不写入仓库或日志。
|
||||
with tempfile.TemporaryDirectory(prefix='incus-authelia-') as directory:
|
||||
root = Path(directory)
|
||||
path = root / 'values.json'
|
||||
path.touch(mode=0o600)
|
||||
path.write_text(json.dumps(after))
|
||||
run(['helm', 'pull', 'authelia/authelia', '--version', '0.11.6', '--destination', directory])
|
||||
chart = str(root / 'authelia-0.11.6.tgz')
|
||||
run(['helm', 'template', 'authelia', chart, '-n', 'authelia', '-f', str(path)])
|
||||
if args.check:
|
||||
print('changed=true: 将只更新 Incus client/policy；固定 chart 渲染通过')
|
||||
return
|
||||
latest = next(x for x in json.loads(run(['helm', 'list', '-n', 'authelia', '-o', 'json'])) if x['name'] == 'authelia')
|
||||
if latest['revision'] != current['revision'] or latest['status'] != 'deployed':
|
||||
raise RuntimeError('Authelia release 在准备期间变化，请重试，避免覆盖并发修改')
|
||||
run(['helm', 'upgrade', 'authelia', chart, '-n', 'authelia', '--reuse-values',
|
||||
'-f', str(path), '--atomic', '--timeout', '5m', '--history-max', '10'])
|
||||
print('changed=true: Incus client/policy 已部署，其他值保留')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,32 @@
|
||||
import unittest
|
||||
|
||||
from reconcile_authelia import merge
|
||||
|
||||
|
||||
class MergeTests(unittest.TestCase):
|
||||
def test_preserves_unrelated_state_and_converges(self):
|
||||
before = {'configMap': {'identity_providers': {'oidc': {
|
||||
'clients': [{'client_id': 'hydra', 'client_secret': 'test-only'}],
|
||||
'claims_policies': {'existing': {'id_token': ['email']}},
|
||||
'authorization_policies': {'existing': {'default_policy': 'two_factor'}},
|
||||
}}}}
|
||||
desired = {'clients': [{'client_id': 'incus', 'public': True}],
|
||||
'authorization_policies': {'incus_admin': {'default_policy': 'deny'}}}
|
||||
after = merge(before, desired)
|
||||
oidc = after['configMap']['identity_providers']['oidc']
|
||||
self.assertEqual(oidc['clients'][0], before['configMap']['identity_providers']['oidc']['clients'][0])
|
||||
self.assertIn('existing', oidc['claims_policies'])
|
||||
self.assertIn('existing', oidc['authorization_policies'])
|
||||
self.assertEqual(len(before['configMap']['identity_providers']['oidc']['clients']), 1)
|
||||
self.assertEqual(merge(after, desired), after)
|
||||
desired['clients'][0]['public'] = False
|
||||
self.assertFalse(merge(after, desired)['configMap']['identity_providers']['oidc']['clients'][1]['public'])
|
||||
|
||||
def test_rejects_duplicate_identity(self):
|
||||
before = {'configMap': {'identity_providers': {'oidc': {'clients': [{'client_id': 'incus'}, {'client_id': 'incus'}]}}}}
|
||||
with self.assertRaises(ValueError):
|
||||
merge(before, {'clients': [{'client_id': 'incus'}], 'authorization_policies': {}})
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
+22
@@ -0,0 +1,22 @@
|
||||
# This file is maintained automatically by "terraform init".
|
||||
# Manual edits may be lost in future updates.
|
||||
|
||||
provider "registry.terraform.io/lxc/incus" {
|
||||
version = "1.2.0"
|
||||
constraints = "1.2.0"
|
||||
hashes = [
|
||||
"h1:9G5MaYQY9mKIpO341aG6f0Bt46DFD/bssrtNB+r861U=",
|
||||
"zh:3be797962ed009eedcd6badb2cce1f707345032d48814f050f2103f00eba0177",
|
||||
"zh:53fd1bf8685ef140372ab3282267fba38219dd2351efa723888a80aa41aa9367",
|
||||
"zh:59a9f9bd027380346b8ebc09da7c98c7ef5aed0783d33aa3d3c4f51ca2fafd0f",
|
||||
"zh:65ca786dd942c068b5953e7bd92c6813b1aeeb8a381da7cc96c45a846f3a5965",
|
||||
"zh:6dd3262124c41f8a416cbd1c289dbf1ab9bf9116fc3f3be4714971272811926a",
|
||||
"zh:8cce2da3db95f1088e02bf7ee68d9cacd55bf7b12dd0dd1e61165d5e5432e38b",
|
||||
"zh:a637c5299aeed3984a0b39b45f4bac026d701a6cc203dff05a82a4f43027f37c",
|
||||
"zh:a9017e39f3e8d2dbbfbbc1c6d663d22465b783d1c0e9a6e3ab324b497d0b14ed",
|
||||
"zh:c3954bf1f4796a529dd76f5617f63f570dee76e9b7c17b12416e3afb059314ba",
|
||||
"zh:dd5a081a9778794d89fa54ccddf4287550e9522d6e69b5e777859857809c9d20",
|
||||
"zh:fa95ea158d045386be416cf9146c74a5e4b3fe5fd85850e1b8f6d2977c45bc9b",
|
||||
"zh:fbe8006406da07ba0c40282050cd34343a584d028890c8601592414044abab7e",
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,147 @@
|
||||
terraform {
|
||||
required_version = ">= 1.10.0"
|
||||
required_providers {
|
||||
incus = {
|
||||
source = "lxc/incus"
|
||||
version = "1.2.0"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
provider "incus" {
|
||||
default_remote = "local"
|
||||
remote {
|
||||
name = "local"
|
||||
address = "unix://"
|
||||
}
|
||||
remote {
|
||||
name = "images"
|
||||
address = "https://images.linuxcontainers.org"
|
||||
protocol = "simplestreams"
|
||||
public = true
|
||||
}
|
||||
}
|
||||
|
||||
# 专用子 dataset，不接管整个宿主 data 池。
|
||||
resource "incus_storage_pool" "ayatori" {
|
||||
name = "ayatori"
|
||||
driver = "zfs"
|
||||
config = {
|
||||
source = "data/incus-ayatori"
|
||||
}
|
||||
lifecycle {
|
||||
prevent_destroy = true
|
||||
}
|
||||
}
|
||||
|
||||
# 地址只在 records.yml 声明一次，AD DNS（ansible/dns.yml）与此处同源读取；
|
||||
# 须位于 NEC IX DHCP 池（.128–.250）之外，池由路由器手工维护，无法声明 reservation。
|
||||
locals {
|
||||
ayatori_ipv4 = {
|
||||
for r in yamldecode(file("${path.module}/../../dns/records.yml")).homelab_dns.samba.records :
|
||||
trimprefix(r.name, "ayatori-") => r.values[0]
|
||||
if r.zone == "ad.ddupan.top" && startswith(r.name, "ayatori-")
|
||||
}
|
||||
}
|
||||
|
||||
# 镜像不含 openssh-server。不设 ssh_pwauth：它会在装包前写出残缺的 sshd_config，
|
||||
# 使包自带的默认配置（UsePAM yes、Include sshd_config.d）无法落地，锁定密码的账号随即被拒。
|
||||
# 改为装包后（defer）再写 drop-in，只覆盖需要收紧的项。
|
||||
locals {
|
||||
ayatori_cloud_config = {
|
||||
manage_etc_hosts = true
|
||||
disable_root = true
|
||||
users = [{
|
||||
name = "panxiao81"
|
||||
groups = ["sudo"]
|
||||
shell = "/bin/bash"
|
||||
sudo = ["ALL=(ALL) NOPASSWD:ALL"]
|
||||
lock_passwd = true
|
||||
ssh_authorized_keys = [trimspace(file("${path.module}/../ansible/files/panxiao81.pub"))]
|
||||
}]
|
||||
# WAN 随机掉线，装包须重试。
|
||||
apt = { conf = "Acquire::Retries \"5\";" }
|
||||
package_update = true
|
||||
packages = ["openssh-server"]
|
||||
write_files = [{
|
||||
path = "/etc/ssh/sshd_config.d/60-homelab.conf"
|
||||
defer = true
|
||||
content = <<-EOT
|
||||
PasswordAuthentication no
|
||||
KbdInteractiveAuthentication no
|
||||
PermitRootLogin no
|
||||
EOT
|
||||
}]
|
||||
}
|
||||
}
|
||||
|
||||
resource "incus_instance" "ayatori" {
|
||||
for_each = toset(["dev", "prod"])
|
||||
name = "ayatori-${each.key}"
|
||||
description = "Ayatori ${each.key} 基础容器；应用由独立部署流程管理"
|
||||
# 跟随 24.04 cloud 最新构建：同一发行版的构建只差安全更新，固定指纹会随上游下架而无法重建。
|
||||
image = "images:ubuntu/24.04/cloud"
|
||||
type = "container"
|
||||
profiles = []
|
||||
running = true
|
||||
config = {
|
||||
"boot.autostart" = "true"
|
||||
"security.privileged" = "false"
|
||||
"security.nesting" = "false"
|
||||
"limits.cpu" = "2"
|
||||
"limits.memory" = "2GiB"
|
||||
"limits.memory.swap" = "false"
|
||||
# ⚠ 镜像模板只在 create/copy 时渲染 cloud-init seed（when: [create, copy]），
|
||||
# 对已有实例修改此键不会生效，重启或 cloud-init clean 也不会；改地址须重建实例。
|
||||
# 网关与 resolver 沿用 LAN DHCP 下发值：.1 网关；Blocky .127 优先、路由器 .1 兜底。
|
||||
"cloud-init.network-config" = yamlencode({
|
||||
version = 2
|
||||
ethernets = {
|
||||
eth0 = {
|
||||
addresses = ["${local.ayatori_ipv4[each.key]}/24"]
|
||||
routes = [{ to = "default", via = "192.168.10.1" }]
|
||||
nameservers = { addresses = ["192.168.10.127", "192.168.10.1"] }
|
||||
}
|
||||
}
|
||||
})
|
||||
"cloud-init.user-data" = "#cloud-config\n${yamlencode(merge(local.ayatori_cloud_config, {
|
||||
hostname = "ayatori-${each.key}"
|
||||
}))}"
|
||||
}
|
||||
device {
|
||||
name = "root"
|
||||
type = "disk"
|
||||
properties = {
|
||||
path = "/"
|
||||
pool = incus_storage_pool.ayatori.name
|
||||
size = "20GiB"
|
||||
}
|
||||
}
|
||||
device {
|
||||
name = "eth0"
|
||||
type = "nic"
|
||||
properties = {
|
||||
name = "eth0"
|
||||
nictype = "bridged"
|
||||
parent = "br0"
|
||||
hwaddr = each.key == "dev" ? "02:16:3e:aa:00:01" : "02:16:3e:aa:00:02"
|
||||
}
|
||||
}
|
||||
wait_for {
|
||||
type = "ipv4"
|
||||
nic = "eth0"
|
||||
}
|
||||
lifecycle {
|
||||
# 镜像只在创建时使用；provider 按字符串比较，改写它不应触发重建现有实例。
|
||||
ignore_changes = [image]
|
||||
prevent_destroy = true
|
||||
}
|
||||
}
|
||||
|
||||
output "containers" {
|
||||
value = { for env, instance in incus_instance.ayatori : env => {
|
||||
name = instance.name
|
||||
ipv4 = instance.ipv4_address
|
||||
mac = instance.mac_address
|
||||
} }
|
||||
}
|
||||
@@ -1,72 +0,0 @@
|
||||
# kata-lab
|
||||
|
||||
`kata-lab` 是位于 Proxmox `pve2` 的可销毁验证环境，用来验证 nested KVM、k3s、
|
||||
Kata Containers，以及后续一 job 一 Kata Pod 的 Gitea Runner。它不是 dev/stg
|
||||
长期服务，也不承载持久数据。
|
||||
|
||||
> 状态：VMID 147 已于 2026-09-14 销毁，Terraform state 为空。目录保留首次验证结果
|
||||
> 与可复现配置；后续验证转向跨 PVE 节点的轻量 LXC worker 方案。
|
||||
|
||||
## Terraform bootstrap
|
||||
|
||||
1. 将 Proxmox API token 写入被 Git 忽略且权限为 `0600` 的
|
||||
`terraform/credentials.auto.tfvars`。
|
||||
2. 取得内部 CA，并仅为当前 Terraform 进程设置 Go TLS trust：
|
||||
|
||||
```bash
|
||||
curl -fsSL https://bao.ad.ddupan.top:8200/v1/pki/ca/pem \
|
||||
-o /tmp/kata-lab-ddupan-ca.pem
|
||||
export SSL_CERT_FILE=/tmp/kata-lab-ddupan-ca.pem
|
||||
```
|
||||
|
||||
3. 初始化并审阅 plan：
|
||||
|
||||
```bash
|
||||
cd infrastructure/kata-lab/terraform
|
||||
terraform init
|
||||
terraform plan
|
||||
```
|
||||
|
||||
Terraform 使用 `laptop:import/noble-server-cloudimg-amd64.qcow2`（40 GiB
|
||||
virtual size），在
|
||||
`pve-rg-hdd` 创建 VM root disk，并把 cloud-init snippet 放到 `laptop`
|
||||
datastore。VM root disk 设为 41 GiB，以容纳 PVE import 的块对齐增量。VM 使用
|
||||
DHCP、`cpu.type = host`、4 vCPU 和 4 GiB 内存；cloud-init
|
||||
安装 `qemu-guest-agent`、单节点 k3s，并记录 `/dev/kvm` 检查结果。
|
||||
|
||||
## Kata Containers
|
||||
|
||||
Ansible 使用 Kata Containers 4.1.0 官方 `kata-deploy` chart。配置明确选择 k3s，
|
||||
只安装 `qemu-runtime-rs`，不安装额外 snapshotter，并采用 `job` 模式，安装结束后
|
||||
不保留 privileged DaemonSet。默认 `RuntimeClass` 名为 `kata`。
|
||||
|
||||
Terraform 输出的 DHCP 地址变化时，先更新 `ansible/inventory/hosts.yml`，然后运行：
|
||||
|
||||
```bash
|
||||
cd infrastructure/kata-lab/ansible
|
||||
ansible-playbook kata.yml
|
||||
ansible-playbook verify.yml
|
||||
```
|
||||
|
||||
`verify.yml` 首先创建一个 `runtimeClassName: kata` 的短生命周期 Pod，并确认 Pod 内
|
||||
看到的 Kata guest kernel 与 k3s node 的宿主 kernel 不同。随后它在另一个 Kata Pod
|
||||
内启动 privileged `dockerd` sidecar，让普通 Docker client 容器通过共享网络命名空间
|
||||
的 `127.0.0.1:2375` 运行一个嵌套容器。这对应后续 Gitea Runner 的目标形态：runner
|
||||
直接执行 job，只在需要 Docker API 时连接同 Pod 的 daemon，而不把 Docker executor
|
||||
作为 job 的外层。Kata 下共享 volume 中的 Unix socket 文件虽然双方可见，但不能跨
|
||||
容器连接，因此不能在这里沿用普通 Pod 常见的 `/var/run/docker.sock` 方案；loopback
|
||||
端口没有通过 Service 暴露到 Pod 外。
|
||||
验证 Pod 会保留以供排查；再次运行时会先替换它们。
|
||||
|
||||
2026-09-13 的首次验证结果：Kata guest kernel 为 `6.18.35`，node host kernel 为
|
||||
`6.8.0-90-generic`；sidecar daemon 成功运行 `busybox:1.37` 嵌套容器。Docker 27.5.1
|
||||
在当前 Kata guest 中无法挂载 overlay2，自动回退到 `vfs`；功能验证通过，但正式用于
|
||||
CI 前需要解决或接受其镜像层复制与磁盘性能开销。
|
||||
|
||||
不要提交 `credentials.auto.tfvars`、Terraform state 或 saved plan。销毁前确认目标
|
||||
仍然是 VMID 147 / `kata-lab`：
|
||||
|
||||
```bash
|
||||
terraform plan -destroy
|
||||
terraform destroy
|
||||
```
|
||||
@@ -1,10 +0,0 @@
|
||||
[defaults]
|
||||
inventory = inventory/hosts.yml
|
||||
host_key_checking = True
|
||||
interpreter_python = auto_silent
|
||||
retry_files_enabled = False
|
||||
local_tmp = /tmp/ansible-kata-lab-local
|
||||
remote_tmp = /tmp/ansible-kata-lab-remote
|
||||
|
||||
[ssh_connection]
|
||||
pipelining = True
|
||||
@@ -1,8 +0,0 @@
|
||||
---
|
||||
all:
|
||||
hosts:
|
||||
kata-lab:
|
||||
ansible_host: 192.168.10.13
|
||||
ansible_user: ansible
|
||||
vars:
|
||||
ansible_become: true
|
||||
@@ -1,88 +0,0 @@
|
||||
---
|
||||
- name: Install Kata Containers in the disposable k3s lab
|
||||
hosts: kata-lab
|
||||
gather_facts: false
|
||||
vars:
|
||||
kata_version: "4.1.0"
|
||||
kata_chart_url: >-
|
||||
https://github.com/kata-containers/kata-containers/releases/download/{{ kata_version }}/kata-deploy-{{ kata_version }}.tgz
|
||||
tasks:
|
||||
- name: Check nested KVM is usable
|
||||
ansible.builtin.stat:
|
||||
path: /dev/kvm
|
||||
register: kata_kvm
|
||||
|
||||
- name: Require nested KVM
|
||||
ansible.builtin.assert:
|
||||
that:
|
||||
- kata_kvm.stat.exists
|
||||
- kata_kvm.stat.ischr
|
||||
fail_msg: /dev/kvm is unavailable; Kata QEMU cannot start efficiently
|
||||
|
||||
- name: Install Kata through the k3s Helm controller
|
||||
ansible.builtin.copy:
|
||||
dest: /var/lib/rancher/k3s/server/manifests/kata-deploy.yaml
|
||||
owner: root
|
||||
group: root
|
||||
mode: "0644"
|
||||
content: |
|
||||
apiVersion: helm.cattle.io/v1
|
||||
kind: HelmChart
|
||||
metadata:
|
||||
name: kata-deploy
|
||||
namespace: kube-system
|
||||
spec:
|
||||
chart: {{ kata_chart_url }}
|
||||
targetNamespace: kata-system
|
||||
createNamespace: true
|
||||
timeout: 15m
|
||||
failurePolicy: abort
|
||||
valuesContent: |-
|
||||
deploymentMode: job
|
||||
k8sDistribution: k3s
|
||||
snapshotter:
|
||||
setup: []
|
||||
shims:
|
||||
disableAll: true
|
||||
qemu-runtime-rs:
|
||||
enabled: true
|
||||
runtimeClasses:
|
||||
enabled: true
|
||||
createDefault: true
|
||||
node-feature-discovery:
|
||||
enabled: false
|
||||
|
||||
- name: Wait for the Helm installation job to appear
|
||||
ansible.builtin.command:
|
||||
cmd: k3s kubectl get job/helm-install-kata-deploy -n kube-system
|
||||
register: kata_helm_job
|
||||
retries: 60
|
||||
delay: 2
|
||||
until: kata_helm_job.rc == 0
|
||||
changed_when: false
|
||||
|
||||
- name: Wait for the Helm installation job to finish
|
||||
ansible.builtin.command:
|
||||
cmd: >-
|
||||
k3s kubectl wait --for=condition=complete
|
||||
job/helm-install-kata-deploy -n kube-system --timeout=20m
|
||||
changed_when: false
|
||||
|
||||
- name: Wait for Kata node installation jobs
|
||||
ansible.builtin.shell:
|
||||
cmd: |
|
||||
set -euo pipefail
|
||||
jobs=$(k3s kubectl get jobs -n kata-system -l app.kubernetes.io/instance=kata-deploy -o name)
|
||||
test -n "${jobs}"
|
||||
k3s kubectl wait --for=condition=complete -n kata-system $jobs --timeout=20m
|
||||
executable: /bin/bash
|
||||
changed_when: false
|
||||
|
||||
- name: Wait for the default Kata RuntimeClass
|
||||
ansible.builtin.command:
|
||||
cmd: k3s kubectl get runtimeclass kata
|
||||
register: kata_runtime_class
|
||||
retries: 30
|
||||
delay: 2
|
||||
until: kata_runtime_class.rc == 0
|
||||
changed_when: false
|
||||
@@ -1,125 +0,0 @@
|
||||
---
|
||||
- name: Verify Kata isolation with a disposable Pod
|
||||
hosts: kata-lab
|
||||
gather_facts: false
|
||||
tasks:
|
||||
- name: Remove an earlier smoke Pod
|
||||
ansible.builtin.command:
|
||||
cmd: k3s kubectl delete pod kata-smoke --ignore-not-found --wait=true
|
||||
changed_when: false
|
||||
|
||||
- name: Create the Kata smoke Pod
|
||||
ansible.builtin.shell:
|
||||
cmd: |
|
||||
set -o pipefail
|
||||
k3s kubectl apply -f - <<'EOF'
|
||||
apiVersion: v1
|
||||
kind: Pod
|
||||
metadata:
|
||||
name: kata-smoke
|
||||
spec:
|
||||
runtimeClassName: kata
|
||||
restartPolicy: Never
|
||||
containers:
|
||||
- name: smoke
|
||||
image: docker.io/library/busybox:1.37
|
||||
command:
|
||||
- sh
|
||||
- -c
|
||||
- 'printf "guest-kernel="; uname -r; test -c /dev/kvm && exit 1 || true'
|
||||
EOF
|
||||
executable: /bin/bash
|
||||
changed_when: true
|
||||
|
||||
- name: Wait for the Kata Pod to finish
|
||||
ansible.builtin.command:
|
||||
cmd: k3s kubectl wait --for=jsonpath='{.status.phase}'=Succeeded pod/kata-smoke --timeout=10m
|
||||
changed_when: false
|
||||
|
||||
- name: Read the guest kernel version
|
||||
ansible.builtin.command:
|
||||
cmd: k3s kubectl logs kata-smoke
|
||||
register: kata_smoke_log
|
||||
changed_when: false
|
||||
|
||||
- name: Read the host kernel version
|
||||
ansible.builtin.command:
|
||||
cmd: uname -r
|
||||
register: kata_host_kernel
|
||||
changed_when: false
|
||||
|
||||
- name: Require a distinct guest kernel
|
||||
ansible.builtin.assert:
|
||||
that:
|
||||
- kata_smoke_log.stdout is match('^guest-kernel=.+')
|
||||
- kata_smoke_log.stdout | regex_replace('^guest-kernel=', '') != kata_host_kernel.stdout
|
||||
success_msg: >-
|
||||
Kata guest {{ kata_smoke_log.stdout }} differs from host-kernel={{ kata_host_kernel.stdout }}
|
||||
|
||||
- name: Remove an earlier Docker sidecar smoke Pod
|
||||
ansible.builtin.command:
|
||||
cmd: k3s kubectl delete pod kata-docker-smoke --ignore-not-found --wait=true
|
||||
changed_when: false
|
||||
|
||||
- name: Create a Kata Pod with a Docker daemon sidecar
|
||||
ansible.builtin.shell:
|
||||
cmd: |
|
||||
set -o pipefail
|
||||
k3s kubectl apply -f - <<'EOF'
|
||||
apiVersion: v1
|
||||
kind: Pod
|
||||
metadata:
|
||||
name: kata-docker-smoke
|
||||
spec:
|
||||
runtimeClassName: kata
|
||||
restartPolicy: Never
|
||||
volumes:
|
||||
- name: docker-run
|
||||
emptyDir: {}
|
||||
containers:
|
||||
- name: dockerd
|
||||
image: docker.io/library/docker:27-dind
|
||||
securityContext:
|
||||
privileged: true
|
||||
env:
|
||||
- name: DOCKER_TLS_CERTDIR
|
||||
value: ""
|
||||
volumeMounts:
|
||||
- name: docker-run
|
||||
mountPath: /var/run
|
||||
- name: client
|
||||
image: docker.io/library/docker:27-cli
|
||||
env:
|
||||
- name: DOCKER_HOST
|
||||
value: tcp://127.0.0.1:2375
|
||||
volumeMounts:
|
||||
- name: docker-run
|
||||
mountPath: /var/run
|
||||
command:
|
||||
- sh
|
||||
- -c
|
||||
- |
|
||||
until docker info >/dev/null 2>&1; do sleep 1; done
|
||||
docker run --rm docker.io/library/busybox:1.37 echo nested-docker-ok
|
||||
EOF
|
||||
executable: /bin/bash
|
||||
changed_when: true
|
||||
|
||||
- name: Wait for the Docker client to finish
|
||||
ansible.builtin.command:
|
||||
cmd: >-
|
||||
k3s kubectl wait --for=jsonpath='{.status.containerStatuses[?(@.name=="client")].state.terminated.exitCode}'=0
|
||||
pod/kata-docker-smoke --timeout=10m
|
||||
changed_when: false
|
||||
|
||||
- name: Read the Docker client result
|
||||
ansible.builtin.command:
|
||||
cmd: k3s kubectl logs kata-docker-smoke -c client
|
||||
register: kata_docker_smoke_log
|
||||
changed_when: false
|
||||
|
||||
- name: Require Docker to run a nested container
|
||||
ansible.builtin.assert:
|
||||
that:
|
||||
- "'nested-docker-ok' in kata_docker_smoke_log.stdout"
|
||||
success_msg: Docker daemon sidecar completed a nested container inside the Kata VM
|
||||
@@ -1,24 +0,0 @@
|
||||
# This file is maintained automatically by "terraform init".
|
||||
# Manual edits may be lost in future updates.
|
||||
|
||||
provider "registry.terraform.io/bpg/proxmox" {
|
||||
version = "0.111.1"
|
||||
constraints = "0.111.1"
|
||||
hashes = [
|
||||
"h1:ML2D3UUZTM99yrll/EBXj7wBYMb8xmQgomqFNybEoxY=",
|
||||
"zh:18fb7c31a08dde6bffa1a4d4a211e604d6d17eec7092fd59331b3db3c6f3742c",
|
||||
"zh:1cd60761538289d4dd2a1086b3ae62a7b0bdd4b1a2f824e9a44e243413168dba",
|
||||
"zh:2eb76f6fc8299b6820ff678c8252332cc3366e226b5ae2e61748fd2449c1ed92",
|
||||
"zh:45e6f7ebd0bf48911d37060359a4f359b5743b3092e985295733990e406d0416",
|
||||
"zh:4aa8ba912eae37975d2e983394d173e595ca34fc76b5bf220b37d0e99d76e98c",
|
||||
"zh:58e0789923103a77d502a0a9fc3eb920625e8eb935ec2d4ac0d006aebd1d186c",
|
||||
"zh:6df8aa85fb8865915537e946c19b02538ad188018a629759c213c6f03730f642",
|
||||
"zh:6ed47bc00d0913a1d0880618fa1376115e9edab6b4a658c081061a7f0e4ca360",
|
||||
"zh:c5b10ff4f33df7e4c29e8f1127d49845b561b37b57517e844fb0954d7923d65e",
|
||||
"zh:d016510e14b738499f0db9d9b3aafe82fc6877fb4ab4e9f831fb68a8d70a1385",
|
||||
"zh:d941f394069bbf24351b363da1c64383f487067aaee0a84f9b96476d4912e212",
|
||||
"zh:ddf271dbc2632ae8ffa8de3972f243ee47d260cb2ac90aa784f2746d98e21a0f",
|
||||
"zh:ed0caa3501c42f611b7e9622c9b1df69fd85dc25a3cd88d3076381829688cd62",
|
||||
"zh:f26e0763dbe6a6b2195c94b44696f2110f7f55433dc142839be16b9697fa5597",
|
||||
]
|
||||
}
|
||||
@@ -1,34 +0,0 @@
|
||||
#cloud-config
|
||||
hostname: ${hostname}
|
||||
manage_etc_hosts: true
|
||||
timezone: Etc/UTC
|
||||
|
||||
users:
|
||||
- default
|
||||
- name: ansible
|
||||
groups: [adm, sudo]
|
||||
shell: /bin/bash
|
||||
lock_passwd: true
|
||||
sudo: ALL=(ALL) NOPASSWD:ALL
|
||||
ssh_authorized_keys:
|
||||
- ${ssh_public_key}
|
||||
|
||||
package_update: true
|
||||
package_upgrade: false
|
||||
packages:
|
||||
- ca-certificates
|
||||
- curl
|
||||
- jq
|
||||
- qemu-guest-agent
|
||||
|
||||
runcmd:
|
||||
- [systemctl, enable, --now, qemu-guest-agent]
|
||||
- [modprobe, kvm_amd]
|
||||
- [sh, -c, 'test -c /dev/kvm && echo available > /var/lib/cloud/nested-kvm.status || echo unavailable > /var/lib/cloud/nested-kvm.status']
|
||||
- [sh, -c, 'curl -sfL https://get.k3s.io -o /tmp/install-k3s.sh']
|
||||
- [chmod, '0755', /tmp/install-k3s.sh]
|
||||
- [sh, -c, 'INSTALL_K3S_VERSION="${k3s_version}" INSTALL_K3S_EXEC="server --disable=traefik --write-kubeconfig-mode=0644" /tmp/install-k3s.sh']
|
||||
- [sh, -c, 'kubectl wait --for=condition=Ready node/${hostname} --timeout=180s']
|
||||
- [touch, /var/lib/cloud/kata-lab-bootstrap.done]
|
||||
|
||||
final_message: "kata-lab bootstrap completed after $UPTIME seconds"
|
||||
@@ -1,3 +0,0 @@
|
||||
# Copy this file to credentials.auto.tfvars and replace the placeholder.
|
||||
# Format: user@realm!token-id=token-secret
|
||||
proxmox_api_token = "REPLACE_ME"
|
||||
@@ -1,86 +0,0 @@
|
||||
locals {
|
||||
ssh_public_key = trimspace(file(pathexpand(var.ssh_public_key_file)))
|
||||
cloud_init = templatefile("${path.module}/cloud-init.yaml.tftpl", {
|
||||
hostname = var.vm_name
|
||||
ssh_public_key = local.ssh_public_key
|
||||
k3s_version = var.k3s_version
|
||||
})
|
||||
}
|
||||
|
||||
data "proxmox_file" "ubuntu_noble" {
|
||||
content_type = "import"
|
||||
datastore_id = var.snippet_datastore_id
|
||||
node_name = var.node_name
|
||||
file_name = "noble-server-cloudimg-amd64.qcow2"
|
||||
}
|
||||
|
||||
resource "proxmox_virtual_environment_file" "cloud_init" {
|
||||
content_type = "snippets"
|
||||
datastore_id = var.snippet_datastore_id
|
||||
node_name = var.node_name
|
||||
|
||||
source_raw {
|
||||
data = local.cloud_init
|
||||
file_name = "${var.vm_name}-cloud-init.yaml"
|
||||
}
|
||||
}
|
||||
|
||||
resource "proxmox_virtual_environment_vm" "kata_lab" {
|
||||
name = var.vm_name
|
||||
description = "Disposable single-node k3s and Kata Containers validation environment."
|
||||
tags = ["terraform", "disposable", "kata"]
|
||||
|
||||
node_name = var.node_name
|
||||
vm_id = var.vm_id
|
||||
|
||||
started = true
|
||||
on_boot = false
|
||||
stop_on_destroy = true
|
||||
|
||||
agent {
|
||||
enabled = true
|
||||
timeout = "15m"
|
||||
}
|
||||
|
||||
cpu {
|
||||
cores = var.vm_cores
|
||||
type = "host"
|
||||
}
|
||||
|
||||
memory {
|
||||
dedicated = var.vm_memory_mb
|
||||
}
|
||||
|
||||
operating_system {
|
||||
type = "l26"
|
||||
}
|
||||
|
||||
scsi_hardware = "virtio-scsi-single"
|
||||
|
||||
disk {
|
||||
datastore_id = var.disk_datastore_id
|
||||
import_from = data.proxmox_file.ubuntu_noble.id
|
||||
interface = "scsi0"
|
||||
iothread = true
|
||||
discard = "on"
|
||||
size = var.vm_disk_gb
|
||||
}
|
||||
|
||||
initialization {
|
||||
datastore_id = var.disk_datastore_id
|
||||
user_data_file_id = proxmox_virtual_environment_file.cloud_init.id
|
||||
|
||||
ip_config {
|
||||
ipv4 {
|
||||
address = "dhcp"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
network_device {
|
||||
bridge = var.network_bridge
|
||||
model = "virtio"
|
||||
}
|
||||
|
||||
serial_device {}
|
||||
}
|
||||
@@ -1,12 +0,0 @@
|
||||
output "vm_id" {
|
||||
value = proxmox_virtual_environment_vm.kata_lab.vm_id
|
||||
}
|
||||
|
||||
output "vm_ipv4_addresses" {
|
||||
description = "QEMU guest agent 报告的地址；忽略 loopback 和 CNI 地址后再用于 SSH。"
|
||||
value = proxmox_virtual_environment_vm.kata_lab.ipv4_addresses
|
||||
}
|
||||
|
||||
output "ssh_user" {
|
||||
value = "ansible"
|
||||
}
|
||||
@@ -1,17 +0,0 @@
|
||||
provider "proxmox" {
|
||||
endpoint = var.proxmox_endpoint
|
||||
api_token = var.proxmox_api_token
|
||||
|
||||
# Snippet uploads use SSH. The provider deliberately does not read
|
||||
# ~/.ssh/config, so map the PVE node explicitly and use the current agent.
|
||||
ssh {
|
||||
agent = false
|
||||
username = "root"
|
||||
private_key = file(pathexpand(var.proxmox_ssh_private_key_file))
|
||||
|
||||
node {
|
||||
name = "pve2"
|
||||
address = "192.168.10.7"
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,83 +0,0 @@
|
||||
variable "proxmox_endpoint" {
|
||||
description = "Proxmox VE API endpoint，不包含 /api2/json。"
|
||||
type = string
|
||||
default = "https://pve1.ad.ddupan.top:8006/"
|
||||
}
|
||||
|
||||
variable "proxmox_api_token" {
|
||||
description = "Proxmox API token，格式为 user@realm!token-id=secret。"
|
||||
type = string
|
||||
sensitive = true
|
||||
}
|
||||
|
||||
variable "node_name" {
|
||||
description = "承载 disposable kata-lab VM 的 PVE 节点。"
|
||||
type = string
|
||||
default = "pve2"
|
||||
}
|
||||
|
||||
variable "vm_id" {
|
||||
description = "kata-lab VMID。"
|
||||
type = number
|
||||
default = 147
|
||||
}
|
||||
|
||||
variable "vm_name" {
|
||||
description = "kata-lab VM 名称和 guest hostname。"
|
||||
type = string
|
||||
default = "kata-lab"
|
||||
}
|
||||
|
||||
variable "vm_memory_mb" {
|
||||
description = "VM 固定内存；pve2 只有约 7.4 GiB 可见内存。"
|
||||
type = number
|
||||
default = 4096
|
||||
}
|
||||
|
||||
variable "vm_cores" {
|
||||
description = "VM vCPU 数量。"
|
||||
type = number
|
||||
default = 4
|
||||
}
|
||||
|
||||
variable "vm_disk_gb" {
|
||||
description = "VM root disk 大小。"
|
||||
type = number
|
||||
default = 41
|
||||
}
|
||||
|
||||
variable "disk_datastore_id" {
|
||||
description = "VM root disk 所在 datastore。"
|
||||
type = string
|
||||
default = "pve-rg-hdd"
|
||||
}
|
||||
|
||||
variable "snippet_datastore_id" {
|
||||
description = "启用 snippets 的共享 datastore。"
|
||||
type = string
|
||||
default = "laptop"
|
||||
}
|
||||
|
||||
variable "network_bridge" {
|
||||
description = "连接 kata-lab VM 的 PVE bridge。"
|
||||
type = string
|
||||
default = "vmbr0"
|
||||
}
|
||||
|
||||
variable "ssh_public_key_file" {
|
||||
description = "注入 cloud-init 用户的 SSH 公钥路径。"
|
||||
type = string
|
||||
default = "~/.ssh/id_ed25519.pub"
|
||||
}
|
||||
|
||||
variable "proxmox_ssh_private_key_file" {
|
||||
description = "provider 上传 snippet 时登录 PVE 节点使用的私钥路径。"
|
||||
type = string
|
||||
default = "~/.ssh/id_ed25519"
|
||||
}
|
||||
|
||||
variable "k3s_version" {
|
||||
description = "可选的固定 k3s 版本；空值使用 stable channel。"
|
||||
type = string
|
||||
default = ""
|
||||
}
|
||||
@@ -1,10 +0,0 @@
|
||||
terraform {
|
||||
required_version = ">= 1.10.0"
|
||||
|
||||
required_providers {
|
||||
proxmox = {
|
||||
source = "bpg/proxmox"
|
||||
version = "0.111.1"
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,180 +0,0 @@
|
||||
# kata-lxc-lab
|
||||
|
||||
在 `pve2` 上验证 privileged LXC 内运行 k3s、Kata/QEMU，并直接使用 PVE host 的
|
||||
`/dev/kvm`、`/dev/vhost-net` 与 `/dev/vhost-vsock`。这是可销毁 PoC，不承载持久数据。
|
||||
|
||||
Terraform 复用原 VM PoC 的 ignored credential 文件，不复制 token：
|
||||
|
||||
```bash
|
||||
cd infrastructure/kata-lxc-lab/terraform
|
||||
export SSL_CERT_FILE=/tmp/kata-lab-ddupan-ca.pem
|
||||
terraform init
|
||||
terraform plan \
|
||||
-var-file=../../kata-lab/terraform/credentials.auto.tfvars
|
||||
terraform apply \
|
||||
-var-file=../../kata-lab/terraform/credentials.auto.tfvars
|
||||
```
|
||||
|
||||
当前 PoC 使用 `pve2`、VMID 147、Ubuntu 24.04 LXC template 和 16 GiB
|
||||
`local-lvm` rootfs。容器为 privileged，并开启 nesting、keyctl、FUSE、mknod；
|
||||
Kata 所需的 KVM、vhost 与 `/dev/net/tun` 设备显式透传；另透传 `/dev/kmsg`，因为
|
||||
kubelet 启动时会打开该设备。
|
||||
PVE 9 的 `force_rw_sys=1` feature 用于开放 kubelet 所需的少量 `/proc/sys` 写操作；
|
||||
provider 0.111.1 尚未暴露该 feature，因此同样由 `pct` 设置。
|
||||
|
||||
k3s/Kata worker 还使用以下 privileged LXC 原生配置；PVE 9 使用 cgroup v2，因此设备
|
||||
规则采用 `lxc.cgroup2.devices.allow`：
|
||||
|
||||
```text
|
||||
lxc.apparmor.profile: unconfined
|
||||
lxc.cgroup2.devices.allow: a
|
||||
lxc.cap.drop:
|
||||
lxc.mount.auto: proc:rw sys:rw
|
||||
```
|
||||
|
||||
这些设置让容器内 kubelet、containerd 和 Kata shim 能管理 mount、cgroup、设备与
|
||||
共享宿主 sysctl。因此外层 LXC 不是安全边界；不可信 CI 的安全边界是内层 Kata VM。
|
||||
`/dev/kmsg` 当前直接透传，没有使用 `/dev/console` symlink fallback。
|
||||
|
||||
PVE 将 privileged LXC 的创建限制为 `root@pam`（API token 即使拥有 `PVEAdmin` 也
|
||||
缺少 root-only 的 `Sys.Modify`）。因此实际创建入口使用 root SSH 上的 Ansible/pct：
|
||||
|
||||
```bash
|
||||
cd infrastructure/kata-lxc-lab/ansible
|
||||
ansible-playbook create.yml
|
||||
ansible-playbook bootstrap.yml
|
||||
ansible-playbook kata.yml
|
||||
ansible-playbook verify.yml
|
||||
ansible-playbook verify-fuse.yml
|
||||
```
|
||||
|
||||
Terraform 文件保留用于记录 provider 权限边界与声明式目标；当前 token 执行 apply 会
|
||||
在创建前返回 403，不会产生资源或 state。
|
||||
|
||||
## 验证结果
|
||||
|
||||
当前管理地址为 `192.168.10.128`：2026-09-14 主 LAN DHCP 池调整后，从 `.11`
|
||||
续租迁移到 `.128`，并同步 Ansible inventory。它仍是动态租约，不是固定地址。
|
||||
|
||||
2026-09-14，pve2 VMID 147 / `192.168.10.11` 验证通过：
|
||||
|
||||
- LXC 内 k3s `v1.36.4+k3s1` node Ready；
|
||||
- Kata Containers 4.1.0 `qemu-runtime-rs` 安装成功；
|
||||
- Kata guest kernel `6.18.35`，外层共享的 PVE kernel `7.0.2-6-pve`；
|
||||
- Kata Pod 内的 Docker 27.5.1 daemon 成功运行 `busybox:1.37` 嵌套容器；
|
||||
- `/dev/net/tun` 是 Kata 创建 TAP/link 的必要设备，未透传时 runtime-rs 在
|
||||
`create link` 返回 `ENOENT`；
|
||||
- `fuse-overlayfs` smoke test 成功：Docker 报告
|
||||
`storage-driver=fuse-overlayfs`，并成功运行嵌套容器。
|
||||
|
||||
正式方案保持 Kata 默认 `emptydir_mode = "shared-fs"`，不向 LXC 暴露 host
|
||||
`/dev/loop-control` 或 `/dev/loopN`。dockerd 镜像需要包含 `fuse-overlayfs`；启动前在
|
||||
Kata guest 内创建临时设备节点 `mknod /dev/fuse c 10 229`，随后强制传入
|
||||
`--storage-driver=fuse-overlayfs`。该 `/dev/fuse` 属于内层 guest kernel，不是 PVE
|
||||
host 设备透传，并随 Kata Pod 一起销毁。
|
||||
|
||||
删除三个 smoke Pod 后，LXC cgroup 统计约 505 MB anonymous memory、1.34 GB file
|
||||
cache；后者主要来自刚拉取的 Kata/Docker images，可由宿主回收。也就是说常驻 worker
|
||||
的不可回收 working set 约 500 MB，8 GiB 配置是 cgroup 上限而非预分配。
|
||||
|
||||
## Gitea ephemeral runner PoC
|
||||
|
||||
`runner/` 定义一次性 Gitea runner Pod。整个 Pod 使用 `runtimeClassName: kata`；
|
||||
runner 通过 guest 内同 Pod 的 dockerd 创建 job container，不连接 LXC/PVE host 的
|
||||
Docker socket。dockerd 使用 zot 中预先发布的 `dind-fuse:29.7.1`，其
|
||||
`/var/lib/docker` 是随 Kata Pod 删除的 `emptyDir`。
|
||||
|
||||
runner 以 `GITEA_RUNNER_EPHEMERAL=1` 注册，只接收一个 `runs-on: kata-poc` job；接单
|
||||
后 Gitea 撤销其 runner credential，job 完成后进程与 Kata VM 一起退出。当前 PoC
|
||||
手动创建一个 runner 等待 `.gitea/workflows/kata-runner-smoke.yml`，尚未部署
|
||||
`workflow_job` webhook controller，因此不会保留常驻空闲 runner。
|
||||
|
||||
注册 token 只在运行时从现有 OpenBao/ESO 消费副本投递到 PoC 集群的临时 Secret，
|
||||
不写入此目录。两个 Pod image 使用 `imagePullPolicy: Never`，部署前从可信工作站将
|
||||
已验证镜像导入 PoC k3s containerd；这是当前 zot 拉取仍要求 SPIFFE 身份时的
|
||||
bootstrap 边界，不是正式镜像分发方案。
|
||||
|
||||
正式接入 zot 前需要把这个 cluster 注册到 SPIRE，并为 runner Pod 创建专用
|
||||
`ClusterSPIFFEID`。届时 runner 与 dockerd sidecar 在同一路径只读挂载 CSI Workload
|
||||
API socket，runner 的 `container.options` 再把该路径 bind-mount 到 job container。
|
||||
zot 只对最终 SPIFFE ID 的 `panxiao81/dind-fuse` 仓库授予
|
||||
`read/create/update`；禁止授予 namespace 或整个 trust domain 写权限。
|
||||
|
||||
2026-09-14 已完成真实 Gitea job 验收：Actions run `242` / job `445` 使用
|
||||
`alpine:3.22`，10 秒成功；runner container 退出码为 `0`，原生 dockerd sidecar 随后
|
||||
由 kubelet 终止，Pod 最终为 `Succeeded`，没有保留空闲 Kata VM。guest 内核为
|
||||
`6.18.35`，dockerd `29.7.1` 使用 `fuse-overlayfs`。
|
||||
|
||||
首次尝试暴露了两个部署陷阱：`COPY` 默认保留源文件 mode，入口脚本原为 `0664`，
|
||||
必须在 Dockerfile 使用 `COPY --chmod=0755`；只配置 `ephemeral-storage: 20Gi` limit
|
||||
会形成同值 request，使 12 GiB PoC 节点无法调度。当前 manifest 显式 request 1 GiB、
|
||||
limit 8 GiB。完整 `ubuntu-latest` job image 也不适合这个小 rootfs，基础生命周期验收
|
||||
改用 Alpine；正式 worker 必须扩容本地 image/cache 空间或使用更精简的 job image。
|
||||
|
||||
## Cloud Hypervisor 内存记账验证
|
||||
|
||||
2026-09-17 在 pve2 的一次性 VMID 148 中安装 k3s v1.36.4+k3s1 与 Kata 4.1.0，
|
||||
只启用 `clh-runtime-rs`，验证 Cloud Hypervisor 在 privileged LXC 内不会重现早期
|
||||
microVM PoC 的 private memfd 双重记账问题。测试完成后已删除 VMID 148。
|
||||
|
||||
Cloud Hypervisor `/api/v1/vm.info` 报告：
|
||||
|
||||
```json
|
||||
{"size":3254779904,"shared":true,"hugepages":false,"prefault":false,"thp":true}
|
||||
```
|
||||
|
||||
Kata guest 在 memory-backed `emptyDir` 中写入 1 GiB 后：
|
||||
|
||||
- VMM `Pss_Shmem` 从约 247 MiB 增至约 1269 MiB；
|
||||
- VMM `Pss_Anon` 写入前后均约 1.2 MiB；
|
||||
- 外层 LXC `shmem` 增加约 1 GiB，`anon` 基本不变；
|
||||
- LXC `memory.events` 的 `max`、`oom` 与 `oom_kill` 均为 0；
|
||||
- 删除 Pod 后 VMM 退出，LXC `shmem` 回落到约 1.6 MiB。
|
||||
|
||||
因此 Kata 的 Cloud Hypervisor handler 已使用 shared memfd，等价于独立 launcher 的
|
||||
`--memory shared=on` 修复；后续 OpenSandbox/Kata 方案无需为 guest RAM 预留近似双倍
|
||||
的 LXC cgroup 内存。部署验收仍应检查 `vm.info.config.memory.shared=true`，防止上游
|
||||
配置或 runtime handler 变化造成回归。
|
||||
|
||||
测试期间还发现 pve2 `local-lvm` thin pool 已达到 100%；临时 rootfs 必须放到
|
||||
`pve-rg-hdd`，不能根据容器内部 `df` 的剩余空间判断 thin pool 是否可写。
|
||||
|
||||
## Block-backed emptyDir 与 CI 构建验证
|
||||
|
||||
2026-09-17 又在 pve2 的一次性 VMID 148 中验证 Kata 4.1.0
|
||||
`clh-runtime-rs` 的以下 drop-in：
|
||||
|
||||
```toml
|
||||
[runtime]
|
||||
emptydir_mode = "block-plain"
|
||||
```
|
||||
|
||||
结果如下：
|
||||
|
||||
- 普通 Kubernetes `emptyDir` 被创建为稀疏 `disk.img`，由 Cloud Hypervisor 直接作为
|
||||
block device 热插拔；guest 内 `/var/lib/docker` 是 `/dev/vdb` 上的 ext4。
|
||||
- Docker 29.1.5 成功强制使用原生 `overlay2`，不再需要 `fuse-overlayfs`；整个过程
|
||||
没有使用 PVE host 或 LXC 的 loop device。
|
||||
- `emptyDir.sizeLimit: 8Gi` 没有决定虚拟磁盘容量。由于 kubelet 所在文件系统约
|
||||
12 GiB，guest 看到的 ext4 约 11.7 GiB；这与 Kata 已知的 block-backed emptyDir
|
||||
限制一致。
|
||||
- 包含 256 MiB payload、2000 个小文件和一个额外复制层的冷 BuildKit 构建耗时
|
||||
46 秒；同一 LXC、同一 DRBD HDD 存储上的 runc 对照为 34 秒，Kata 约慢 35%。
|
||||
- 在 overlay2 容器层内写入并 fsync 512 MiB：Kata 为 12 秒（40.5 MB/s），runc 为
|
||||
10 秒（50.3 MB/s）；创建一万个小文件两者均约 2 秒。
|
||||
- kind v0.27.0 / Kubernetes v1.32.2 首次启动失败是因为 Kata guest 内的 dockerd
|
||||
容器没有 `/dev/kmsg`，不是 block storage 或 cgroup 故障。在 privileged dockerd
|
||||
启动前执行 `mknod /dev/kmsg c 1 11` 后，kind 创建成功：总计 122 秒，进入等待阶段
|
||||
后 23 秒 control-plane Ready，随后可正常删除。
|
||||
- 测试时稀疏 backing file 逻辑大小约 12 GiB，构建和 kind 后实际占用约 1.8 GiB。
|
||||
删除 Pod 后 backing file、Cloud Hypervisor 进程和 kind 容器全部消失；host/LXC
|
||||
均无 loop device 残留，LXC 没有 OOM 事件。
|
||||
|
||||
因此 `block-plain` 能解决 virtio-fs 不能作为 overlayfs upperdir 的问题，并满足
|
||||
BuildKit 与 kind 的功能要求。正式 CI RuntimeClass 必须使用独立的 block-backed
|
||||
配置，并在 dockerd bootstrap 中创建 guest `/dev/kmsg`。它仍比同机 runc 构建慢约
|
||||
20–35%，且 `emptyDir.sizeLimit` 不能可靠限制虚拟磁盘容量。PoC 曾据此建议为 kubelet
|
||||
目录提供独立文件系统；正式 sandbox 集群最终没有采用该设计：独立 volume 增加了
|
||||
DRBD 与 LXC 挂载生命周期复杂度，却没有隔离 containerd 等其他节点数据。正式节点
|
||||
改用单一、容量明确的 rootfs，并以磁盘占用监控、Pod 删除后的 backing-file 回收检查
|
||||
和实际构建基准作为上线门槛。
|
||||
@@ -1,10 +0,0 @@
|
||||
[defaults]
|
||||
inventory = inventory/hosts.yml
|
||||
host_key_checking = True
|
||||
interpreter_python = auto_silent
|
||||
retry_files_enabled = False
|
||||
local_tmp = /tmp/ansible-kata-lxc-local
|
||||
remote_tmp = /tmp/ansible-kata-lxc-remote
|
||||
|
||||
[ssh_connection]
|
||||
pipelining = True
|
||||
@@ -1,58 +0,0 @@
|
||||
---
|
||||
- name: Bootstrap k3s in the Kata LXC worker
|
||||
hosts: pve2
|
||||
gather_facts: false
|
||||
vars:
|
||||
kata_lxc_id: 147
|
||||
tasks:
|
||||
- name: Check base packages in LXC
|
||||
ansible.builtin.command:
|
||||
cmd: >-
|
||||
pct exec {{ kata_lxc_id }} -- dpkg-query -W
|
||||
ca-certificates curl jq openssh-server
|
||||
register: kata_lxc_packages
|
||||
changed_when: false
|
||||
failed_when: false
|
||||
|
||||
- name: Install base packages and SSH server in LXC
|
||||
ansible.builtin.command:
|
||||
cmd: >-
|
||||
pct exec {{ kata_lxc_id }} -- bash -lc
|
||||
'apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y
|
||||
ca-certificates curl jq openssh-server'
|
||||
when: kata_lxc_packages.rc != 0
|
||||
|
||||
- name: Install k3s in LXC
|
||||
ansible.builtin.shell:
|
||||
cmd: |
|
||||
set -euo pipefail
|
||||
pct exec {{ kata_lxc_id }} -- bash -lc '
|
||||
if ! command -v k3s >/dev/null; then
|
||||
curl -sfL https://get.k3s.io -o /tmp/install-k3s.sh
|
||||
chmod 0755 /tmp/install-k3s.sh
|
||||
INSTALL_K3S_EXEC="server --disable=traefik --write-kubeconfig-mode=0644" /tmp/install-k3s.sh
|
||||
fi
|
||||
'
|
||||
executable: /bin/bash
|
||||
register: kata_lxc_k3s_install
|
||||
changed_when: "'No change detected' not in kata_lxc_k3s_install.stdout"
|
||||
|
||||
- name: Wait for k3s node readiness
|
||||
ansible.builtin.command:
|
||||
cmd: >-
|
||||
pct exec {{ kata_lxc_id }} -- /usr/local/bin/k3s kubectl wait
|
||||
--for=condition=Ready node/kata-lxc-lab --timeout=180s
|
||||
changed_when: false
|
||||
|
||||
- name: Check LXC virtualization and devices
|
||||
ansible.builtin.command:
|
||||
cmd: >-
|
||||
pct exec {{ kata_lxc_id }} -- bash -lc
|
||||
'systemd-detect-virt; ls -l /dev/kvm /dev/vhost-net /dev/vhost-vsock;
|
||||
/usr/local/bin/k3s kubectl get node -o wide'
|
||||
register: kata_lxc_ready
|
||||
changed_when: false
|
||||
|
||||
- name: Report k3s readiness
|
||||
ansible.builtin.debug:
|
||||
var: kata_lxc_ready.stdout_lines
|
||||
@@ -1,137 +0,0 @@
|
||||
---
|
||||
- name: Create the disposable Kata LXC worker
|
||||
hosts: pve2
|
||||
gather_facts: false
|
||||
vars:
|
||||
kata_lxc_id: 147
|
||||
kata_lxc_template: laptop:vztmpl/ubuntu-24.04-standard_24.04-2_amd64.tar.zst
|
||||
tasks:
|
||||
- name: Check whether the LXC already exists
|
||||
ansible.builtin.stat:
|
||||
path: /etc/pve/lxc/{{ kata_lxc_id }}.conf
|
||||
register: kata_lxc_config
|
||||
|
||||
- name: Create privileged LXC with Kata host devices
|
||||
ansible.builtin.command:
|
||||
argv:
|
||||
- pct
|
||||
- create
|
||||
- "{{ kata_lxc_id }}"
|
||||
- "{{ kata_lxc_template }}"
|
||||
- --hostname
|
||||
- kata-lxc-lab
|
||||
- --ostype
|
||||
- ubuntu
|
||||
- --rootfs
|
||||
- local-lvm:12
|
||||
- --cores
|
||||
- "4"
|
||||
- --memory
|
||||
- "8192"
|
||||
- --swap
|
||||
- "0"
|
||||
- --net0
|
||||
- name=eth0,bridge=vmbr0,ip=dhcp
|
||||
- --features
|
||||
- nesting=1,keyctl=1,fuse=1,mknod=1,force_rw_sys=1
|
||||
- --unprivileged
|
||||
- "0"
|
||||
- --onboot
|
||||
- "0"
|
||||
- --ssh-public-keys
|
||||
- /root/.ssh/authorized_keys
|
||||
- --dev0
|
||||
- path=/dev/kvm,mode=0660
|
||||
- --dev1
|
||||
- path=/dev/vhost-net,mode=0660
|
||||
- --dev2
|
||||
- path=/dev/vhost-vsock,mode=0660
|
||||
- --dev3
|
||||
- path=/dev/kmsg,mode=0660
|
||||
- --dev4
|
||||
- path=/dev/net/tun,mode=0666
|
||||
- --tags
|
||||
- disposable;kata;lxc;ansible
|
||||
when: not kata_lxc_config.stat.exists
|
||||
|
||||
- name: Read LXC status
|
||||
ansible.builtin.command:
|
||||
cmd: pct status {{ kata_lxc_id }}
|
||||
register: kata_lxc_status
|
||||
changed_when: false
|
||||
|
||||
- name: Read current LXC configuration
|
||||
ansible.builtin.command:
|
||||
cmd: pct config {{ kata_lxc_id }}
|
||||
register: kata_lxc_current_config
|
||||
changed_when: false
|
||||
|
||||
- name: Ensure kubelet kernel log device is present
|
||||
ansible.builtin.command:
|
||||
cmd: pct set {{ kata_lxc_id }} --dev3 path=/dev/kmsg,mode=0660
|
||||
register: kata_lxc_kmsg
|
||||
when: "'/dev/kmsg' not in kata_lxc_current_config.stdout"
|
||||
|
||||
- name: Enable writable sysctls required by kubelet
|
||||
ansible.builtin.command:
|
||||
cmd: >-
|
||||
pct set {{ kata_lxc_id }} --features
|
||||
nesting=1,keyctl=1,fuse=1,mknod=1,force_rw_sys=1
|
||||
register: kata_lxc_rw_sys
|
||||
when: "'force_rw_sys=1' not in kata_lxc_current_config.stdout"
|
||||
|
||||
- name: Ensure TUN device required by Kata networking is present
|
||||
ansible.builtin.command:
|
||||
cmd: pct set {{ kata_lxc_id }} --dev4 path=/dev/net/tun,mode=0666
|
||||
register: kata_lxc_tun
|
||||
when: "'/dev/net/tun' not in kata_lxc_current_config.stdout"
|
||||
|
||||
- name: Configure privileged nested-runtime LXC directives
|
||||
ansible.builtin.lineinfile:
|
||||
path: /etc/pve/lxc/{{ kata_lxc_id }}.conf
|
||||
regexp: "^{{ item.key | regex_escape }}:"
|
||||
line: "{{ item.key }}: {{ item.value }}"
|
||||
loop:
|
||||
- key: lxc.apparmor.profile
|
||||
value: unconfined
|
||||
- key: lxc.cgroup2.devices.allow
|
||||
value: a
|
||||
- key: lxc.cap.drop
|
||||
value: ""
|
||||
- key: lxc.mount.auto
|
||||
value: proc:rw sys:rw
|
||||
register: kata_lxc_raw_config
|
||||
|
||||
- name: Restart LXC after device configuration change
|
||||
ansible.builtin.command:
|
||||
cmd: pct reboot {{ kata_lxc_id }}
|
||||
when: >-
|
||||
kata_lxc_kmsg.changed or kata_lxc_rw_sys.changed or kata_lxc_tun.changed or
|
||||
kata_lxc_raw_config.changed
|
||||
|
||||
- name: Start LXC
|
||||
ansible.builtin.command:
|
||||
cmd: pct start {{ kata_lxc_id }}
|
||||
when: "'status: stopped' in kata_lxc_status.stdout"
|
||||
|
||||
- name: Wait for systemd in LXC
|
||||
ansible.builtin.command:
|
||||
cmd: pct exec {{ kata_lxc_id }} -- systemctl is-system-running --wait
|
||||
register: kata_lxc_systemd
|
||||
retries: 30
|
||||
delay: 2
|
||||
until: kata_lxc_systemd.rc in [0, 1]
|
||||
changed_when: false
|
||||
failed_when: kata_lxc_systemd.rc not in [0, 1]
|
||||
|
||||
- name: Show LXC address and Kata devices
|
||||
ansible.builtin.command:
|
||||
cmd: >-
|
||||
pct exec {{ kata_lxc_id }} -- sh -c
|
||||
'hostname -I; ls -l /dev/kvm /dev/vhost-net /dev/vhost-vsock /dev/net/tun'
|
||||
register: kata_lxc_facts
|
||||
changed_when: false
|
||||
|
||||
- name: Report LXC address and devices
|
||||
ansible.builtin.debug:
|
||||
var: kata_lxc_facts.stdout_lines
|
||||
@@ -1,9 +0,0 @@
|
||||
---
|
||||
all:
|
||||
hosts:
|
||||
pve2:
|
||||
ansible_host: 192.168.10.7
|
||||
ansible_user: root
|
||||
kata-lab:
|
||||
ansible_host: 192.168.10.128
|
||||
ansible_user: root
|
||||
@@ -1,3 +0,0 @@
|
||||
---
|
||||
# Reuse the proven Kata 4.1.0 installation playbook against the LXC inventory.
|
||||
- import_playbook: ../../kata-lab/ansible/kata.yml
|
||||
@@ -1,73 +0,0 @@
|
||||
---
|
||||
- name: Verify fuse-overlayfs for Docker inside Kata
|
||||
hosts: kata-lab
|
||||
gather_facts: false
|
||||
tasks:
|
||||
- name: Remove an earlier fuse-overlayfs smoke Pod
|
||||
ansible.builtin.command:
|
||||
cmd: k3s kubectl delete pod kata-fuse-smoke --ignore-not-found --wait=true
|
||||
changed_when: false
|
||||
|
||||
- name: Create Kata Docker Pod using fuse-overlayfs
|
||||
ansible.builtin.shell:
|
||||
cmd: |
|
||||
set -o pipefail
|
||||
k3s kubectl apply -f - <<'EOF'
|
||||
apiVersion: v1
|
||||
kind: Pod
|
||||
metadata:
|
||||
name: kata-fuse-smoke
|
||||
spec:
|
||||
runtimeClassName: kata
|
||||
restartPolicy: Never
|
||||
containers:
|
||||
- name: dockerd
|
||||
image: docker.io/library/docker:27-dind
|
||||
securityContext:
|
||||
privileged: true
|
||||
env:
|
||||
- name: DOCKER_TLS_CERTDIR
|
||||
value: ""
|
||||
command:
|
||||
- sh
|
||||
- -c
|
||||
- |
|
||||
apk add --no-cache fuse-overlayfs
|
||||
test -e /dev/fuse || mknod /dev/fuse c 10 229
|
||||
chmod 0666 /dev/fuse
|
||||
exec dockerd-entrypoint.sh --storage-driver=fuse-overlayfs
|
||||
- name: client
|
||||
image: docker.io/library/docker:27-cli
|
||||
env:
|
||||
- name: DOCKER_HOST
|
||||
value: tcp://127.0.0.1:2375
|
||||
command:
|
||||
- sh
|
||||
- -c
|
||||
- |
|
||||
until docker info >/dev/null 2>&1; do sleep 1; done
|
||||
docker info --format 'storage-driver={{ "{{" }}.Driver{{ "}}" }}'
|
||||
docker run --rm docker.io/library/busybox:1.37 echo fuse-nested-docker-ok
|
||||
EOF
|
||||
executable: /bin/bash
|
||||
changed_when: true
|
||||
|
||||
- name: Wait for the Docker client to finish
|
||||
ansible.builtin.command:
|
||||
cmd: >-
|
||||
k3s kubectl wait --for=jsonpath='{.status.containerStatuses[?(@.name=="client")].state.terminated.exitCode}'=0
|
||||
pod/kata-fuse-smoke --timeout=10m
|
||||
changed_when: false
|
||||
|
||||
- name: Read Docker client result
|
||||
ansible.builtin.command:
|
||||
cmd: k3s kubectl logs kata-fuse-smoke -c client
|
||||
register: kata_fuse_smoke_log
|
||||
changed_when: false
|
||||
|
||||
- name: Require fuse-overlayfs and nested Docker
|
||||
ansible.builtin.assert:
|
||||
that:
|
||||
- "'storage-driver=fuse-overlayfs' in kata_fuse_smoke_log.stdout"
|
||||
- "'fuse-nested-docker-ok' in kata_fuse_smoke_log.stdout"
|
||||
success_msg: Docker used fuse-overlayfs and completed a nested container inside Kata
|
||||
@@ -1,3 +0,0 @@
|
||||
---
|
||||
# Reuse the guest-kernel and Docker sidecar smoke tests against the LXC worker.
|
||||
- import_playbook: ../../kata-lab/ansible/verify.yml
|
||||
@@ -1,35 +0,0 @@
|
||||
apiVersion: v1
|
||||
kind: Namespace
|
||||
metadata:
|
||||
name: gitea-actions
|
||||
labels:
|
||||
pod-security.kubernetes.io/enforce: privileged
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: ConfigMap
|
||||
metadata:
|
||||
name: kata-ephemeral-runner-config
|
||||
namespace: gitea-actions
|
||||
data:
|
||||
config.yaml: |
|
||||
log:
|
||||
level: info
|
||||
runner:
|
||||
file: /runner-state/.runner
|
||||
capacity: 1
|
||||
envs:
|
||||
DOCKER_HOST: tcp://host.docker.internal:2375
|
||||
timeout: 3h
|
||||
shutdown_timeout: 5m
|
||||
labels:
|
||||
- kata-poc:docker://docker.io/library/alpine:3.22
|
||||
cache:
|
||||
enabled: false
|
||||
container:
|
||||
network: ""
|
||||
options: --add-host=host.docker.internal:host-gateway
|
||||
docker_host: tcp://127.0.0.1:2375
|
||||
require_docker: true
|
||||
docker_timeout: 5m
|
||||
force_pull: true
|
||||
bind_workdir: false
|
||||
@@ -1,111 +0,0 @@
|
||||
apiVersion: v1
|
||||
kind: Pod
|
||||
metadata:
|
||||
generateName: kata-ephemeral-runner-
|
||||
namespace: gitea-actions
|
||||
labels:
|
||||
app.kubernetes.io/name: kata-ephemeral-runner
|
||||
spec:
|
||||
runtimeClassName: kata
|
||||
restartPolicy: Never
|
||||
terminationGracePeriodSeconds: 330
|
||||
automountServiceAccountToken: false
|
||||
initContainers:
|
||||
# Native sidecar: startupProbe gates the runner, and kubelet terminates this
|
||||
# container after the ephemeral runner exits so the Kata sandbox can end.
|
||||
- name: docker
|
||||
image: zot.ad.ddupan.top/panxiao81/dind-fuse:29.7.1
|
||||
imagePullPolicy: Never
|
||||
restartPolicy: Always
|
||||
env:
|
||||
- name: DOCKER_TLS_CERTDIR
|
||||
value: ""
|
||||
- name: DIND_LOOPBACK_SIZE
|
||||
value: 6G
|
||||
- name: DIND_LOOPBACK_FILE
|
||||
value: /loopback/docker.img
|
||||
args:
|
||||
- --host=tcp://0.0.0.0:2375
|
||||
- --tls=false
|
||||
- --mtu=1450
|
||||
- --default-network-opt=bridge=com.docker.network.driver.mtu=1450
|
||||
securityContext:
|
||||
privileged: true
|
||||
resources:
|
||||
requests:
|
||||
cpu: 250m
|
||||
memory: 512Mi
|
||||
ephemeral-storage: 1Gi
|
||||
limits:
|
||||
cpu: "4"
|
||||
memory: 6Gi
|
||||
ephemeral-storage: 8Gi
|
||||
volumeMounts:
|
||||
- name: docker-data
|
||||
mountPath: /var/lib/docker
|
||||
- name: loopback-data
|
||||
mountPath: /loopback
|
||||
startupProbe:
|
||||
tcpSocket:
|
||||
port: 2375
|
||||
periodSeconds: 2
|
||||
failureThreshold: 150
|
||||
readinessProbe:
|
||||
tcpSocket:
|
||||
port: 2375
|
||||
periodSeconds: 5
|
||||
containers:
|
||||
- name: runner
|
||||
image: docker.io/gitea/runner:2.3.0
|
||||
imagePullPolicy: Never
|
||||
env:
|
||||
- name: CONFIG_FILE
|
||||
value: /config/config.yaml
|
||||
- name: DOCKER_HOST
|
||||
value: tcp://127.0.0.1:2375
|
||||
- name: GITEA_INSTANCE_URL
|
||||
value: https://git.ddupan.top
|
||||
- name: GITEA_RUNNER_EPHEMERAL
|
||||
value: "1"
|
||||
- name: GITEA_RUNNER_NAME
|
||||
valueFrom:
|
||||
fieldRef:
|
||||
fieldPath: metadata.name
|
||||
- name: GITEA_RUNNER_LABELS
|
||||
value: kata-poc:docker://docker.io/library/alpine:3.22
|
||||
- name: GITEA_RUNNER_REGISTRATION_TOKEN_FILE
|
||||
value: /registration/token
|
||||
resources:
|
||||
requests:
|
||||
cpu: 100m
|
||||
memory: 128Mi
|
||||
limits:
|
||||
cpu: "1"
|
||||
memory: 1Gi
|
||||
volumeMounts:
|
||||
- name: config
|
||||
mountPath: /config
|
||||
readOnly: true
|
||||
- name: registration
|
||||
mountPath: /registration
|
||||
readOnly: true
|
||||
- name: runner-state
|
||||
mountPath: /runner-state
|
||||
volumes:
|
||||
- name: config
|
||||
configMap:
|
||||
name: kata-ephemeral-runner-config
|
||||
- name: registration
|
||||
secret:
|
||||
secretName: gitea-runner-registration
|
||||
- name: runner-state
|
||||
emptyDir:
|
||||
medium: Memory
|
||||
sizeLimit: 16Mi
|
||||
- name: docker-data
|
||||
emptyDir:
|
||||
sizeLimit: 8Gi
|
||||
- name: loopback-data
|
||||
emptyDir:
|
||||
medium: Memory
|
||||
sizeLimit: 6Gi
|
||||
@@ -1,24 +0,0 @@
|
||||
# This file is maintained automatically by "terraform init".
|
||||
# Manual edits may be lost in future updates.
|
||||
|
||||
provider "registry.terraform.io/bpg/proxmox" {
|
||||
version = "0.111.1"
|
||||
constraints = "0.111.1"
|
||||
hashes = [
|
||||
"h1:ML2D3UUZTM99yrll/EBXj7wBYMb8xmQgomqFNybEoxY=",
|
||||
"zh:18fb7c31a08dde6bffa1a4d4a211e604d6d17eec7092fd59331b3db3c6f3742c",
|
||||
"zh:1cd60761538289d4dd2a1086b3ae62a7b0bdd4b1a2f824e9a44e243413168dba",
|
||||
"zh:2eb76f6fc8299b6820ff678c8252332cc3366e226b5ae2e61748fd2449c1ed92",
|
||||
"zh:45e6f7ebd0bf48911d37060359a4f359b5743b3092e985295733990e406d0416",
|
||||
"zh:4aa8ba912eae37975d2e983394d173e595ca34fc76b5bf220b37d0e99d76e98c",
|
||||
"zh:58e0789923103a77d502a0a9fc3eb920625e8eb935ec2d4ac0d006aebd1d186c",
|
||||
"zh:6df8aa85fb8865915537e946c19b02538ad188018a629759c213c6f03730f642",
|
||||
"zh:6ed47bc00d0913a1d0880618fa1376115e9edab6b4a658c081061a7f0e4ca360",
|
||||
"zh:c5b10ff4f33df7e4c29e8f1127d49845b561b37b57517e844fb0954d7923d65e",
|
||||
"zh:d016510e14b738499f0db9d9b3aafe82fc6877fb4ab4e9f831fb68a8d70a1385",
|
||||
"zh:d941f394069bbf24351b363da1c64383f487067aaee0a84f9b96476d4912e212",
|
||||
"zh:ddf271dbc2632ae8ffa8de3972f243ee47d260cb2ac90aa784f2746d98e21a0f",
|
||||
"zh:ed0caa3501c42f611b7e9622c9b1df69fd85dc25a3cd88d3076381829688cd62",
|
||||
"zh:f26e0763dbe6a6b2195c94b44696f2110f7f55433dc142839be16b9697fa5597",
|
||||
]
|
||||
}
|
||||
@@ -1,95 +0,0 @@
|
||||
locals {
|
||||
ssh_public_key = trimspace(file(pathexpand(var.ssh_public_key_file)))
|
||||
}
|
||||
|
||||
data "proxmox_file" "ubuntu_noble" {
|
||||
content_type = "vztmpl"
|
||||
datastore_id = var.template_datastore
|
||||
node_name = var.proxmox_node
|
||||
file_name = var.template_file_name
|
||||
}
|
||||
|
||||
resource "proxmox_virtual_environment_container" "kata_lxc_lab" {
|
||||
node_name = var.proxmox_node
|
||||
vm_id = var.container_id
|
||||
|
||||
description = "Disposable privileged LXC for validating k3s and Kata with direct host KVM access."
|
||||
unprivileged = false
|
||||
started = true
|
||||
start_on_boot = false
|
||||
tags = ["disposable", "kata", "lxc", "terraform"]
|
||||
|
||||
cpu {
|
||||
cores = 4
|
||||
}
|
||||
|
||||
memory {
|
||||
dedicated = 8192
|
||||
swap = 0
|
||||
}
|
||||
|
||||
disk {
|
||||
datastore_id = var.root_datastore
|
||||
size = 16
|
||||
}
|
||||
|
||||
features {
|
||||
fuse = true
|
||||
keyctl = true
|
||||
mknod = true
|
||||
nesting = true
|
||||
}
|
||||
|
||||
device_passthrough {
|
||||
path = "/dev/kvm"
|
||||
mode = "0660"
|
||||
}
|
||||
|
||||
device_passthrough {
|
||||
path = "/dev/vhost-net"
|
||||
mode = "0660"
|
||||
}
|
||||
|
||||
device_passthrough {
|
||||
path = "/dev/vhost-vsock"
|
||||
mode = "0660"
|
||||
}
|
||||
|
||||
device_passthrough {
|
||||
path = "/dev/kmsg"
|
||||
mode = "0660"
|
||||
}
|
||||
|
||||
device_passthrough {
|
||||
path = "/dev/net/tun"
|
||||
mode = "0666"
|
||||
}
|
||||
|
||||
initialization {
|
||||
hostname = var.container_name
|
||||
|
||||
ip_config {
|
||||
ipv4 {
|
||||
address = "dhcp"
|
||||
}
|
||||
}
|
||||
|
||||
user_account {
|
||||
keys = [local.ssh_public_key]
|
||||
}
|
||||
}
|
||||
|
||||
network_interface {
|
||||
name = "eth0"
|
||||
bridge = "vmbr0"
|
||||
}
|
||||
|
||||
operating_system {
|
||||
template_file_id = data.proxmox_file.ubuntu_noble.id
|
||||
type = "ubuntu"
|
||||
}
|
||||
|
||||
wait_for_ip {
|
||||
ipv4 = true
|
||||
}
|
||||
}
|
||||
@@ -1,7 +0,0 @@
|
||||
output "container_id" {
|
||||
value = proxmox_virtual_environment_container.kata_lxc_lab.vm_id
|
||||
}
|
||||
|
||||
output "ipv4" {
|
||||
value = proxmox_virtual_environment_container.kata_lxc_lab.ipv4
|
||||
}
|
||||
@@ -1,15 +0,0 @@
|
||||
provider "proxmox" {
|
||||
endpoint = var.proxmox_endpoint
|
||||
api_token = var.proxmox_api_token
|
||||
|
||||
ssh {
|
||||
agent = false
|
||||
username = "root"
|
||||
private_key = file(pathexpand(var.proxmox_ssh_private_key_file))
|
||||
|
||||
node {
|
||||
name = var.proxmox_node
|
||||
address = var.proxmox_node_address
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,54 +0,0 @@
|
||||
variable "proxmox_endpoint" {
|
||||
type = string
|
||||
default = "https://pve1.ad.ddupan.top:8006"
|
||||
}
|
||||
|
||||
variable "proxmox_api_token" {
|
||||
type = string
|
||||
sensitive = true
|
||||
}
|
||||
|
||||
variable "proxmox_node" {
|
||||
type = string
|
||||
default = "pve2"
|
||||
}
|
||||
|
||||
variable "proxmox_node_address" {
|
||||
type = string
|
||||
default = "192.168.10.7"
|
||||
}
|
||||
|
||||
variable "proxmox_ssh_private_key_file" {
|
||||
type = string
|
||||
default = "~/.ssh/id_ed25519"
|
||||
}
|
||||
|
||||
variable "ssh_public_key_file" {
|
||||
type = string
|
||||
default = "~/.ssh/id_ed25519.pub"
|
||||
}
|
||||
|
||||
variable "container_id" {
|
||||
type = number
|
||||
default = 147
|
||||
}
|
||||
|
||||
variable "container_name" {
|
||||
type = string
|
||||
default = "kata-lxc-lab"
|
||||
}
|
||||
|
||||
variable "template_datastore" {
|
||||
type = string
|
||||
default = "laptop"
|
||||
}
|
||||
|
||||
variable "template_file_name" {
|
||||
type = string
|
||||
default = "ubuntu-24.04-standard_24.04-2_amd64.tar.zst"
|
||||
}
|
||||
|
||||
variable "root_datastore" {
|
||||
type = string
|
||||
default = "local-lvm"
|
||||
}
|
||||
@@ -1,10 +0,0 @@
|
||||
terraform {
|
||||
required_version = ">= 1.10"
|
||||
|
||||
required_providers {
|
||||
proxmox = {
|
||||
source = "bpg/proxmox"
|
||||
version = "0.111.1"
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,53 +0,0 @@
|
||||
# Gitea kind microVM runner
|
||||
|
||||
`kind-microvm` 是专门运行 kind / nested Kubernetes CI 的 runner label。每个 job
|
||||
独占一台 Cloud Hypervisor VM；guest 内直接运行 Docker 与 ephemeral Gitea Runner，
|
||||
不经过 Kata。VM 完成一个 job 后关机，临时 COW 磁盘随即删除。
|
||||
|
||||
集群内的 `controller.py` 消费 Gitea `workflow_job` webhook。仅当
|
||||
`action=queued` 且 `workflow_job.labels` 包含 `kind-microvm` 时，向 NATS JetStream
|
||||
的 `ci.runner.kind-microvm` subject 发布消息。`workflow_job.id` 写入
|
||||
`Nats-Msg-Id`，重复 webhook 不会生成第二个任务。
|
||||
|
||||
pve2 LXC 内的 `worker.py` 使用 durable pull consumer 领取消息。领取后启动一台
|
||||
microVM，运行期间每分钟发送 `InProgress`；VM/ephemeral runner 正常退出才 ACK，
|
||||
启动失败则 NAK 并延迟重试。当前 `RUNNER_CAPACITY=1` 只是 pve2 的资源配置；以后可
|
||||
提高单 worker capacity，或在其他宿主机增加共享同一 durable consumer 的 worker，
|
||||
而无需修改 webhook/controller。相同 runner 类型必须共享 durable；WorkQueue stream
|
||||
不允许多个 consumer 使用重叠的 subject filter。
|
||||
|
||||
worker 的凭据位于：
|
||||
|
||||
```text
|
||||
/etc/microvm-runner/registration-token
|
||||
/etc/microvm-runner/nats-password
|
||||
```
|
||||
|
||||
二者必须为 root-only 文件，不能写入镜像、cloud-init seed 或仓库。worker 为每次
|
||||
启动生成一次性 nonce；guest 只能从 TAP 网段请求一次 registration token，请求后
|
||||
立即从 worker 内存删除。runner 注册成功后删除 guest 内的 token 文件，再启动
|
||||
daemon。这样 job 无法从 seed disk 或 Docker inspect 取回可复用 registration token。
|
||||
|
||||
NATS stream 使用 `WorkQueuePolicy`，ACK 后立即删除消息；同时限制为 24 小时、
|
||||
10000 条和 256 MiB，异常积压也不会无限增长。NATS 平台配置见
|
||||
`../../platform/nats/`。
|
||||
|
||||
2026-09-16 PoC 已验证：Cloud Hypervisor v52.0、Ubuntu 24.04、4 vCPU、3 GiB RAM、
|
||||
10 GiB COW disk 中，Docker 29.1.3 成功创建 kind v0.27.0 / Kubernetes v1.32.2
|
||||
集群，并运行 `busybox:1.37` smoke Pod。冷启动约 20 秒。pve2 资源只允许一台：
|
||||
VMM cgroup（guest RAM 与 page cache）峰值约 6.2 GiB。
|
||||
|
||||
正式启用前还需要安装 launcher、制作不含凭据的基础镜像、部署 controller，并在
|
||||
Gitea 配置只发送 `workflow_job` 的 instance/organization webhook。workflow 使用：
|
||||
|
||||
```yaml
|
||||
runs-on: kind-microvm
|
||||
```
|
||||
|
||||
当前 pve2 配置：
|
||||
|
||||
```ini
|
||||
RUNNER_CAPACITY=1
|
||||
NATS_DURABLE=kind-microvm
|
||||
RUNNER_MAX_INFLIGHT=64
|
||||
```
|
||||
@@ -1,113 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Gitea workflow_job webhook to NATS JetStream producer."""
|
||||
|
||||
import hashlib
|
||||
import hmac
|
||||
import json
|
||||
import os
|
||||
import ssl
|
||||
from pathlib import Path
|
||||
|
||||
import nats
|
||||
from aiohttp import web
|
||||
from nats.js.api import DiscardPolicy, RetentionPolicy, StorageType, StreamConfig
|
||||
from nats.js.errors import NotFoundError
|
||||
|
||||
LABEL = os.environ.get("RUNNER_LABEL", "kind-microvm")
|
||||
SUBJECT = os.environ.get("NATS_SUBJECT", f"ci.runner.{LABEL}")
|
||||
STREAM = os.environ.get("NATS_STREAM", "CI_RUNNER")
|
||||
NATS_URL = os.environ.get("NATS_URL", "tls://nats.nats.svc.cluster.local:4222")
|
||||
NATS_USER = os.environ.get("NATS_USER", "ci-producer")
|
||||
NATS_PASSWORD_FILE = Path(os.environ.get("NATS_PASSWORD_FILE", "/run/secrets/nats/password"))
|
||||
NATS_CA_FILE = os.environ.get("NATS_CA_FILE", "/etc/ssl/certs/ca-certificates.crt")
|
||||
WEBHOOK_SECRET_FILE = Path(os.environ.get("WEBHOOK_SECRET_FILE", "/run/secrets/gitea/webhook-secret"))
|
||||
|
||||
|
||||
def accepts(payload: object) -> tuple[bool, str | None]:
|
||||
if not isinstance(payload, dict) or payload.get("action") != "queued":
|
||||
return False, None
|
||||
job = payload.get("workflow_job")
|
||||
if not isinstance(job, dict) or LABEL not in job.get("labels", []):
|
||||
return False, None
|
||||
job_id = job.get("id")
|
||||
if not isinstance(job_id, int):
|
||||
return False, None
|
||||
return True, str(job_id)
|
||||
|
||||
|
||||
def valid_signature(body: bytes, signature: str) -> bool:
|
||||
expected = hmac.new(WEBHOOK_SECRET_FILE.read_bytes().strip(), body, hashlib.sha256).hexdigest()
|
||||
return hmac.compare_digest(signature, expected)
|
||||
|
||||
|
||||
async def ensure_stream(js: object) -> None:
|
||||
config = StreamConfig(
|
||||
name=STREAM,
|
||||
subjects=["ci.runner.*"],
|
||||
retention=RetentionPolicy.WORK_QUEUE,
|
||||
storage=StorageType.FILE,
|
||||
discard=DiscardPolicy.OLD,
|
||||
max_age=24 * 60 * 60,
|
||||
max_msgs=10_000,
|
||||
max_bytes=256 * 1024 * 1024,
|
||||
duplicate_window=24 * 60 * 60,
|
||||
)
|
||||
try:
|
||||
await js.stream_info(STREAM)
|
||||
except NotFoundError:
|
||||
await js.add_stream(config=config)
|
||||
else:
|
||||
await js.update_stream(config=config)
|
||||
|
||||
|
||||
async def webhook(request: web.Request) -> web.Response:
|
||||
body = await request.read()
|
||||
if not valid_signature(body, request.headers.get("X-Gitea-Signature", "")):
|
||||
raise web.HTTPUnauthorized()
|
||||
try:
|
||||
payload = json.loads(body)
|
||||
except json.JSONDecodeError as error:
|
||||
raise web.HTTPBadRequest(text="invalid JSON\n") from error
|
||||
accepted, job_id = accepts(payload)
|
||||
if not accepted:
|
||||
return web.Response(status=204)
|
||||
await request.app["js"].publish(
|
||||
SUBJECT,
|
||||
body,
|
||||
headers={"Nats-Msg-Id": f"gitea-workflow-job-{job_id}"},
|
||||
)
|
||||
return web.Response(status=202, text="queued\n")
|
||||
|
||||
|
||||
async def health(request: web.Request) -> web.Response:
|
||||
return web.Response(text="ok\n" if request.app["nc"].is_connected else "disconnected\n",
|
||||
status=200 if request.app["nc"].is_connected else 503)
|
||||
|
||||
|
||||
async def nats_context(app: web.Application):
|
||||
tls = ssl.create_default_context(cafile=NATS_CA_FILE)
|
||||
nc = await nats.connect(
|
||||
NATS_URL,
|
||||
user=NATS_USER,
|
||||
password=NATS_PASSWORD_FILE.read_text().strip(),
|
||||
tls=tls,
|
||||
name="microvm-runner-controller",
|
||||
)
|
||||
app["nc"] = nc
|
||||
app["js"] = nc.jetstream()
|
||||
await ensure_stream(app["js"])
|
||||
yield
|
||||
await nc.drain()
|
||||
|
||||
|
||||
def create_app() -> web.Application:
|
||||
app = web.Application(client_max_size=1024 * 1024)
|
||||
app.cleanup_ctx.append(nats_context)
|
||||
app.router.add_post("/webhook", webhook)
|
||||
app.router.add_get("/healthz", health)
|
||||
return app
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
web.run_app(create_app(), host=os.environ.get("LISTEN", "0.0.0.0"),
|
||||
port=int(os.environ.get("PORT", "8787")))
|
||||
@@ -1,19 +0,0 @@
|
||||
[Unit]
|
||||
Description=Gitea kind microVM runner worker
|
||||
After=network-online.target
|
||||
Wants=network-online.target
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
EnvironmentFile=-/etc/microvm-runner/worker.env
|
||||
ExecStart=/opt/microvm-runner/venv/bin/python /opt/microvm-runner/worker.py
|
||||
Restart=on-failure
|
||||
RestartSec=5s
|
||||
SupplementaryGroups=kvm
|
||||
PrivateTmp=yes
|
||||
ProtectHome=yes
|
||||
ProtectSystem=strict
|
||||
ReadWritePaths=/var/lib/microvm-runner /run/microvm-runner
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
@@ -1,2 +0,0 @@
|
||||
aiohttp==3.12.15
|
||||
nats-py==2.11.0
|
||||
@@ -1,138 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Capacity-bounded JetStream consumer that launches one ephemeral VM per job."""
|
||||
|
||||
import asyncio
|
||||
import os
|
||||
import secrets
|
||||
import ssl
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
|
||||
import nats
|
||||
from aiohttp import web
|
||||
from nats.errors import TimeoutError
|
||||
from nats.js.api import AckPolicy, ConsumerConfig
|
||||
|
||||
CAPACITY = int(os.environ.get("RUNNER_CAPACITY", "1"))
|
||||
SUBJECT = os.environ.get("NATS_SUBJECT", "ci.runner.kind-microvm")
|
||||
STREAM = os.environ.get("NATS_STREAM", "CI_RUNNER")
|
||||
DURABLE = os.environ.get("NATS_DURABLE", "kind-microvm")
|
||||
MAX_INFLIGHT = int(os.environ.get("RUNNER_MAX_INFLIGHT", "64"))
|
||||
NATS_URL = os.environ.get("NATS_URL", "tls://nats.ad.ddupan.top:4222")
|
||||
NATS_USER = os.environ.get("NATS_USER", "ci-worker")
|
||||
NATS_PASSWORD_FILE = Path(os.environ.get("NATS_PASSWORD_FILE", "/etc/microvm-runner/nats-password"))
|
||||
NATS_CA_FILE = os.environ.get("NATS_CA_FILE", "/etc/ssl/certs/ca-certificates.crt")
|
||||
REGISTRATION_TOKEN_FILE = Path(os.environ.get("REGISTRATION_TOKEN_FILE", "/etc/microvm-runner/registration-token"))
|
||||
LAUNCHER = os.environ.get("LAUNCHER", "/usr/local/libexec/microvm-runner-launch")
|
||||
TOKEN_LISTEN = os.environ.get("TOKEN_LISTEN", "172.30.0.1")
|
||||
TOKEN_PORT = int(os.environ.get("TOKEN_PORT", "8787"))
|
||||
|
||||
tokens: dict[str, bytes] = {}
|
||||
token_lock = asyncio.Lock()
|
||||
|
||||
|
||||
async def token(request: web.Request) -> web.Response:
|
||||
nonce = request.match_info["nonce"]
|
||||
async with token_lock:
|
||||
value = tokens.pop(nonce, None)
|
||||
if value is None:
|
||||
raise web.HTTPNotFound()
|
||||
return web.Response(body=value, headers={"Cache-Control": "no-store"})
|
||||
|
||||
|
||||
async def heartbeat(message: object, stop: asyncio.Event) -> None:
|
||||
while True:
|
||||
try:
|
||||
await asyncio.wait_for(stop.wait(), timeout=60)
|
||||
return
|
||||
except asyncio.TimeoutError:
|
||||
await message.in_progress()
|
||||
|
||||
|
||||
async def run_one(message: object) -> None:
|
||||
instance_id = str(uuid.uuid4())
|
||||
nonce = secrets.token_urlsafe(32)
|
||||
async with token_lock:
|
||||
tokens[nonce] = REGISTRATION_TOKEN_FILE.read_bytes().strip()
|
||||
stop = asyncio.Event()
|
||||
pulse = asyncio.create_task(heartbeat(message, stop))
|
||||
try:
|
||||
process = await asyncio.create_subprocess_exec(LAUNCHER, instance_id, nonce)
|
||||
return_code = await process.wait()
|
||||
if return_code == 0:
|
||||
await message.ack()
|
||||
else:
|
||||
await message.nak(delay=30)
|
||||
except Exception:
|
||||
await message.nak(delay=30)
|
||||
raise
|
||||
finally:
|
||||
stop.set()
|
||||
await pulse
|
||||
async with token_lock:
|
||||
tokens.pop(nonce, None)
|
||||
|
||||
|
||||
async def consume() -> None:
|
||||
if CAPACITY < 1:
|
||||
raise ValueError("RUNNER_CAPACITY must be at least 1")
|
||||
tls = ssl.create_default_context(cafile=NATS_CA_FILE)
|
||||
nc = await nats.connect(
|
||||
NATS_URL,
|
||||
user=NATS_USER,
|
||||
password=NATS_PASSWORD_FILE.read_text().strip(),
|
||||
tls=tls,
|
||||
name=DURABLE,
|
||||
)
|
||||
js = nc.jetstream()
|
||||
subscription = await js.pull_subscribe(
|
||||
SUBJECT,
|
||||
durable=DURABLE,
|
||||
stream=STREAM,
|
||||
config=ConsumerConfig(
|
||||
durable_name=DURABLE,
|
||||
filter_subject=SUBJECT,
|
||||
ack_policy=AckPolicy.EXPLICIT,
|
||||
ack_wait=5 * 60,
|
||||
# This durable consumer is shared by every host of this runner type.
|
||||
# Local CAPACITY controls each host; this is only a global safety cap.
|
||||
max_ack_pending=MAX_INFLIGHT,
|
||||
max_deliver=5,
|
||||
),
|
||||
)
|
||||
active: set[asyncio.Task[None]] = set()
|
||||
try:
|
||||
while True:
|
||||
active = {task for task in active if not task.done()}
|
||||
free = CAPACITY - len(active)
|
||||
if free == 0:
|
||||
await asyncio.wait(active, return_when=asyncio.FIRST_COMPLETED)
|
||||
continue
|
||||
try:
|
||||
messages = await subscription.fetch(batch=free, timeout=5)
|
||||
except TimeoutError:
|
||||
continue
|
||||
for message in messages:
|
||||
task = asyncio.create_task(run_one(message))
|
||||
task.add_done_callback(lambda done: done.exception())
|
||||
active.add(task)
|
||||
finally:
|
||||
if active:
|
||||
await asyncio.gather(*active, return_exceptions=True)
|
||||
await nc.drain()
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
app = web.Application()
|
||||
app.router.add_get("/token/{nonce}", token)
|
||||
runner = web.AppRunner(app)
|
||||
await runner.setup()
|
||||
await web.TCPSite(runner, TOKEN_LISTEN, TOKEN_PORT).start()
|
||||
try:
|
||||
await consume()
|
||||
finally:
|
||||
await runner.cleanup()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
@@ -74,6 +74,6 @@ sudo journalctl -u homelab-data-collect.service --since '2 hours ago'
|
||||
|
||||
备份年龄不能证明恢复能力或 WAL 连续性；定期隔离恢复演练仍必要。证书指标读取磁盘文件，
|
||||
不证明运行进程已重载该证书。dev 检查不验证外部 DNS/TLS 路径、不执行写入。
|
||||
SQL 级指标见 [共享 PG](../shared-postgresql/README.md#sql-级指标2026-10-01-主机端上线)；异地备份及应用迁移不在本次范围。
|
||||
SQL 级 exporter、异地备份及应用迁移不在本次范围。
|
||||
|
||||
跨服务状态见 [共享 PG wiki](https://git.ddupan.top/panxiao81/homelab-wiki/src/branch/main/services/shared-postgresql.md)。
|
||||
|
||||
@@ -69,10 +69,7 @@ Ayatori 单独使用 `kv/infra/postgresql/ayatori/{prod,dev}`，仅有 username/
|
||||
6. `bootstrap-backup.yml` 验证 WAL 与首备成功后启用每日 timer；`proxy.yml` 要求唯一 primary 才发布入口。
|
||||
7. DNS 由 `infrastructure/dns/records.yml` 与 Samba `provision-dc.yml --tags dns` 管理。
|
||||
8. `controller.yml` 安装 PG 证书每日续签调度，预检通过才启用。
|
||||
9. `monitoring.yml` 激活 SQL 指标：先由 `site.yml` 写入模板，再执行本 play。它只在配置未生效时
|
||||
向 Patroni 发 HUP、对 pending 成员 `patronictl restart --pending`（primary 原位重启，不 failover，
|
||||
写入短暂中断）或重启 dev；随后在可写实例创建监控角色与扩展并安装 exporter。不需要 Bao。
|
||||
10. Ayatori 独立控制面就绪后按 [接入说明](ayatori/README.md) 注册；CNPG 迁移另行按应用验收。
|
||||
9. Ayatori 独立控制面就绪后按 [接入说明](ayatori/README.md) 注册；CNPG 迁移另行按应用验收。
|
||||
|
||||
普通 `site.yml` 不代表所有配置已在线激活；尚未实现通用参数变更的 rolling restart。
|
||||
专用 `renew-certificates.yml` 只处理证书，60 天有效期、剩余不足 14 天续签；逐主机检查服务，
|
||||
@@ -100,30 +97,10 @@ Ayatori 角色，随后停库清理；不覆盖生产 PGDATA。全站恢复及
|
||||
|
||||
现有 VictoriaMetrics/Alertmanager 已采集生产两节点 Patroni 原生指标，规则位于
|
||||
`platform/observability/metrics/{scrapes,rules}/shared-postgresql.yaml`。覆盖成员不可采集、无可见主库、
|
||||
多主、replica 未流复制及 PG 进程停止。没有部署另一套监控栈。dev SQL、备份年龄、证书和续签检查的主机采集已于 2026-09-27 部署，
|
||||
多主、replica 未流复制及 PG 进程停止。没有部署另一套监控栈；数据库 SQL 级 exporter 仍后置。dev SQL、备份年龄、证书和续签检查的主机采集已于 2026-09-27 部署，
|
||||
新增规则已随 [PR #166](https://git.ddupan.top/panxiao81/homelab-infra/pulls/166) 合并并由 Flux 接管；
|
||||
阈值、验证边界和排障见 [维护监控](../shared-data-monitoring/README.md)。
|
||||
|
||||
### SQL 级指标（2026-10-01 主机端上线）
|
||||
|
||||
每个实例在自身主机上运行 [prometheus-community/postgres_exporter](https://github.com/prometheus-community/postgres_exporter)
|
||||
（`pg_exporter_version`，安装包与官方 `sha256sums.txt` 同源校验）：prod 为 laptop/pve1 的 `:9187`，
|
||||
dev 为 laptop `:9188`，仅监听实例内网地址。exporter 以实例 OS 用户走本地 socket，由
|
||||
`local all homelab_monitor peer map=monitor` 与 ident 映射登录到 `homelab_monitor`
|
||||
（仅 `pg_monitor`、无密码、`CONNECTION LIMIT 3`）。这只约束 exporter 自己的查询，**不是隔离边界**：
|
||||
同一 OS 用户仍可经 peer 以 superuser 登录，exporter 被攻破等同实例属主；systemd
|
||||
`InaccessiblePaths` 只让它读不到数据目录、私钥与 pgpass。prod/dev 启用 `pg_stat_statements`。
|
||||
|
||||
规则 `platform/observability/metrics/rules/shared-postgresql-sql.yaml`（PrometheusRule）以上游
|
||||
`postgres_mixin` 为底改写：采集/SQL 不通、复制回放落后 > 1 MiB（与 failover 阈值一致）、主库复制槽
|
||||
保留 > 1 GiB、主库落在 pve1（HDD）超过 15 分钟、WAL 归档失败、XID/MXID 年龄、客户端连接 > 80%、空闲事务 > 10 分钟、事务 > 1 小时、
|
||||
死锁、频繁请求 checkpoint。目标带 `env` 标签，Alertmanager 把 `env="dev"` 送到独立 dev 频道。
|
||||
|
||||
2026-10-01 验证：隔离集成测试覆盖 peer+ident 登录（主/备/dev）、读取 `pg_stat_statements`、
|
||||
监控角色不可写；线上激活时两个 prod 成员按 pending 重启，时间线未变（未切换）；三个端点 `pg_up 1`、
|
||||
无采集失败，exporter 各约 8 MiB；重跑 `changed=0`；vmagent Pod 可达三个端点。规则 promtool 场景通过。
|
||||
`monitoring.yml --check` 不会显示 HUP、restart 与建角色（均为 command 任务），只能作参考。
|
||||
|
||||
2026-09-25 已验证：
|
||||
|
||||
- 真实 prod 主从、dev 初始化；Ayatori 管理账号 verify-full TLS/权限校验；生产稳定入口 SQL 写入。
|
||||
|
||||
@@ -19,7 +19,6 @@ pg_profiles:
|
||||
user: pgprod
|
||||
port: 5432
|
||||
api_port: 8008
|
||||
exporter_port: 9187
|
||||
dns: pg-prod.ad.ddupan.top
|
||||
memory_high: 768M
|
||||
memory_max: 1G
|
||||
@@ -29,7 +28,6 @@ pg_profiles:
|
||||
user: pgdev
|
||||
port: 5433
|
||||
dns: pg-dev.ad.ddupan.top
|
||||
exporter_port: 9188
|
||||
memory_high: 256M
|
||||
memory_max: 384M
|
||||
shared_buffers: 32MB
|
||||
@@ -40,8 +38,3 @@ pg_replication_addresses: ['192.168.10.127/32', '10.60.0.20/32']
|
||||
pg_repo_host: 10.60.0.21
|
||||
pg_repo_user: pgbackup
|
||||
pg_repo_path: /var/lib/homelab-pgbackrest
|
||||
# SQL 指标：exporter 以实例 OS 用户经本地 socket 连接，peer + ident 映射到只读监控角色，
|
||||
# 因此无需密码或 Bao；角色仅有 pg_monitor，不复用实例管理账号。
|
||||
pg_monitor_user: homelab_monitor
|
||||
# 升级只改版本；安装包与同版本官方 sha256sums.txt 一起下载校验，不写死摘要。
|
||||
pg_exporter_version: 0.20.1
|
||||
|
||||
@@ -1,81 +0,0 @@
|
||||
---
|
||||
# SQL 指标：激活监控所需的 HBA/ident 与 pg_stat_statements，创建只读监控角色并安装 exporter。
|
||||
# 不需要 Bao：模板先由 site.yml 写入；本 play 只做激活与验证。会重启实例（仅在预加载库未生效时），
|
||||
# 因此不放进 site.yml。重复执行应为 changed=0。
|
||||
- name: 激活实例监控配置
|
||||
hosts: pg_hosts
|
||||
become: true
|
||||
gather_facts: false
|
||||
serial: 1
|
||||
any_errors_fatal: true
|
||||
tasks:
|
||||
- name: 逐实例激活
|
||||
ansible.builtin.include_tasks: tasks/monitoring-activate.yml
|
||||
loop: "{{ pg_instances }}"
|
||||
loop_control:
|
||||
loop_var: pg_instance
|
||||
|
||||
- name: 重启 pending 的生产成员以加载 pg_stat_statements
|
||||
hosts: pg-laptop
|
||||
become: true
|
||||
become_user: "{{ pg_profiles.prod.user }}"
|
||||
gather_facts: false
|
||||
vars:
|
||||
pg_patronictl: [/opt/homelab-patroni/bin/patronictl, -c, "{{ pg_config_root }}/prod/patroni.yml"]
|
||||
tasks:
|
||||
# JSON 输出会省略空值字段：没有 pending 的成员不带 "Pending restart" 键。
|
||||
- name: 读取集群成员状态
|
||||
ansible.builtin.command:
|
||||
argv: "{{ pg_patronictl + ['list', '-f', 'json'] }}"
|
||||
register: pg_members
|
||||
changed_when: false
|
||||
check_mode: false
|
||||
# Patroni 在原位重启 primary，不触发 failover；写入会短暂中断，客户端需重试。
|
||||
- name: 仅重启 pending restart 的成员
|
||||
ansible.builtin.command:
|
||||
argv: "{{ pg_patronictl + ['restart', 'pg-prod', '--pending', '--force'] }}"
|
||||
when: pg_members.stdout | from_json | selectattr('Pending restart', 'defined') | selectattr('Pending restart', 'equalto', '*') | list | length > 0
|
||||
changed_when: true
|
||||
- name: 等待两个成员恢复且无 pending
|
||||
ansible.builtin.command:
|
||||
argv: "{{ pg_patronictl + ['list', '-f', 'json'] }}"
|
||||
register: pg_members_after
|
||||
changed_when: false
|
||||
when: not ansible_check_mode
|
||||
retries: 30
|
||||
delay: 4
|
||||
until: >-
|
||||
(pg_members_after.stdout | from_json | length) == 2
|
||||
and (pg_members_after.stdout | from_json | selectattr('Pending restart', 'defined') | selectattr('Pending restart', 'equalto', '*') | list | length) == 0
|
||||
and (pg_members_after.stdout | from_json | selectattr('State', 'in', ['running', 'streaming']) | list | length) == 2
|
||||
|
||||
- name: 创建监控角色并安装 exporter
|
||||
hosts: pg_hosts
|
||||
become: true
|
||||
gather_facts: false
|
||||
tasks:
|
||||
- name: 下载 postgres_exporter 并校验同版本官方摘要
|
||||
ansible.builtin.get_url:
|
||||
url: "{{ pg_exporter_base }}/postgres_exporter-{{ pg_exporter_version }}.linux-amd64.tar.gz"
|
||||
dest: /opt/homelab-postgres-exporter.tar.gz
|
||||
checksum: "sha256:{{ pg_exporter_base }}/sha256sums.txt"
|
||||
mode: '0644'
|
||||
vars:
|
||||
pg_exporter_base: https://github.com/prometheus-community/postgres_exporter/releases/download/v{{ pg_exporter_version }}
|
||||
register: pg_exporter_download
|
||||
retries: 5
|
||||
delay: 10
|
||||
until: pg_exporter_download is succeeded
|
||||
- name: 解压 exporter（保留上游许可证）
|
||||
ansible.builtin.unarchive:
|
||||
src: /opt/homelab-postgres-exporter.tar.gz
|
||||
dest: /opt
|
||||
remote_src: true
|
||||
creates: /opt/postgres_exporter-{{ pg_exporter_version }}.linux-amd64/postgres_exporter
|
||||
# check 模式不下载，首次时没有可解压的文件。
|
||||
when: not (ansible_check_mode and pg_exporter_download is changed)
|
||||
- name: 逐实例配置
|
||||
ansible.builtin.include_tasks: tasks/monitoring-exporter.yml
|
||||
loop: "{{ pg_instances }}"
|
||||
loop_control:
|
||||
loop_var: pg_instance
|
||||
@@ -71,7 +71,7 @@
|
||||
owner: root
|
||||
group: "{{ pg_profile.user }}"
|
||||
mode: '0640'
|
||||
loop: "{{ ['patroni.yml', 'pgbackrest.conf'] if pg_instance == 'prod' else ['postgresql.conf', 'pg_hba.conf', 'pg_ident.conf'] }}"
|
||||
loop: "{{ ['patroni.yml', 'pgbackrest.conf'] if pg_instance == 'prod' else ['postgresql.conf', 'pg_hba.conf'] }}"
|
||||
no_log: true
|
||||
- name: 安装实例专用服务（不自动启动或重启）
|
||||
ansible.builtin.template:
|
||||
|
||||
@@ -66,8 +66,6 @@ postgresql:
|
||||
unix_socket_directories: {{ pg_socket_dir }}
|
||||
unix_socket_permissions: '0700'
|
||||
password_encryption: scram-sha-256
|
||||
# postmaster 级参数：变更后需由 monitoring.yml 执行 Patroni pending restart。
|
||||
shared_preload_libraries: pg_stat_statements
|
||||
shared_buffers: {{ pg_profile.shared_buffers }}
|
||||
work_mem: 4MB
|
||||
maintenance_work_mem: 64MB
|
||||
@@ -78,7 +76,6 @@ postgresql:
|
||||
log_statement: none
|
||||
log_min_error_statement: panic
|
||||
pg_hba:
|
||||
- local all {{ pg_monitor_user }} peer map=monitor
|
||||
- local all {{ pg_profile.user }} peer
|
||||
- local replication {{ pg_profile.user }} peer
|
||||
- local replication replicator scram-sha-256
|
||||
@@ -91,9 +88,6 @@ postgresql:
|
||||
{% endfor %}
|
||||
- host all all 0.0.0.0/0 reject
|
||||
- host all all ::/0 reject
|
||||
# 只有实例 OS 用户（exporter 运行身份）可映射为监控角色。
|
||||
pg_ident:
|
||||
- monitor {{ pg_profile.user }} {{ pg_monitor_user }}
|
||||
remove_data_directory_on_rewind_failure: false
|
||||
remove_data_directory_on_diverged_timelines: false
|
||||
watchdog:
|
||||
|
||||
@@ -1,4 +1,3 @@
|
||||
local all {{ pg_monitor_user }} peer map=monitor
|
||||
local all {{ pg_profile.user }} peer
|
||||
{% for cidr in pg_client_cidrs %}
|
||||
hostssl all all {{ cidr }} scram-sha-256
|
||||
|
||||
@@ -1,2 +0,0 @@
|
||||
# 只有实例 OS 用户（exporter 运行身份）可映射为监控角色。
|
||||
monitor {{ pg_profile.user }} {{ pg_monitor_user }}
|
||||
-3
@@ -3,7 +3,6 @@ port = {{ pg_profile.port }}
|
||||
unix_socket_directories = '{{ pg_socket_dir }}'
|
||||
unix_socket_permissions = 0700
|
||||
hba_file = '{{ pg_config_dir }}/pg_hba.conf'
|
||||
ident_file = '{{ pg_config_dir }}/pg_ident.conf'
|
||||
password_encryption = 'scram-sha-256'
|
||||
shared_buffers = '{{ pg_profile.shared_buffers }}'
|
||||
max_connections = {{ pg_profile.max_connections }}
|
||||
@@ -17,5 +16,3 @@ ssl_ca_file = '{{ pg_config_dir }}/ca.crt'
|
||||
archive_mode = off
|
||||
log_statement = none
|
||||
log_min_error_statement = panic
|
||||
# postmaster 级参数：变更后需由 monitoring.yml 重启 dev 实例。
|
||||
shared_preload_libraries = 'pg_stat_statements'
|
||||
|
||||
@@ -1,73 +0,0 @@
|
||||
---
|
||||
- name: 设置现有实例路径
|
||||
ansible.builtin.set_fact:
|
||||
pg_profile: "{{ pg_profiles[pg_instance] }}"
|
||||
pg_config_dir: "{{ pg_config_root }}/{{ pg_instance }}"
|
||||
pg_socket_dir: "/run/homelab-postgresql-{{ pg_instance }}"
|
||||
- name: 激活前必须存在运行中的受管服务
|
||||
ansible.builtin.command:
|
||||
argv: [systemctl, is-active, "homelab-postgresql-{{ pg_instance }}.service"]
|
||||
changed_when: false
|
||||
check_mode: false
|
||||
# 生产：Patroni 只在自身 reload 时把本地配置写成数据目录里的 postgresql.conf/pg_ident.conf，
|
||||
# 所以文件视图能说明 Patroni 是否已接收新配置；预加载库是否已生效另看 current_setting。
|
||||
# 重启前 pg_file_settings 对该行报 "setting could not be applied"，属预期，不能按 error 过滤。
|
||||
# dev：ident_file 是 postmaster 级参数，视图读取的是“当前生效”的那个文件，
|
||||
# 所以启动时仍指向旧路径就会读不到映射；两列都为真才算已加载。
|
||||
- name: 读取监控配置的激活状态
|
||||
ansible.builtin.command:
|
||||
argv:
|
||||
- "{{ pg_bin_dir }}/psql"
|
||||
- -X
|
||||
- -At
|
||||
- -h
|
||||
- "{{ pg_socket_dir }}"
|
||||
- -p
|
||||
- "{{ pg_profile.port | string }}"
|
||||
- -d
|
||||
- postgres
|
||||
- -c
|
||||
- >-
|
||||
SELECT
|
||||
EXISTS (SELECT 1 FROM pg_ident_file_mappings WHERE map_name = 'monitor' AND error IS NULL)
|
||||
AND EXISTS (SELECT 1 FROM pg_file_settings WHERE name = 'shared_preload_libraries'
|
||||
AND setting ~ 'pg_stat_statements'),
|
||||
current_setting('shared_preload_libraries') ~ 'pg_stat_statements'
|
||||
become: true
|
||||
become_user: "{{ pg_profile.user }}"
|
||||
changed_when: false
|
||||
check_mode: false
|
||||
register: pg_monitor_state
|
||||
- name: 让 Patroni 接收新的本地配置（reload，不重启）
|
||||
ansible.builtin.command:
|
||||
argv: [systemctl, kill, --kill-whom=main, --signal=HUP, homelab-postgresql-prod.service]
|
||||
when: pg_instance == 'prod' and pg_monitor_state.stdout.split('|')[0] != 't'
|
||||
changed_when: true
|
||||
- name: 等待 Patroni 写出新配置
|
||||
ansible.builtin.command:
|
||||
argv: "{{ pg_monitor_state.cmd }}"
|
||||
become: true
|
||||
become_user: "{{ pg_profile.user }}"
|
||||
when: pg_instance == 'prod' and pg_monitor_state.stdout.split('|')[0] != 't' and not ansible_check_mode
|
||||
changed_when: false
|
||||
register: pg_monitor_reloaded
|
||||
retries: 10
|
||||
delay: 3
|
||||
until: pg_monitor_reloaded.stdout.split('|')[0] == 't'
|
||||
# dev 没有 Patroni；预加载库与 ident_file 都是 postmaster 级参数，只能重启。
|
||||
- name: 重启 dev 以加载监控配置
|
||||
ansible.builtin.systemd_service:
|
||||
name: homelab-postgresql-dev.service
|
||||
state: restarted
|
||||
when: pg_instance == 'dev' and pg_monitor_state.stdout != 't|t'
|
||||
- name: 检查 dev 重启后监控配置已生效
|
||||
ansible.builtin.command:
|
||||
argv: "{{ pg_monitor_state.cmd }}"
|
||||
become: true
|
||||
become_user: "{{ pg_profile.user }}"
|
||||
when: pg_instance == 'dev' and not ansible_check_mode
|
||||
changed_when: false
|
||||
register: pg_monitor_dev
|
||||
retries: 10
|
||||
delay: 3
|
||||
until: pg_monitor_dev.rc == 0 and pg_monitor_dev.stdout == 't|t'
|
||||
@@ -1,65 +0,0 @@
|
||||
---
|
||||
- name: 设置现有实例路径
|
||||
ansible.builtin.set_fact:
|
||||
pg_profile: "{{ pg_profiles[pg_instance] }}"
|
||||
pg_config_dir: "{{ pg_config_root }}/{{ pg_instance }}"
|
||||
pg_parent_dir: "{{ pg_data_root }}/{{ pg_instance }}"
|
||||
pg_socket_dir: "/run/homelab-postgresql-{{ pg_instance }}"
|
||||
pg_psql: ["{{ pg_bin_dir }}/psql", -X, -At, -v, ON_ERROR_STOP=1, -h, "/run/homelab-postgresql-{{ pg_instance }}",
|
||||
-p, "{{ pg_profiles[pg_instance].port | string }}", -d, postgres]
|
||||
# 角色与扩展只在可写实例上创建，standby 经复制获得。
|
||||
- name: 检查监控角色与扩展是否符合声明
|
||||
ansible.builtin.command:
|
||||
argv: "{{ pg_psql + ['-c', pg_monitor_check] }}"
|
||||
vars:
|
||||
pg_monitor_check: >-
|
||||
SELECT pg_is_in_recovery(),
|
||||
COALESCE((SELECT rolcanlogin AND NOT rolsuper AND NOT rolcreaterole AND NOT rolcreatedb
|
||||
AND NOT rolreplication AND NOT rolbypassrls AND rolconnlimit = 3 AND rolpassword IS NULL
|
||||
AND pg_has_role(oid, 'pg_monitor', 'MEMBER')
|
||||
FROM pg_authid WHERE rolname = '{{ pg_monitor_user }}'), false)
|
||||
AND EXISTS (SELECT 1 FROM pg_extension WHERE extname = 'pg_stat_statements')
|
||||
become: true
|
||||
become_user: "{{ pg_profile.user }}"
|
||||
changed_when: false
|
||||
check_mode: false
|
||||
register: pg_monitor_role
|
||||
- name: 创建或修正监控角色（无密码，仅能经本地 ident 映射登录）
|
||||
ansible.builtin.command:
|
||||
argv: "{{ pg_psql }}"
|
||||
stdin: |
|
||||
SELECT 'CREATE ROLE {{ pg_monitor_user }}'
|
||||
WHERE NOT EXISTS (SELECT 1 FROM pg_roles WHERE rolname = '{{ pg_monitor_user }}') \gexec
|
||||
ALTER ROLE {{ pg_monitor_user }} LOGIN NOSUPERUSER NOCREATEDB NOCREATEROLE NOREPLICATION NOBYPASSRLS
|
||||
CONNECTION LIMIT 3 PASSWORD NULL;
|
||||
GRANT pg_monitor TO {{ pg_monitor_user }};
|
||||
CREATE EXTENSION IF NOT EXISTS pg_stat_statements;
|
||||
become: true
|
||||
become_user: "{{ pg_profile.user }}"
|
||||
when: pg_monitor_role.stdout == 'f|f'
|
||||
changed_when: true
|
||||
- name: 安装实例 exporter 服务
|
||||
ansible.builtin.template:
|
||||
src: postgres-exporter.service.j2
|
||||
dest: "/etc/systemd/system/homelab-postgresql-{{ pg_instance }}-exporter.service"
|
||||
mode: '0644'
|
||||
register: pg_exporter_unit
|
||||
- name: 启用 exporter
|
||||
ansible.builtin.systemd_service:
|
||||
name: "homelab-postgresql-{{ pg_instance }}-exporter.service"
|
||||
daemon_reload: "{{ pg_exporter_unit is changed }}"
|
||||
enabled: true
|
||||
state: "{{ 'restarted' if pg_exporter_unit is changed else 'started' }}"
|
||||
# check 模式下 unit 文件尚未写入，systemd 找不到服务。
|
||||
when: not (ansible_check_mode and pg_exporter_unit is changed)
|
||||
# 同时证明 HBA/ident 已加载、角色可登录、exporter 能查询。
|
||||
- name: 检查 exporter 已连上数据库
|
||||
ansible.builtin.uri:
|
||||
url: "http://{{ pg_address }}:{{ pg_profile.exporter_port }}/metrics"
|
||||
return_content: true
|
||||
register: pg_exporter_metrics
|
||||
changed_when: false
|
||||
when: not ansible_check_mode
|
||||
retries: 10
|
||||
delay: 3
|
||||
until: pg_exporter_metrics.status == 200 and pg_exporter_metrics.content is search('(?m)^pg_up 1$')
|
||||
@@ -1,27 +0,0 @@
|
||||
[Unit]
|
||||
Description=PostgreSQL {{ pg_instance }} SQL metrics
|
||||
After=homelab-postgresql-{{ pg_instance }}.service
|
||||
[Service]
|
||||
# 本地 socket 上 TLS 无意义，sslmode=disable 只是避免驱动默认尝试 TLS。
|
||||
# 以实例 OS 用户运行才能进入 0700 的 socket 目录；peer + ident 把它映射成只读监控角色。
|
||||
# 监控角色只约束 exporter 自己发出的查询；这不是隔离边界——同一 OS 用户仍可经 peer 以
|
||||
# superuser 登录，exporter 被攻破即等同实例属主。InaccessiblePaths 只是不让它直接读写
|
||||
# 数据目录、私钥与 pgpass。暴露面因此仅限内网监听地址。
|
||||
User={{ pg_profile.user }}
|
||||
Group={{ pg_profile.user }}
|
||||
Environment="DATA_SOURCE_NAME=host={{ pg_socket_dir }} port={{ pg_profile.port }} user={{ pg_monitor_user }} dbname=postgres sslmode=disable application_name=postgres_exporter"
|
||||
ExecStart=/opt/postgres_exporter-{{ pg_exporter_version }}.linux-amd64/postgres_exporter \
|
||||
--web.listen-address={{ pg_address }}:{{ pg_profile.exporter_port }} \
|
||||
--collector.database_wraparound --collector.long_running_transactions \
|
||||
--collector.stat_checkpointer --collector.stat_wal_receiver --collector.stat_statements
|
||||
Restart=on-failure
|
||||
RestartSec=5
|
||||
InaccessiblePaths={{ pg_parent_dir }} {{ pg_config_dir }}
|
||||
NoNewPrivileges=true
|
||||
ProtectSystem=strict
|
||||
ProtectHome=true
|
||||
PrivateTmp=true
|
||||
PrivateDevices=true
|
||||
MemoryMax=64M
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
@@ -88,9 +88,9 @@ def main():
|
||||
pg_etcd_endpoints=[f'{h["etcd_address"]}:23379' for h in hosts.values()],
|
||||
pg_config_dir='/node/config',pg_parent_dir='/node',pg_data_dir='/node/data',pg_socket_dir='/node/socket',
|
||||
pg_bin_dir='/usr/lib/postgresql/18/bin',pg_replication_addresses=['127.0.0.0/8'],pg_client_cidrs=['127.0.0.0/8'],
|
||||
pg_repo_remote=False,pg_repo_path='/repo',pg_monitor_user='homelab_monitor')
|
||||
pg_repo_remote=False,pg_repo_path='/repo')
|
||||
values[name]=v
|
||||
for template in (['postgresql.conf','pg_hba.conf','pg_ident.conf'] if name=='dev' else ['patroni.yml','pgbackrest.conf']):
|
||||
for template in (['postgresql.conf','pg_hba.conf'] if name=='dev' else ['patroni.yml','pgbackrest.conf']):
|
||||
(cfg/template).write_text(env.get_template(template+'.j2').render(**v))
|
||||
for folder in ['socket','backup-lock','backup-spool']:(d/folder).mkdir()
|
||||
(pg/'repo').mkdir();(pg/'restore').mkdir();(pg/'pitr').mkdir()
|
||||
@@ -131,20 +131,6 @@ def main():
|
||||
initial_backup=json.loads(backrest('p1','--output=json','info').stdout)[0]['backup'][-1]['label']
|
||||
recovery_target=sql('p1','SELECT clock_timestamp()::text')
|
||||
print('PASS: separate dev, TLS verify-full, non-superuser Ayatori native privileges, initial backup',flush=True)
|
||||
# 与 monitoring.yml 相同的声明：无密码，只能由实例 OS 用户经 peer + ident 映射登录。
|
||||
monitor_role="CREATE ROLE homelab_monitor LOGIN CONNECTION LIMIT 3 PASSWORD NULL; GRANT pg_monitor TO homelab_monitor; CREATE EXTENSION pg_stat_statements;"
|
||||
sql('p1',monitor_role);sql('dev',monitor_role)
|
||||
def monitor(name,query):
|
||||
return run(['docker','exec','-i',prefix+'-'+name,'psql','-X','-v','ON_ERROR_STOP=1','-At','-h','/node/socket','-p','25432','-U','homelab_monitor','postgres'],input=query).stdout.strip()
|
||||
for name in ['p1','p2','dev']:
|
||||
wait_for(lambda: monitor(name,"SELECT current_setting('shared_preload_libraries')||(SELECT count(*)>=0 FROM pg_stat_statements)")=='pg_stat_statementstrue')
|
||||
for name in ['p1','dev']:
|
||||
try:
|
||||
monitor(name,'CREATE TABLE public.monitor_write(id int)')
|
||||
raise AssertionError(f'monitor role could write on {name}')
|
||||
except subprocess.CalledProcessError:
|
||||
pass
|
||||
print('PASS: monitor role logs in via peer+ident on primary/standby/dev, reads pg_stat_statements, cannot write',flush=True)
|
||||
proxycfg=w/'proxy.cfg'
|
||||
# 文件父目录仍由调用者拥有，新增代理配置不触碰 PG 文件。
|
||||
proxycfg.write_text(env.get_template('haproxy.cfg.j2').render(pg_proxy_bind='127.0.2.10:26432',pg_proxy_members=[dict(name=n,address=values[n]['pg_address'],port=25432,api_port=28008) for n in ['p1','p2']]))
|
||||
|
||||
@@ -1,26 +0,0 @@
|
||||
# CI pushes only :latest from main; follow the digest behind it. Old digests
|
||||
# lose their tag and are collected by zot GC, so the deployment must keep
|
||||
# tracking the current one.
|
||||
apiVersion: image.toolkit.fluxcd.io/v1
|
||||
kind: ImageRepository
|
||||
metadata:
|
||||
name: backstage
|
||||
namespace: flux-system
|
||||
spec:
|
||||
image: zot.ad.ddupan.top/panxiao81/backstage
|
||||
interval: 1m
|
||||
---
|
||||
apiVersion: image.toolkit.fluxcd.io/v1
|
||||
kind: ImagePolicy
|
||||
metadata:
|
||||
name: backstage
|
||||
namespace: flux-system
|
||||
spec:
|
||||
imageRepositoryRef:
|
||||
name: backstage
|
||||
filterTags:
|
||||
pattern: '^latest$'
|
||||
policy:
|
||||
alphabetical: {}
|
||||
digestReflectionPolicy: Always
|
||||
interval: 1m
|
||||
@@ -1,19 +0,0 @@
|
||||
# Gitea token of the private `flux-bot` user: collaborator with write on
|
||||
# homelab-infra only, token scoped to write:repository. Stored in OpenBao as
|
||||
# username/password because that is the basic-auth Secret shape Flux reads.
|
||||
apiVersion: external-secrets.io/v1
|
||||
kind: ExternalSecret
|
||||
metadata:
|
||||
name: homelab-infra-write
|
||||
namespace: flux-system
|
||||
spec:
|
||||
refreshInterval: 1h
|
||||
secretStoreRef:
|
||||
kind: ClusterSecretStore
|
||||
name: openbao
|
||||
target:
|
||||
creationPolicy: Owner
|
||||
name: homelab-infra-write
|
||||
dataFrom:
|
||||
- extract:
|
||||
key: k8s/flux-image-automation
|
||||
@@ -1,15 +0,0 @@
|
||||
# Write-capable source used only by ImageUpdateAutomation. The flux-system
|
||||
# GitRepository stays anonymous and read-only.
|
||||
apiVersion: source.toolkit.fluxcd.io/v1
|
||||
kind: GitRepository
|
||||
metadata:
|
||||
name: homelab-infra-write
|
||||
namespace: flux-system
|
||||
spec:
|
||||
interval: 10m
|
||||
ref:
|
||||
branch: main
|
||||
secretRef:
|
||||
name: homelab-infra-write
|
||||
timeout: 60s
|
||||
url: http://gitea-http.gitea.svc.cluster.local:3000/panxiao81/homelab-infra.git
|
||||
@@ -1,28 +0,0 @@
|
||||
# Commits digest changes straight to main (no PR): dev images are expected to
|
||||
# deploy as soon as CI pushes them. Scoped to apps/backstage so a stray setter
|
||||
# elsewhere cannot be rewritten.
|
||||
apiVersion: image.toolkit.fluxcd.io/v1
|
||||
kind: ImageUpdateAutomation
|
||||
metadata:
|
||||
name: homelab-infra
|
||||
namespace: flux-system
|
||||
spec:
|
||||
interval: 1m
|
||||
sourceRef:
|
||||
kind: GitRepository
|
||||
name: homelab-infra-write
|
||||
git:
|
||||
checkout:
|
||||
ref:
|
||||
branch: main
|
||||
commit:
|
||||
author:
|
||||
name: flux-bot
|
||||
email: [email protected]
|
||||
messageTemplate: |
|
||||
chore(image): update {{ range .Changed.Changes }}{{ .OldValue }} -> {{ .NewValue }} {{ end }}
|
||||
push:
|
||||
branch: main
|
||||
update:
|
||||
path: ./apps/backstage
|
||||
strategy: Setters
|
||||
@@ -1,7 +0,0 @@
|
||||
apiVersion: kustomize.config.k8s.io/v1beta1
|
||||
kind: Kustomization
|
||||
resources:
|
||||
- external-secret.yaml
|
||||
- gitrepository.yaml
|
||||
- imageupdateautomation.yaml
|
||||
- backstage.yaml
|
||||
@@ -1,9 +0,0 @@
|
||||
ARG DOCKER_VERSION=29.7.1
|
||||
FROM docker:${DOCKER_VERSION}-dind
|
||||
|
||||
RUN apk add --no-cache fuse-overlayfs
|
||||
|
||||
COPY --chmod=0755 entrypoint.sh /usr/local/bin/dind-fuse-entrypoint
|
||||
|
||||
ENTRYPOINT ["/usr/local/bin/dind-fuse-entrypoint"]
|
||||
CMD []
|
||||
@@ -1,42 +0,0 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
|
||||
# /dev belongs to the Kata guest. A privileged dockerd container can create
|
||||
# this standard FUSE node without exposing the PVE host's /dev/fuse to the LXC.
|
||||
if [ ! -e /dev/fuse ]; then
|
||||
mknod /dev/fuse c 10 229
|
||||
fi
|
||||
chmod 0666 /dev/fuse
|
||||
|
||||
# kind's nested kubelet opens /dev/kmsg. This node belongs to the Kata guest;
|
||||
# exposing it to a kind node does not expose the LXC or PVE host kernel log.
|
||||
if [ ! -e /dev/kmsg ]; then
|
||||
mknod /dev/kmsg c 1 11
|
||||
fi
|
||||
chmod 0600 /dev/kmsg
|
||||
|
||||
storage_driver=fuse-overlayfs
|
||||
|
||||
# kind's kubelet/cAdvisor cannot map a virtiofs-backed fuse-overlayfs root to a
|
||||
# block device. When requested, create an ext4 filesystem on a guest-local loop
|
||||
# device. The backing file, loop device and filesystem all die with the Kata VM.
|
||||
if [ -n "${DIND_LOOPBACK_SIZE:-}" ]; then
|
||||
[ -e /dev/loop-control ] || mknod /dev/loop-control c 10 237
|
||||
i=0
|
||||
while [ "$i" -lt 8 ]; do
|
||||
[ -e "/dev/loop$i" ] || mknod "/dev/loop$i" b 7 "$i"
|
||||
i=$((i + 1))
|
||||
done
|
||||
|
||||
backing_file="${DIND_LOOPBACK_FILE:-/var/lib/docker-loopback.img}"
|
||||
truncate -s "$DIND_LOOPBACK_SIZE" "$backing_file"
|
||||
# Alpine's BusyBox losetup supports -f, but not util-linux's
|
||||
# --find/--show long options.
|
||||
loop_device="$(losetup -f)"
|
||||
losetup "$loop_device" "$backing_file"
|
||||
mkfs.ext4 -q -F -m 0 "$loop_device"
|
||||
mount "$loop_device" /var/lib/docker
|
||||
storage_driver=overlay2
|
||||
fi
|
||||
|
||||
exec dockerd-entrypoint.sh --storage-driver="$storage_driver" "$@"
|
||||
@@ -25,8 +25,6 @@ resources:
|
||||
- scrapes/shared-etcd.yaml
|
||||
- rules/shared-postgresql.yaml
|
||||
- scrapes/shared-postgresql.yaml
|
||||
- rules/shared-postgresql-sql.yaml
|
||||
- scrapes/shared-postgresql-sql.yaml
|
||||
- scrapes/platform-controllers.yaml
|
||||
- rules/platform-controllers.yaml
|
||||
- scrapes/cnpg.yaml
|
||||
|
||||
@@ -58,7 +58,6 @@ spec:
|
||||
- alert: SharedPostgresDevUnavailable
|
||||
expr: homelab_data_dev_sql_ready{job="shared-data-maintenance"} == 0
|
||||
for: 3m
|
||||
# env 供 Alertmanager 把 dev 分流到独立 channel。
|
||||
labels: {severity: warning, service: shared-postgresql, env: dev}
|
||||
labels: {severity: warning, service: shared-postgresql}
|
||||
annotations:
|
||||
summary: 共享 PG 开发实例本地 SQL 不可用或处于只读状态
|
||||
|
||||
@@ -1,127 +0,0 @@
|
||||
# 以 postgres_exporter 上游 postgres_mixin 为底按本环境改写：实例停止、多主、角色变化已由
|
||||
# Patroni 规则（shared-postgresql.yaml）覆盖；QPS/缓存命中率/回滚数不是故障信号，未采用；
|
||||
# 上游 PostgresHasReplicationSlotUsed 会被 Patroni 在 standby 保留的非活跃槽持续触发，未采用。
|
||||
# 每条告警都带抓取目标的 env 标签，Alertmanager 按它把 dev 分流到独立 channel。
|
||||
apiVersion: monitoring.coreos.com/v1
|
||||
kind: PrometheusRule
|
||||
metadata:
|
||||
name: shared-postgresql-sql
|
||||
namespace: monitoring
|
||||
spec:
|
||||
groups:
|
||||
- name: shared-postgresql-sql
|
||||
rules:
|
||||
- alert: SharedPostgresSqlExporterDown
|
||||
expr: up{job="shared-postgresql-sql"} == 0
|
||||
for: 3m
|
||||
labels: {severity: warning, service: shared-postgresql}
|
||||
annotations:
|
||||
summary: '共享 PG {{ $labels.env }} {{ $labels.member }} 的 SQL exporter 无法采集'
|
||||
# 进程停止已有 Patroni critical；这里覆盖进程在但 SQL 不通（HBA、连接耗尽、dev 实例）。
|
||||
- alert: SharedPostgresSqlUnreachable
|
||||
expr: pg_up{job="shared-postgresql-sql"} == 0
|
||||
for: 2m
|
||||
labels: {severity: warning, service: shared-postgresql}
|
||||
annotations:
|
||||
summary: '共享 PG {{ $labels.env }} {{ $labels.member }} 无法以监控角色执行 SQL'
|
||||
# 正常 primary 在 laptop ZFS SSD；pve1 的数据盘是 HDD。自动切换后主库安全落到 pve1
|
||||
# 不会触发任何“无主/多主”规则（2026-09-30 的切换整整一天无人察觉），所以单独提示回切。
|
||||
# 15m 留给计划内切换演练。
|
||||
- alert: SharedPostgresPrimaryOnStandbyHost
|
||||
expr: pg_replication_is_replica{job="shared-postgresql-sql", env="prod", member="pg-pve1"} == 0
|
||||
for: 15m
|
||||
labels: {severity: warning, service: shared-postgresql}
|
||||
annotations:
|
||||
summary: 共享 PG 生产主库运行在 pg-pve1（HDD），请确认切换原因并计划回切 laptop
|
||||
- alert: SharedPostgresSqlCollectorFailing
|
||||
expr: pg_scrape_collector_success{job="shared-postgresql-sql"} == 0 or pg_exporter_last_scrape_error{job="shared-postgresql-sql"} == 1
|
||||
for: 10m
|
||||
labels: {severity: warning, service: shared-postgresql}
|
||||
annotations:
|
||||
summary: '共享 PG {{ $labels.env }} {{ $labels.member }} 部分 SQL 指标采集失败 {{ $labels.collector }}'
|
||||
# 与 Patroni maximum_lag_on_failover（1 MiB）一致：持续超过即该 standby 不再是合格的切换候选。
|
||||
- alert: SharedPostgresReplicationLagging
|
||||
expr: pg_stat_replication_pg_wal_lsn_diff{job="shared-postgresql-sql"} > 1048576
|
||||
for: 5m
|
||||
labels: {severity: warning, service: shared-postgresql}
|
||||
annotations:
|
||||
summary: '共享 PG {{ $labels.env }} 副本 {{ $labels.application_name }} 回放落后超过 1 MiB，已不满足自动切换条件'
|
||||
# 只看可写成员：standby 上 Patroni 维护的槽副本不反映真实保留压力。
|
||||
# 1 GiB 是 max_slot_wal_keep_size（2GB）的一半，留出处理时间。
|
||||
- alert: SharedPostgresReplicationSlotRetention
|
||||
expr: >-
|
||||
pg_replication_slots_pg_wal_lsn_diff{job="shared-postgresql-sql"} > 1073741824
|
||||
and on(instance) pg_replication_is_replica{job="shared-postgresql-sql"} == 0
|
||||
for: 10m
|
||||
labels: {severity: warning, service: shared-postgresql}
|
||||
annotations:
|
||||
summary: '共享 PG {{ $labels.env }} 复制槽 {{ $labels.slot_name }} 保留 WAL 超过 1 GiB'
|
||||
# 计数器规则不用 increase()：MetricsQL 把新序列首个样本视为从 0 增长，exporter 新上线或
|
||||
# 角色变化时，历史累计值（如 2026-09-25 搭建期的 9 次归档失败）会被当成刚发生而误报。
|
||||
# 与 offset 相减对新序列返回空；计数器重置（重启/stats reset）得负值，同样不告警。
|
||||
- alert: SharedPostgresWalArchiveFailing
|
||||
expr: >-
|
||||
pg_stat_archiver_failed_count{job="shared-postgresql-sql"}
|
||||
- pg_stat_archiver_failed_count{job="shared-postgresql-sql"} offset 15m > 0
|
||||
labels: {severity: warning, service: shared-postgresql}
|
||||
annotations:
|
||||
summary: '共享 PG {{ $labels.env }} {{ $labels.member }} WAL 归档失败，PITR 链可能中断'
|
||||
# XID/MXID 上限约 2^31；超过一半说明 freeze 跟不上，超过 75% 已接近强制停写；critical 时不重复 warning。
|
||||
- alert: SharedPostgresWraparoundRisk
|
||||
expr: >-
|
||||
(max without(datname) (pg_database_wraparound_age_datfrozenxid_seconds{job="shared-postgresql-sql"}) > 1e9
|
||||
or max without(datname) (pg_database_wraparound_age_datminmxid_seconds{job="shared-postgresql-sql"}) > 1e9)
|
||||
unless (max without(datname) (pg_database_wraparound_age_datfrozenxid_seconds{job="shared-postgresql-sql"}) > 1.6e9
|
||||
or max without(datname) (pg_database_wraparound_age_datminmxid_seconds{job="shared-postgresql-sql"}) > 1.6e9)
|
||||
for: 10m
|
||||
labels: {severity: warning, service: shared-postgresql}
|
||||
annotations:
|
||||
summary: '共享 PG {{ $labels.env }} {{ $labels.member }} XID/MXID 年龄超过 10 亿'
|
||||
- alert: SharedPostgresWraparoundImminent
|
||||
expr: >-
|
||||
max without(datname) (pg_database_wraparound_age_datfrozenxid_seconds{job="shared-postgresql-sql"}) > 1.6e9
|
||||
or max without(datname) (pg_database_wraparound_age_datminmxid_seconds{job="shared-postgresql-sql"}) > 1.6e9
|
||||
for: 5m
|
||||
labels: {severity: critical, service: shared-postgresql}
|
||||
annotations:
|
||||
summary: '共享 PG {{ $labels.env }} {{ $labels.member }} XID/MXID 接近回卷停写，需立即 VACUUM FREEZE'
|
||||
- alert: SharedPostgresConnectionsHigh
|
||||
expr: >-
|
||||
sum without(datname, state, usename, application_name, backend_type, wait_event_type, wait_event)
|
||||
(pg_stat_activity_count{job="shared-postgresql-sql", backend_type="client backend"})
|
||||
/ pg_settings_max_connections{job="shared-postgresql-sql"} > 0.8
|
||||
for: 5m
|
||||
labels: {severity: warning, service: shared-postgresql}
|
||||
annotations:
|
||||
summary: '共享 PG {{ $labels.env }} {{ $labels.member }} 客户端连接超过 max_connections 的 80%'
|
||||
# 长事务阻止 vacuum 回收并持有锁；idle in transaction 通常是应用忘记提交。
|
||||
- alert: SharedPostgresIdleInTransaction
|
||||
expr: >-
|
||||
max without(datname, state, usename, application_name, backend_type, wait_event_type, wait_event)
|
||||
(pg_stat_activity_max_tx_duration{job="shared-postgresql-sql", state=~"idle in transaction.*"}) > 600
|
||||
for: 1m
|
||||
labels: {severity: warning, service: shared-postgresql}
|
||||
annotations:
|
||||
summary: '共享 PG {{ $labels.env }} {{ $labels.member }} 有事务空闲未提交超过 10 分钟'
|
||||
- alert: SharedPostgresLongTransaction
|
||||
expr: pg_long_running_transactions_oldest_timestamp_seconds{job="shared-postgresql-sql"} > 3600
|
||||
for: 1m
|
||||
labels: {severity: warning, service: shared-postgresql}
|
||||
annotations:
|
||||
summary: '共享 PG {{ $labels.env }} {{ $labels.member }} 最老事务已运行超过 1 小时'
|
||||
- alert: SharedPostgresDeadlocks
|
||||
expr: >-
|
||||
sum without(datname) (pg_stat_database_deadlocks{job="shared-postgresql-sql"}
|
||||
- pg_stat_database_deadlocks{job="shared-postgresql-sql"} offset 15m) > 0
|
||||
labels: {severity: warning, service: shared-postgresql}
|
||||
annotations:
|
||||
summary: '共享 PG {{ $labels.env }} {{ $labels.member }} 15 分钟内出现死锁'
|
||||
# 每日 pgBackRest full 会请求一次 checkpoint；每小时多于 4 次通常意味着 max_wal_size 偏小。
|
||||
- alert: SharedPostgresCheckpointsRequested
|
||||
expr: >-
|
||||
pg_stat_checkpointer_num_requested_total{job="shared-postgresql-sql"}
|
||||
- pg_stat_checkpointer_num_requested_total{job="shared-postgresql-sql"} offset 1h > 4
|
||||
for: 15m
|
||||
labels: {severity: warning, service: shared-postgresql}
|
||||
annotations:
|
||||
summary: '共享 PG {{ $labels.env }} {{ $labels.member }} 频繁请求 checkpoint，max_wal_size 可能偏小'
|
||||
@@ -1,18 +0,0 @@
|
||||
# postgres_exporter 跑在数据库主机上，按实例一个端口；env 标签供 Alertmanager 分流 dev。
|
||||
apiVersion: operator.victoriametrics.com/v1beta1
|
||||
kind: VMStaticScrape
|
||||
metadata:
|
||||
name: shared-postgresql-sql
|
||||
namespace: monitoring
|
||||
spec:
|
||||
jobName: shared-postgresql-sql
|
||||
targetEndpoints:
|
||||
- targets: ['192.168.10.127:9187']
|
||||
labels: {cluster: homelab-pg-prod, member: pg-laptop, env: prod}
|
||||
interval: 30s
|
||||
- targets: ['10.60.0.20:9187']
|
||||
labels: {cluster: homelab-pg-prod, member: pg-pve1, env: prod}
|
||||
interval: 30s
|
||||
- targets: ['192.168.10.127:9188']
|
||||
labels: {cluster: homelab-pg-dev, member: pg-laptop, env: dev}
|
||||
interval: 30s
|
||||
@@ -10,7 +10,7 @@ import sys
|
||||
import yaml
|
||||
|
||||
groups = []
|
||||
for name in ("kubernetes-health", "host-health", "monitoring-delivery", "platform-controllers", "data-services", "openbao", "shared-data-maintenance", "shared-postgresql-sql"):
|
||||
for name in ("kubernetes-health", "host-health", "monitoring-delivery", "platform-controllers", "data-services", "openbao", "shared-data-maintenance"):
|
||||
path = pathlib.Path(sys.argv[1]) / f"{name}.yaml"
|
||||
groups.extend(yaml.safe_load(path.read_text())["spec"]["groups"])
|
||||
pathlib.Path(sys.argv[2]).write_text(yaml.safe_dump({"groups": groups}, allow_unicode=True))
|
||||
|
||||
@@ -125,7 +125,6 @@ tests:
|
||||
instance: host:9109
|
||||
severity: warning
|
||||
service: shared-postgresql
|
||||
env: dev
|
||||
exp_annotations:
|
||||
summary: 共享 PG 开发实例本地 SQL 不可用或处于只读状态
|
||||
- name: dev 恢复后解除
|
||||
|
||||
@@ -1,195 +0,0 @@
|
||||
rule_files:
|
||||
- rules.yaml
|
||||
evaluation_interval: 1m
|
||||
tests:
|
||||
- name: dev SQL 不通时告警带 env=dev 以便分流
|
||||
interval: 1m
|
||||
input_series:
|
||||
- series: pg_up{job="shared-postgresql-sql",instance="h:9188",cluster="homelab-pg-dev",member="pg-laptop",env="dev"}
|
||||
values: '0x5'
|
||||
alert_rule_test:
|
||||
- eval_time: 5m
|
||||
alertname: SharedPostgresSqlUnreachable
|
||||
exp_alerts:
|
||||
- exp_labels: {job: shared-postgresql-sql, instance: 'h:9188', cluster: homelab-pg-dev, member: pg-laptop, env: dev, severity: warning, service: shared-postgresql}
|
||||
exp_annotations:
|
||||
summary: 共享 PG dev pg-laptop 无法以监控角色执行 SQL
|
||||
- name: 回放落后持续超过 1 MiB 告警
|
||||
interval: 1m
|
||||
input_series:
|
||||
- series: pg_stat_replication_pg_wal_lsn_diff{job="shared-postgresql-sql",instance="p:9187",member="pg-pve1",env="prod",application_name="pg-laptop"}
|
||||
values: '2097152x10'
|
||||
alert_rule_test:
|
||||
- eval_time: 8m
|
||||
alertname: SharedPostgresReplicationLagging
|
||||
exp_alerts:
|
||||
- exp_labels: {job: shared-postgresql-sql, instance: 'p:9187', member: pg-pve1, env: prod, application_name: pg-laptop, severity: warning, service: shared-postgresql}
|
||||
exp_annotations:
|
||||
summary: 共享 PG prod 副本 pg-laptop 回放落后超过 1 MiB，已不满足自动切换条件
|
||||
- name: 短暂落后不告警
|
||||
interval: 1m
|
||||
input_series:
|
||||
- series: pg_stat_replication_pg_wal_lsn_diff{job="shared-postgresql-sql",instance="p:9187",env="prod",application_name="pg-laptop"}
|
||||
values: '2097152x2 0x8'
|
||||
alert_rule_test:
|
||||
- eval_time: 8m
|
||||
alertname: SharedPostgresReplicationLagging
|
||||
exp_alerts: []
|
||||
- name: 复制槽积压只在主库告警
|
||||
interval: 1m
|
||||
input_series:
|
||||
- series: pg_replication_slots_pg_wal_lsn_diff{job="shared-postgresql-sql",instance="p:9187",env="prod",slot_name="pg_laptop"}
|
||||
values: '2147483648x15'
|
||||
- series: pg_replication_is_replica{job="shared-postgresql-sql",instance="p:9187",env="prod"}
|
||||
values: '0x15'
|
||||
- series: pg_replication_slots_pg_wal_lsn_diff{job="shared-postgresql-sql",instance="r:9187",env="prod",slot_name="pg_pve1"}
|
||||
values: '2147483648x15'
|
||||
- series: pg_replication_is_replica{job="shared-postgresql-sql",instance="r:9187",env="prod"}
|
||||
values: '1x15'
|
||||
alert_rule_test:
|
||||
- eval_time: 12m
|
||||
alertname: SharedPostgresReplicationSlotRetention
|
||||
exp_alerts:
|
||||
- exp_labels: {job: shared-postgresql-sql, instance: 'p:9187', env: prod, slot_name: pg_laptop, severity: warning, service: shared-postgresql}
|
||||
exp_annotations:
|
||||
summary: 共享 PG prod 复制槽 pg_laptop 保留 WAL 超过 1 GiB
|
||||
- name: 回卷 critical 时不重复 warning
|
||||
interval: 1m
|
||||
input_series:
|
||||
- series: pg_database_wraparound_age_datfrozenxid_seconds{job="shared-postgresql-sql",instance="p:9187",member="pg-pve1",env="prod",datname="app"}
|
||||
values: '1700000000x15'
|
||||
- series: pg_database_wraparound_age_datminmxid_seconds{job="shared-postgresql-sql",instance="p:9187",member="pg-pve1",env="prod",datname="app"}
|
||||
values: '1200000000x15'
|
||||
alert_rule_test:
|
||||
- eval_time: 12m
|
||||
alertname: SharedPostgresWraparoundRisk
|
||||
exp_alerts: []
|
||||
- eval_time: 12m
|
||||
alertname: SharedPostgresWraparoundImminent
|
||||
exp_alerts:
|
||||
- exp_labels: {job: shared-postgresql-sql, instance: 'p:9187', member: pg-pve1, env: prod, severity: critical, service: shared-postgresql}
|
||||
exp_annotations:
|
||||
summary: 共享 PG prod pg-pve1 XID/MXID 接近回卷停写，需立即 VACUUM FREEZE
|
||||
- name: 回卷 warning
|
||||
interval: 1m
|
||||
input_series:
|
||||
- series: pg_database_wraparound_age_datfrozenxid_seconds{job="shared-postgresql-sql",instance="p:9187",member="pg-pve1",env="prod",datname="app"}
|
||||
values: '1200000000x15'
|
||||
alert_rule_test:
|
||||
- eval_time: 12m
|
||||
alertname: SharedPostgresWraparoundRisk
|
||||
exp_alerts:
|
||||
- exp_labels: {job: shared-postgresql-sql, instance: 'p:9187', member: pg-pve1, env: prod, severity: warning, service: shared-postgresql}
|
||||
exp_annotations:
|
||||
summary: 共享 PG prod pg-pve1 XID/MXID 年龄超过 10 亿
|
||||
- name: 只计客户端连接
|
||||
interval: 1m
|
||||
input_series:
|
||||
- series: pg_stat_activity_count{job="shared-postgresql-sql",instance="p:9187",member="pg-pve1",env="prod",datname="app",state="active",usename="a",application_name="x",backend_type="client backend",wait_event_type="",wait_event=""}
|
||||
values: '50x10'
|
||||
- series: pg_stat_activity_count{job="shared-postgresql-sql",instance="p:9187",member="pg-pve1",env="prod",datname="app",state="idle",usename="b",application_name="y",backend_type="client backend",wait_event_type="",wait_event=""}
|
||||
values: '20x10'
|
||||
- series: pg_stat_activity_count{job="shared-postgresql-sql",instance="p:9187",member="pg-pve1",env="prod",datname="",state="",usename="",application_name="",backend_type="autovacuum worker",wait_event_type="",wait_event=""}
|
||||
values: '30x10'
|
||||
- series: pg_settings_max_connections{job="shared-postgresql-sql",instance="p:9187",member="pg-pve1",env="prod"}
|
||||
values: '80x10'
|
||||
alert_rule_test:
|
||||
- eval_time: 8m
|
||||
alertname: SharedPostgresConnectionsHigh
|
||||
exp_alerts:
|
||||
- exp_labels: {job: shared-postgresql-sql, instance: 'p:9187', member: pg-pve1, env: prod, severity: warning, service: shared-postgresql}
|
||||
exp_annotations:
|
||||
summary: 共享 PG prod pg-pve1 客户端连接超过 max_connections 的 80%
|
||||
- name: 后台进程不计入连接使用率
|
||||
interval: 1m
|
||||
input_series:
|
||||
- series: pg_stat_activity_count{job="shared-postgresql-sql",instance="p:9187",env="prod",datname="app",state="active",usename="a",application_name="x",backend_type="client backend",wait_event_type="",wait_event=""}
|
||||
values: '40x10'
|
||||
- series: pg_stat_activity_count{job="shared-postgresql-sql",instance="p:9187",env="prod",datname="",state="",usename="",application_name="",backend_type="autovacuum worker",wait_event_type="",wait_event=""}
|
||||
values: '40x10'
|
||||
- series: pg_settings_max_connections{job="shared-postgresql-sql",instance="p:9187",env="prod"}
|
||||
values: '80x10'
|
||||
alert_rule_test:
|
||||
- eval_time: 8m
|
||||
alertname: SharedPostgresConnectionsHigh
|
||||
exp_alerts: []
|
||||
- name: 死锁计数增长告警
|
||||
interval: 1m
|
||||
input_series:
|
||||
- series: pg_stat_database_deadlocks{job="shared-postgresql-sql",instance="p:9187",member="pg-pve1",env="prod",datname="app"}
|
||||
values: '0x20 1x5'
|
||||
alert_rule_test:
|
||||
- eval_time: 22m
|
||||
alertname: SharedPostgresDeadlocks
|
||||
exp_alerts:
|
||||
- exp_labels: {job: shared-postgresql-sql, instance: 'p:9187', member: pg-pve1, env: prod, severity: warning, service: shared-postgresql}
|
||||
exp_annotations:
|
||||
summary: 共享 PG prod pg-pve1 15 分钟内出现死锁
|
||||
- name: 空闲事务超过 10 分钟
|
||||
interval: 1m
|
||||
input_series:
|
||||
- series: pg_stat_activity_max_tx_duration{job="shared-postgresql-sql",instance="p:9187",member="pg-pve1",env="prod",datname="app",state="idle in transaction",usename="a",application_name="x",backend_type="client backend",wait_event_type="Client",wait_event="ClientRead"}
|
||||
values: '700x5'
|
||||
alert_rule_test:
|
||||
- eval_time: 3m
|
||||
alertname: SharedPostgresIdleInTransaction
|
||||
exp_alerts:
|
||||
- exp_labels: {job: shared-postgresql-sql, instance: 'p:9187', member: pg-pve1, env: prod, severity: warning, service: shared-postgresql}
|
||||
exp_annotations:
|
||||
summary: 共享 PG prod pg-pve1 有事务空闲未提交超过 10 分钟
|
||||
- name: 主库落在 pve1 持续 15 分钟提示回切
|
||||
interval: 1m
|
||||
input_series:
|
||||
- series: pg_replication_is_replica{job="shared-postgresql-sql",instance="10.60.0.20:9187",cluster="homelab-pg-prod",member="pg-pve1",env="prod"}
|
||||
values: '0x20'
|
||||
- series: pg_replication_is_replica{job="shared-postgresql-sql",instance="192.168.10.127:9187",cluster="homelab-pg-prod",member="pg-laptop",env="prod"}
|
||||
values: '1x20'
|
||||
alert_rule_test:
|
||||
- eval_time: 10m
|
||||
alertname: SharedPostgresPrimaryOnStandbyHost
|
||||
exp_alerts: []
|
||||
- eval_time: 16m
|
||||
alertname: SharedPostgresPrimaryOnStandbyHost
|
||||
exp_alerts:
|
||||
- exp_labels: {job: shared-postgresql-sql, instance: '10.60.0.20:9187', cluster: homelab-pg-prod, member: pg-pve1, env: prod, severity: warning, service: shared-postgresql}
|
||||
exp_annotations:
|
||||
summary: 共享 PG 生产主库运行在 pg-pve1（HDD），请确认切换原因并计划回切 laptop
|
||||
- name: 主库在 laptop 不提示
|
||||
interval: 1m
|
||||
input_series:
|
||||
- series: pg_replication_is_replica{job="shared-postgresql-sql",instance="10.60.0.20:9187",member="pg-pve1",env="prod"}
|
||||
values: '1x20'
|
||||
alert_rule_test:
|
||||
- eval_time: 16m
|
||||
alertname: SharedPostgresPrimaryOnStandbyHost
|
||||
exp_alerts: []
|
||||
- name: 新出现的计数器序列带历史累计值不误报归档失败
|
||||
interval: 1m
|
||||
input_series:
|
||||
- series: pg_stat_archiver_failed_count{job="shared-postgresql-sql",instance="l:9187",member="pg-laptop",env="prod"}
|
||||
values: '_x20 9x20'
|
||||
alert_rule_test:
|
||||
- eval_time: 25m
|
||||
alertname: SharedPostgresWalArchiveFailing
|
||||
exp_alerts: []
|
||||
- name: 归档失败计数增长告警
|
||||
interval: 1m
|
||||
input_series:
|
||||
- series: pg_stat_archiver_failed_count{job="shared-postgresql-sql",instance="l:9187",member="pg-laptop",env="prod"}
|
||||
values: '9x20 10x5'
|
||||
alert_rule_test:
|
||||
- eval_time: 22m
|
||||
alertname: SharedPostgresWalArchiveFailing
|
||||
exp_alerts:
|
||||
- exp_labels: {job: shared-postgresql-sql, instance: 'l:9187', member: pg-laptop, env: prod, severity: warning, service: shared-postgresql}
|
||||
exp_annotations:
|
||||
summary: 共享 PG prod pg-laptop WAL 归档失败，PITR 链可能中断
|
||||
- name: 计数器重置不误报 checkpoint
|
||||
interval: 1m
|
||||
input_series:
|
||||
- series: pg_stat_checkpointer_num_requested_total{job="shared-postgresql-sql",instance="l:9187",member="pg-laptop",env="prod"}
|
||||
values: '16x60 0x30'
|
||||
alert_rule_test:
|
||||
- eval_time: 85m
|
||||
alertname: SharedPostgresCheckpointsRequested
|
||||
exp_alerts: []
|
||||
@@ -16,10 +16,6 @@ spec:
|
||||
group_interval: 5m
|
||||
repeat_interval: 4h
|
||||
routes:
|
||||
# dev 必须排第一且不 continue：否则 dev 告警会被后面的 severity 路由送进生产频道。
|
||||
- receiver: telegram-dev
|
||||
matchers:
|
||||
- env="dev"
|
||||
- receiver: telegram
|
||||
matchers:
|
||||
- severity="critical"
|
||||
@@ -41,11 +37,6 @@ spec:
|
||||
- bot_token_file: /etc/vm/secrets/alertmanager-telegram/telegram_bot_token
|
||||
chat_id: -1003956377923
|
||||
send_resolved: true
|
||||
- name: telegram-dev
|
||||
telegram_configs:
|
||||
- bot_token_file: /etc/vm/secrets/alertmanager-telegram/telegram_bot_token
|
||||
chat_id: -1003651477106
|
||||
send_resolved: true
|
||||
resources:
|
||||
requests:
|
||||
cpu: 25m
|
||||
|
||||
Reference in New Issue
Block a user