Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
 panxiao81
|
84a1e9e37f
|
@@ -1,32 +0,0 @@
|
|||||||
---
|
|
||||||
name: dind-fuse-image
|
|
||||||
|
|
||||||
on:
|
|
||||||
workflow_dispatch:
|
|
||||||
push:
|
|
||||||
branches: [main]
|
|
||||||
paths:
|
|
||||||
- platform/gitea-runner/images/dind-fuse/**
|
|
||||||
- .gitea/workflows/dind-fuse-image.yml
|
|
||||||
|
|
||||||
jobs:
|
|
||||||
publish:
|
|
||||||
runs-on: [self-hosted, pod]
|
|
||||||
steps:
|
|
||||||
- uses: actions/checkout@v4
|
|
||||||
|
|
||||||
- name: Log in to the Gitea container registry
|
|
||||||
uses: docker/login-action@v3
|
|
||||||
with:
|
|
||||||
registry: git.ddupan.top
|
|
||||||
username: ${{ gitea.actor }}
|
|
||||||
password: ${{ secrets.REGISTRY_TOKEN }}
|
|
||||||
|
|
||||||
- name: Build and publish
|
|
||||||
uses: docker/build-push-action@v6
|
|
||||||
with:
|
|
||||||
context: platform/gitea-runner/images/dind-fuse
|
|
||||||
push: true
|
|
||||||
tags: |
|
|
||||||
git.ddupan.top/panxiao81/dind-fuse:29.7.1
|
|
||||||
git.ddupan.top/panxiao81/dind-fuse:latest
|
|
||||||
@@ -1,24 +0,0 @@
|
|||||||
---
|
|
||||||
name: kata-runner-smoke
|
|
||||||
|
|
||||||
on:
|
|
||||||
workflow_dispatch:
|
|
||||||
push:
|
|
||||||
branches:
|
|
||||||
- poc/kata-runner-smoke
|
|
||||||
paths:
|
|
||||||
- .gitea/workflows/kata-runner-smoke.yml
|
|
||||||
|
|
||||||
jobs:
|
|
||||||
smoke:
|
|
||||||
runs-on: [self-hosted, pod]
|
|
||||||
steps:
|
|
||||||
- name: Verify isolated job environment
|
|
||||||
shell: sh
|
|
||||||
run: |
|
|
||||||
set -eu
|
|
||||||
uname -a
|
|
||||||
grep -q '^ID=alpine$' /etc/os-release
|
|
||||||
test -f /.dockerenv
|
|
||||||
test ! -e /dev/kvm
|
|
||||||
printf 'kata ephemeral runner job ok\n'
|
|
||||||
@@ -1,63 +0,0 @@
|
|||||||
---
|
|
||||||
name: kind-on-kata-smoke
|
|
||||||
|
|
||||||
on:
|
|
||||||
push:
|
|
||||||
branches:
|
|
||||||
- poc/kind-on-kata
|
|
||||||
paths:
|
|
||||||
- .gitea/workflows/kind-on-kata-smoke.yml
|
|
||||||
workflow_dispatch:
|
|
||||||
|
|
||||||
jobs:
|
|
||||||
smoke:
|
|
||||||
runs-on: [self-hosted, pod]
|
|
||||||
steps:
|
|
||||||
- name: Create nested kind cluster
|
|
||||||
shell: sh
|
|
||||||
env:
|
|
||||||
KIND_VERSION: v0.27.0
|
|
||||||
KIND_NODE_IMAGE: kindest/node:v1.32.2@sha256:f226345927d7e348497136874b6d207e0b32cc52154ad8323129352923a3142f
|
|
||||||
run: |
|
|
||||||
set -eu
|
|
||||||
apk add --no-cache ca-certificates curl docker-cli
|
|
||||||
curl --retry 5 --retry-all-errors --connect-timeout 15 -fsSLo /tmp/kind \
|
|
||||||
"https://kind.sigs.k8s.io/dl/${KIND_VERSION}/kind-linux-amd64"
|
|
||||||
curl --retry 5 --retry-all-errors --connect-timeout 15 -fsSLo /tmp/kind.sha256sum \
|
|
||||||
"https://kind.sigs.k8s.io/dl/${KIND_VERSION}/kind-linux-amd64.sha256sum"
|
|
||||||
expected="$(awk '{print $1}' /tmp/kind.sha256sum)"
|
|
||||||
printf '%s %s\n' "$expected" /tmp/kind | sha256sum -c -
|
|
||||||
install -m 0755 /tmp/kind /usr/local/bin/kind
|
|
||||||
docker info --format 'kernel={{.KernelVersion}} driver={{.Driver}}'
|
|
||||||
test "$(docker info --format '{{.Driver}}')" = overlay2
|
|
||||||
|
|
||||||
cleanup() {
|
|
||||||
kind delete cluster --name nested >/dev/null 2>&1 || true
|
|
||||||
}
|
|
||||||
trap cleanup EXIT
|
|
||||||
|
|
||||||
cat >/tmp/kind-config.yaml <<'EOF'
|
|
||||||
kind: Cluster
|
|
||||||
apiVersion: kind.x-k8s.io/v1alpha4
|
|
||||||
name: nested
|
|
||||||
nodes:
|
|
||||||
- role: control-plane
|
|
||||||
extraMounts:
|
|
||||||
- hostPath: /dev/kmsg
|
|
||||||
containerPath: /dev/kmsg
|
|
||||||
EOF
|
|
||||||
kind create cluster -v 9 --retain --config /tmp/kind-config.yaml --image "$KIND_NODE_IMAGE" --wait 5m
|
|
||||||
docker exec nested-control-plane kubectl \
|
|
||||||
--kubeconfig=/etc/kubernetes/admin.conf wait \
|
|
||||||
--for=condition=Ready node/nested-control-plane --timeout=2m
|
|
||||||
docker exec nested-control-plane kubectl \
|
|
||||||
--kubeconfig=/etc/kubernetes/admin.conf run smoke \
|
|
||||||
--image=docker.io/library/busybox:1.37 --restart=Never \
|
|
||||||
--command -- sh -c 'echo kind-on-kata-ok'
|
|
||||||
docker exec nested-control-plane kubectl \
|
|
||||||
--kubeconfig=/etc/kubernetes/admin.conf wait \
|
|
||||||
--for=jsonpath='{.status.phase}'=Succeeded pod/smoke --timeout=2m
|
|
||||||
test "$(docker exec nested-control-plane kubectl \
|
|
||||||
--kubeconfig=/etc/kubernetes/admin.conf logs smoke)" = kind-on-kata-ok
|
|
||||||
kind delete cluster --name nested
|
|
||||||
trap - EXIT
|
|
||||||
@@ -18,13 +18,3 @@ OCI digest 固定镜像。
|
|||||||
|
|
||||||
Flux 等待 ExternalSecret 和 Deployment 就绪；任何前置缺失都会使该
|
Flux 等待 ExternalSecret 和 Deployment 就绪；任何前置缺失都会使该
|
||||||
Kustomization 保持 NotReady，而不会回退到明文 Secret。
|
Kustomization 保持 NotReady，而不会回退到明文 Secret。
|
||||||
|
|
||||||
## 镜像更新
|
|
||||||
|
|
||||||
`panxiao81/backstage` 的 CI 在 main push 后只推送 `:latest`。
|
|
||||||
`platform/flux-image-automation` 跟踪 `latest` 背后的 digest，由 `flux-bot`
|
|
||||||
直接提交到本仓库 main，不经过 PR。`deployment.yaml` 中 image 行的
|
|
||||||
`$imagepolicy` 注释是 setter 标记，删除后自动更新会静默停止。
|
|
||||||
|
|
||||||
旧 digest 失去 tag 后会被 zot GC 回收（24h），因此不要把 Deployment 手工固定到
|
|
||||||
一个已不是 `latest` 的 digest 并长期保留。
|
|
||||||
|
|||||||
@@ -27,7 +27,7 @@ spec:
|
|||||||
type: RuntimeDefault
|
type: RuntimeDefault
|
||||||
containers:
|
containers:
|
||||||
- name: backstage
|
- name: backstage
|
||||||
image: zot.ad.ddupan.top/panxiao81/backstage:latest@sha256:da55be5c2f5c8b33de2d87ee0ef02edeacf52a349c8b19a5fac9be4152b2723d # {"$imagepolicy": "flux-system:backstage"}
|
image: zot.ad.ddupan.top/panxiao81/backstage@sha256:e5a12550726f19a680bc7c40e2cc07cc624318f9279ee121814d293a006ef210
|
||||||
imagePullPolicy: IfNotPresent
|
imagePullPolicy: IfNotPresent
|
||||||
env:
|
env:
|
||||||
- name: BACKSTAGE_BASE_URL
|
- name: BACKSTAGE_BASE_URL
|
||||||
|
|||||||
+6
-61
@@ -1,69 +1,14 @@
|
|||||||
# Hydra 与独立 IAM 登录接入
|
# Hydra 与 OIDC Login/Consent PoC
|
||||||
|
|
||||||
本目录提供独立 Hydra 签发服务。当前分支将实验性 Hydra 登录入口接至 Spring `iam-login`
|
本目录提供独立 Hydra 签发服务，以及一个薄的 **OIDC 上游适配器**。当前上游配置为
|
||||||
开发实例，由其执行 AD 密码、WebAuthn 和授权确认；不改变 issuer、Gitea 登录源或数据库。
|
Authelia；适配器不连接 LDAP，也不管理用户目录。Samba AD、密码和 MFA 继续由现有
|
||||||
旧 Go OIDC 适配器继续部署以便回退，Gitea 的直接 Authelia 登录源也保留。
|
Authelia 链路负责。第一轮只接入人类和 Gitea，不实现 agent 动态授权。
|
||||||
该切换已于 2026-10-01 经 PR #168 合并并由 Flux 应用；下面原有 Go/Authelia 说明保留为回退路径资料。
|
|
||||||
|
|
||||||
```text
|
```text
|
||||||
Gitea → Hydra → iam-login（Tailscale 开发实例）→ Samba AD + WebAuthn
|
Gitea → Hydra → OIDC Login/Consent → Authelia → Samba AD
|
||||||
|
← OIDC ← 经验证的上游身份 ← OIDC callback
|
||||||
```
|
```
|
||||||
|
|
||||||
## Spring 开发路径验收
|
|
||||||
|
|
||||||
源码版本：[iam-login 5f49a7c](https://git.ddupan.top/panxiao81/iam-login/commit/5f49a7c)。
|
|
||||||
开发 UI 是 `https://laptop.tail7e769.ts.net:18082`，已有 Passkey RP ID 保持不变。
|
|
||||||
Hydra 回调为 `/oauth2/start`、`/oauth2/consent`、`/oauth2/logout`；默认注销回到 `/signin`。
|
|
||||||
浏览器需要 Tailscale 连通性；开发服务暂由本地 JVM 容器运行，并非生产 Native 部署。
|
|
||||||
|
|
||||||
开发服务通过 `kubectl -n hydra port-forward --address 127.0.0.1 service/hydra-admin 18445:4445`
|
|
||||||
访问私有 Admin API。转发需在 Hydra 滚动更新后重连；验收运行器会循环重建该通道。
|
|
||||||
该通道与本地容器是临时验收依赖，必须保持运行；不修改 NetworkPolicy、
|
|
||||||
不新增 Admin HTTPRoute。临时运行配置与旧版回退备份仅保存在受限本地运行目录，不提交秘密。
|
|
||||||
|
|
||||||
验收前已只读核对目标 Gitea 账号的现有 Hydra 外部关联，并将旧 sub 显式绑定到 AD
|
|
||||||
稳定主体；不按邮箱重建关联、不修改 Gitea 账号或仓库权限。客户端管理仅允许有效 MFA 且
|
|
||||||
直接属于 AD Domain Admins 的用户；这是验收期明确指定的粗粒度管理组。
|
|
||||||
|
|
||||||
### 客户端管理 `/console`
|
|
||||||
|
|
||||||
iam-login 的客户端管理 API 只接受 Hydra 签发、带 `iam.clients.manage` scope 的 access token；
|
|
||||||
`/console` 是调用该 API 的前端，在 Hydra 中登记为普通 **public** 客户端 `iam-admin-ui`。
|
|
||||||
该 scope 只在 consent 时发给 `iam-admin-ui` 且直接属于管理组的用户，规则在 iam-login。
|
|
||||||
|
|
||||||
浏览器直接向 Hydra 换取 token，因此 `hydra.yaml` 为 public 端口开启 CORS，只允许开发实例
|
|
||||||
origin；不放行 cookie 凭据。Gitea 等服务端客户端不受影响。
|
|
||||||
|
|
||||||
`iam-admin-ui` 无 secret，与 `gitea` 一样经 Admin 带外登记（它是 public 客户端，iam-login API
|
|
||||||
看不到也删不掉它，恢复同样走此通道）：
|
|
||||||
|
|
||||||
```sh
|
|
||||||
curl -fsS -X POST http://127.0.0.1:18445/admin/clients -H 'Content-Type: application/json' -d '{
|
|
||||||
"client_id": "iam-admin-ui", "client_name": "IAM 管理",
|
|
||||||
"redirect_uris": ["https://laptop.tail7e769.ts.net:18082/console/callback"],
|
|
||||||
"grant_types": ["authorization_code"], "response_types": ["code"],
|
|
||||||
"scope": "openid iam.clients.manage", "token_endpoint_auth_method": "none",
|
|
||||||
"subject_type": "public", "metadata": {"iam_login_enabled": true}}'
|
|
||||||
```
|
|
||||||
|
|
||||||
开发实例另需 `iam.clients.admin-ui-client-id=iam-admin-ui`；未配置时 `/console` 不提供，API
|
|
||||||
无法获得可用 token（fail closed）。
|
|
||||||
|
|
||||||
从 Gitea `/user/oauth2/hydra` 发起，应进入新密码/Passkey 页，确认授权后返回原账号，核对
|
|
||||||
仓库权限。当前 Gitea client 未登记 front/back-channel 或 post-logout 回调，不能声称 Gitea 会话会
|
|
||||||
随 IAM 注销。应用的注销协议兼容性需单独验收。旧 `authelia` 登录源始终保留。
|
|
||||||
|
|
||||||
开发配置关闭 Boot 默认 LDAP health indicator：它未配置目录连接，不对应同用户 bind 的
|
|
||||||
AD 仓储。总健康检查仍验证数据库，真实 AD 认证由本次人类登录验收，不将 readiness 当作
|
|
||||||
AD 凭据有效性的证明。
|
|
||||||
|
|
||||||
Hydra 的 login/consent URL 是全局配置，本次影响全部经 Hydra 的登录请求（当前接入 Gitea），
|
|
||||||
不影响直接走 Authelia 的应用。不要在无法访问 Tailscale 开发实例时合并此切换。
|
|
||||||
验收失败时 revert 本次回调变更，恢复旧 Go 的 `/login`、`/consent`；不删除客户端、签名密钥、
|
|
||||||
数据库、用户关联或 Passkey。最终 `auth.ddupan.top` 同源部署另开 PR，不在此次切换范围内。
|
|
||||||
|
|
||||||
## 原 Go/Authelia 路径与回退资料
|
|
||||||
|
|
||||||
目标入口：
|
目标入口：
|
||||||
|
|
||||||
- `https://hydra.ad.ddupan.top`：Hydra 公共 OAuth2/OIDC endpoint。
|
- `https://hydra.ad.ddupan.top`：Hydra 公共 OAuth2/OIDC endpoint。
|
||||||
|
|||||||
+2
-18
@@ -1,20 +1,6 @@
|
|||||||
serve:
|
serve:
|
||||||
public:
|
public:
|
||||||
port: 4444
|
port: 4444
|
||||||
# The iam-login /console admin UI is a public OIDC client that exchanges its code from the
|
|
||||||
# browser, so only that origin may call the public endpoints cross-origin. Server-side
|
|
||||||
# clients such as Gitea are unaffected by CORS.
|
|
||||||
cors:
|
|
||||||
enabled: true
|
|
||||||
allowed_origins:
|
|
||||||
- https://laptop.tail7e769.ts.net:18082
|
|
||||||
allowed_methods:
|
|
||||||
- GET
|
|
||||||
- POST
|
|
||||||
allowed_headers:
|
|
||||||
- Authorization
|
|
||||||
- Content-Type
|
|
||||||
allow_credentials: false
|
|
||||||
admin:
|
admin:
|
||||||
port: 4445
|
port: 4445
|
||||||
tls:
|
tls:
|
||||||
@@ -26,10 +12,8 @@ urls:
|
|||||||
self:
|
self:
|
||||||
issuer: https://hydra.ad.ddupan.top
|
issuer: https://hydra.ad.ddupan.top
|
||||||
public: https://hydra.ad.ddupan.top
|
public: https://hydra.ad.ddupan.top
|
||||||
login: https://laptop.tail7e769.ts.net:18082/oauth2/start
|
login: https://hydra-login.ad.ddupan.top/login
|
||||||
consent: https://laptop.tail7e769.ts.net:18082/oauth2/consent
|
consent: https://hydra-login.ad.ddupan.top/consent
|
||||||
logout: https://laptop.tail7e769.ts.net:18082/oauth2/logout
|
|
||||||
post_logout_redirect: https://laptop.tail7e769.ts.net:18082/signin
|
|
||||||
ttl:
|
ttl:
|
||||||
access_token: 15m
|
access_token: 15m
|
||||||
id_token: 15m
|
id_token: 15m
|
||||||
|
|||||||
@@ -1,16 +0,0 @@
|
|||||||
apiVersion: kustomize.toolkit.fluxcd.io/v1
|
|
||||||
kind: Kustomization
|
|
||||||
metadata:
|
|
||||||
name: flux-image-automation
|
|
||||||
namespace: flux-system
|
|
||||||
spec:
|
|
||||||
dependsOn:
|
|
||||||
- name: external-secrets
|
|
||||||
interval: 10m
|
|
||||||
path: ./platform/flux-image-automation
|
|
||||||
prune: true
|
|
||||||
sourceRef:
|
|
||||||
kind: GitRepository
|
|
||||||
name: flux-system
|
|
||||||
timeout: 5m
|
|
||||||
wait: true
|
|
||||||
File diff suppressed because it is too large
Load Diff
@@ -8,7 +8,6 @@ resources:
|
|||||||
- apps/external-secrets.yaml
|
- apps/external-secrets.yaml
|
||||||
- apps/gitea.yaml
|
- apps/gitea.yaml
|
||||||
- apps/backstage.yaml
|
- apps/backstage.yaml
|
||||||
- apps/flux-image-automation.yaml
|
|
||||||
- apps/http-echo.yaml
|
- apps/http-echo.yaml
|
||||||
- apps/openebs.yaml
|
- apps/openebs.yaml
|
||||||
- apps/nats.yaml
|
- apps/nats.yaml
|
||||||
|
|||||||
@@ -1,72 +0,0 @@
|
|||||||
# kata-lab
|
|
||||||
|
|
||||||
`kata-lab` 是位于 Proxmox `pve2` 的可销毁验证环境，用来验证 nested KVM、k3s、
|
|
||||||
Kata Containers，以及后续一 job 一 Kata Pod 的 Gitea Runner。它不是 dev/stg
|
|
||||||
长期服务，也不承载持久数据。
|
|
||||||
|
|
||||||
> 状态：VMID 147 已于 2026-09-14 销毁，Terraform state 为空。目录保留首次验证结果
|
|
||||||
> 与可复现配置；后续验证转向跨 PVE 节点的轻量 LXC worker 方案。
|
|
||||||
|
|
||||||
## Terraform bootstrap
|
|
||||||
|
|
||||||
1. 将 Proxmox API token 写入被 Git 忽略且权限为 `0600` 的
|
|
||||||
`terraform/credentials.auto.tfvars`。
|
|
||||||
2. 取得内部 CA，并仅为当前 Terraform 进程设置 Go TLS trust：
|
|
||||||
|
|
||||||
```bash
|
|
||||||
curl -fsSL https://bao.ad.ddupan.top:8200/v1/pki/ca/pem \
|
|
||||||
-o /tmp/kata-lab-ddupan-ca.pem
|
|
||||||
export SSL_CERT_FILE=/tmp/kata-lab-ddupan-ca.pem
|
|
||||||
```
|
|
||||||
|
|
||||||
3. 初始化并审阅 plan：
|
|
||||||
|
|
||||||
```bash
|
|
||||||
cd infrastructure/kata-lab/terraform
|
|
||||||
terraform init
|
|
||||||
terraform plan
|
|
||||||
```
|
|
||||||
|
|
||||||
Terraform 使用 `laptop:import/noble-server-cloudimg-amd64.qcow2`（40 GiB
|
|
||||||
virtual size），在
|
|
||||||
`pve-rg-hdd` 创建 VM root disk，并把 cloud-init snippet 放到 `laptop`
|
|
||||||
datastore。VM root disk 设为 41 GiB，以容纳 PVE import 的块对齐增量。VM 使用
|
|
||||||
DHCP、`cpu.type = host`、4 vCPU 和 4 GiB 内存；cloud-init
|
|
||||||
安装 `qemu-guest-agent`、单节点 k3s，并记录 `/dev/kvm` 检查结果。
|
|
||||||
|
|
||||||
## Kata Containers
|
|
||||||
|
|
||||||
Ansible 使用 Kata Containers 4.1.0 官方 `kata-deploy` chart。配置明确选择 k3s，
|
|
||||||
只安装 `qemu-runtime-rs`，不安装额外 snapshotter，并采用 `job` 模式，安装结束后
|
|
||||||
不保留 privileged DaemonSet。默认 `RuntimeClass` 名为 `kata`。
|
|
||||||
|
|
||||||
Terraform 输出的 DHCP 地址变化时，先更新 `ansible/inventory/hosts.yml`，然后运行：
|
|
||||||
|
|
||||||
```bash
|
|
||||||
cd infrastructure/kata-lab/ansible
|
|
||||||
ansible-playbook kata.yml
|
|
||||||
ansible-playbook verify.yml
|
|
||||||
```
|
|
||||||
|
|
||||||
`verify.yml` 首先创建一个 `runtimeClassName: kata` 的短生命周期 Pod，并确认 Pod 内
|
|
||||||
看到的 Kata guest kernel 与 k3s node 的宿主 kernel 不同。随后它在另一个 Kata Pod
|
|
||||||
内启动 privileged `dockerd` sidecar，让普通 Docker client 容器通过共享网络命名空间
|
|
||||||
的 `127.0.0.1:2375` 运行一个嵌套容器。这对应后续 Gitea Runner 的目标形态：runner
|
|
||||||
直接执行 job，只在需要 Docker API 时连接同 Pod 的 daemon，而不把 Docker executor
|
|
||||||
作为 job 的外层。Kata 下共享 volume 中的 Unix socket 文件虽然双方可见，但不能跨
|
|
||||||
容器连接，因此不能在这里沿用普通 Pod 常见的 `/var/run/docker.sock` 方案；loopback
|
|
||||||
端口没有通过 Service 暴露到 Pod 外。
|
|
||||||
验证 Pod 会保留以供排查；再次运行时会先替换它们。
|
|
||||||
|
|
||||||
2026-09-13 的首次验证结果：Kata guest kernel 为 `6.18.35`，node host kernel 为
|
|
||||||
`6.8.0-90-generic`；sidecar daemon 成功运行 `busybox:1.37` 嵌套容器。Docker 27.5.1
|
|
||||||
在当前 Kata guest 中无法挂载 overlay2，自动回退到 `vfs`；功能验证通过，但正式用于
|
|
||||||
CI 前需要解决或接受其镜像层复制与磁盘性能开销。
|
|
||||||
|
|
||||||
不要提交 `credentials.auto.tfvars`、Terraform state 或 saved plan。销毁前确认目标
|
|
||||||
仍然是 VMID 147 / `kata-lab`：
|
|
||||||
|
|
||||||
```bash
|
|
||||||
terraform plan -destroy
|
|
||||||
terraform destroy
|
|
||||||
```
|
|
||||||
@@ -1,10 +0,0 @@
|
|||||||
[defaults]
|
|
||||||
inventory = inventory/hosts.yml
|
|
||||||
host_key_checking = True
|
|
||||||
interpreter_python = auto_silent
|
|
||||||
retry_files_enabled = False
|
|
||||||
local_tmp = /tmp/ansible-kata-lab-local
|
|
||||||
remote_tmp = /tmp/ansible-kata-lab-remote
|
|
||||||
|
|
||||||
[ssh_connection]
|
|
||||||
pipelining = True
|
|
||||||
@@ -1,8 +0,0 @@
|
|||||||
---
|
|
||||||
all:
|
|
||||||
hosts:
|
|
||||||
kata-lab:
|
|
||||||
ansible_host: 192.168.10.13
|
|
||||||
ansible_user: ansible
|
|
||||||
vars:
|
|
||||||
ansible_become: true
|
|
||||||
@@ -1,88 +0,0 @@
|
|||||||
---
|
|
||||||
- name: Install Kata Containers in the disposable k3s lab
|
|
||||||
hosts: kata-lab
|
|
||||||
gather_facts: false
|
|
||||||
vars:
|
|
||||||
kata_version: "4.1.0"
|
|
||||||
kata_chart_url: >-
|
|
||||||
https://github.com/kata-containers/kata-containers/releases/download/{{ kata_version }}/kata-deploy-{{ kata_version }}.tgz
|
|
||||||
tasks:
|
|
||||||
- name: Check nested KVM is usable
|
|
||||||
ansible.builtin.stat:
|
|
||||||
path: /dev/kvm
|
|
||||||
register: kata_kvm
|
|
||||||
|
|
||||||
- name: Require nested KVM
|
|
||||||
ansible.builtin.assert:
|
|
||||||
that:
|
|
||||||
- kata_kvm.stat.exists
|
|
||||||
- kata_kvm.stat.ischr
|
|
||||||
fail_msg: /dev/kvm is unavailable; Kata QEMU cannot start efficiently
|
|
||||||
|
|
||||||
- name: Install Kata through the k3s Helm controller
|
|
||||||
ansible.builtin.copy:
|
|
||||||
dest: /var/lib/rancher/k3s/server/manifests/kata-deploy.yaml
|
|
||||||
owner: root
|
|
||||||
group: root
|
|
||||||
mode: "0644"
|
|
||||||
content: |
|
|
||||||
apiVersion: helm.cattle.io/v1
|
|
||||||
kind: HelmChart
|
|
||||||
metadata:
|
|
||||||
name: kata-deploy
|
|
||||||
namespace: kube-system
|
|
||||||
spec:
|
|
||||||
chart: {{ kata_chart_url }}
|
|
||||||
targetNamespace: kata-system
|
|
||||||
createNamespace: true
|
|
||||||
timeout: 15m
|
|
||||||
failurePolicy: abort
|
|
||||||
valuesContent: |-
|
|
||||||
deploymentMode: job
|
|
||||||
k8sDistribution: k3s
|
|
||||||
snapshotter:
|
|
||||||
setup: []
|
|
||||||
shims:
|
|
||||||
disableAll: true
|
|
||||||
qemu-runtime-rs:
|
|
||||||
enabled: true
|
|
||||||
runtimeClasses:
|
|
||||||
enabled: true
|
|
||||||
createDefault: true
|
|
||||||
node-feature-discovery:
|
|
||||||
enabled: false
|
|
||||||
|
|
||||||
- name: Wait for the Helm installation job to appear
|
|
||||||
ansible.builtin.command:
|
|
||||||
cmd: k3s kubectl get job/helm-install-kata-deploy -n kube-system
|
|
||||||
register: kata_helm_job
|
|
||||||
retries: 60
|
|
||||||
delay: 2
|
|
||||||
until: kata_helm_job.rc == 0
|
|
||||||
changed_when: false
|
|
||||||
|
|
||||||
- name: Wait for the Helm installation job to finish
|
|
||||||
ansible.builtin.command:
|
|
||||||
cmd: >-
|
|
||||||
k3s kubectl wait --for=condition=complete
|
|
||||||
job/helm-install-kata-deploy -n kube-system --timeout=20m
|
|
||||||
changed_when: false
|
|
||||||
|
|
||||||
- name: Wait for Kata node installation jobs
|
|
||||||
ansible.builtin.shell:
|
|
||||||
cmd: |
|
|
||||||
set -euo pipefail
|
|
||||||
jobs=$(k3s kubectl get jobs -n kata-system -l app.kubernetes.io/instance=kata-deploy -o name)
|
|
||||||
test -n "${jobs}"
|
|
||||||
k3s kubectl wait --for=condition=complete -n kata-system $jobs --timeout=20m
|
|
||||||
executable: /bin/bash
|
|
||||||
changed_when: false
|
|
||||||
|
|
||||||
- name: Wait for the default Kata RuntimeClass
|
|
||||||
ansible.builtin.command:
|
|
||||||
cmd: k3s kubectl get runtimeclass kata
|
|
||||||
register: kata_runtime_class
|
|
||||||
retries: 30
|
|
||||||
delay: 2
|
|
||||||
until: kata_runtime_class.rc == 0
|
|
||||||
changed_when: false
|
|
||||||
@@ -1,125 +0,0 @@
|
|||||||
---
|
|
||||||
- name: Verify Kata isolation with a disposable Pod
|
|
||||||
hosts: kata-lab
|
|
||||||
gather_facts: false
|
|
||||||
tasks:
|
|
||||||
- name: Remove an earlier smoke Pod
|
|
||||||
ansible.builtin.command:
|
|
||||||
cmd: k3s kubectl delete pod kata-smoke --ignore-not-found --wait=true
|
|
||||||
changed_when: false
|
|
||||||
|
|
||||||
- name: Create the Kata smoke Pod
|
|
||||||
ansible.builtin.shell:
|
|
||||||
cmd: |
|
|
||||||
set -o pipefail
|
|
||||||
k3s kubectl apply -f - <<'EOF'
|
|
||||||
apiVersion: v1
|
|
||||||
kind: Pod
|
|
||||||
metadata:
|
|
||||||
name: kata-smoke
|
|
||||||
spec:
|
|
||||||
runtimeClassName: kata
|
|
||||||
restartPolicy: Never
|
|
||||||
containers:
|
|
||||||
- name: smoke
|
|
||||||
image: docker.io/library/busybox:1.37
|
|
||||||
command:
|
|
||||||
- sh
|
|
||||||
- -c
|
|
||||||
- 'printf "guest-kernel="; uname -r; test -c /dev/kvm && exit 1 || true'
|
|
||||||
EOF
|
|
||||||
executable: /bin/bash
|
|
||||||
changed_when: true
|
|
||||||
|
|
||||||
- name: Wait for the Kata Pod to finish
|
|
||||||
ansible.builtin.command:
|
|
||||||
cmd: k3s kubectl wait --for=jsonpath='{.status.phase}'=Succeeded pod/kata-smoke --timeout=10m
|
|
||||||
changed_when: false
|
|
||||||
|
|
||||||
- name: Read the guest kernel version
|
|
||||||
ansible.builtin.command:
|
|
||||||
cmd: k3s kubectl logs kata-smoke
|
|
||||||
register: kata_smoke_log
|
|
||||||
changed_when: false
|
|
||||||
|
|
||||||
- name: Read the host kernel version
|
|
||||||
ansible.builtin.command:
|
|
||||||
cmd: uname -r
|
|
||||||
register: kata_host_kernel
|
|
||||||
changed_when: false
|
|
||||||
|
|
||||||
- name: Require a distinct guest kernel
|
|
||||||
ansible.builtin.assert:
|
|
||||||
that:
|
|
||||||
- kata_smoke_log.stdout is match('^guest-kernel=.+')
|
|
||||||
- kata_smoke_log.stdout | regex_replace('^guest-kernel=', '') != kata_host_kernel.stdout
|
|
||||||
success_msg: >-
|
|
||||||
Kata guest {{ kata_smoke_log.stdout }} differs from host-kernel={{ kata_host_kernel.stdout }}
|
|
||||||
|
|
||||||
- name: Remove an earlier Docker sidecar smoke Pod
|
|
||||||
ansible.builtin.command:
|
|
||||||
cmd: k3s kubectl delete pod kata-docker-smoke --ignore-not-found --wait=true
|
|
||||||
changed_when: false
|
|
||||||
|
|
||||||
- name: Create a Kata Pod with a Docker daemon sidecar
|
|
||||||
ansible.builtin.shell:
|
|
||||||
cmd: |
|
|
||||||
set -o pipefail
|
|
||||||
k3s kubectl apply -f - <<'EOF'
|
|
||||||
apiVersion: v1
|
|
||||||
kind: Pod
|
|
||||||
metadata:
|
|
||||||
name: kata-docker-smoke
|
|
||||||
spec:
|
|
||||||
runtimeClassName: kata
|
|
||||||
restartPolicy: Never
|
|
||||||
volumes:
|
|
||||||
- name: docker-run
|
|
||||||
emptyDir: {}
|
|
||||||
containers:
|
|
||||||
- name: dockerd
|
|
||||||
image: docker.io/library/docker:27-dind
|
|
||||||
securityContext:
|
|
||||||
privileged: true
|
|
||||||
env:
|
|
||||||
- name: DOCKER_TLS_CERTDIR
|
|
||||||
value: ""
|
|
||||||
volumeMounts:
|
|
||||||
- name: docker-run
|
|
||||||
mountPath: /var/run
|
|
||||||
- name: client
|
|
||||||
image: docker.io/library/docker:27-cli
|
|
||||||
env:
|
|
||||||
- name: DOCKER_HOST
|
|
||||||
value: tcp://127.0.0.1:2375
|
|
||||||
volumeMounts:
|
|
||||||
- name: docker-run
|
|
||||||
mountPath: /var/run
|
|
||||||
command:
|
|
||||||
- sh
|
|
||||||
- -c
|
|
||||||
- |
|
|
||||||
until docker info >/dev/null 2>&1; do sleep 1; done
|
|
||||||
docker run --rm docker.io/library/busybox:1.37 echo nested-docker-ok
|
|
||||||
EOF
|
|
||||||
executable: /bin/bash
|
|
||||||
changed_when: true
|
|
||||||
|
|
||||||
- name: Wait for the Docker client to finish
|
|
||||||
ansible.builtin.command:
|
|
||||||
cmd: >-
|
|
||||||
k3s kubectl wait --for=jsonpath='{.status.containerStatuses[?(@.name=="client")].state.terminated.exitCode}'=0
|
|
||||||
pod/kata-docker-smoke --timeout=10m
|
|
||||||
changed_when: false
|
|
||||||
|
|
||||||
- name: Read the Docker client result
|
|
||||||
ansible.builtin.command:
|
|
||||||
cmd: k3s kubectl logs kata-docker-smoke -c client
|
|
||||||
register: kata_docker_smoke_log
|
|
||||||
changed_when: false
|
|
||||||
|
|
||||||
- name: Require Docker to run a nested container
|
|
||||||
ansible.builtin.assert:
|
|
||||||
that:
|
|
||||||
- "'nested-docker-ok' in kata_docker_smoke_log.stdout"
|
|
||||||
success_msg: Docker daemon sidecar completed a nested container inside the Kata VM
|
|
||||||
@@ -1,24 +0,0 @@
|
|||||||
# This file is maintained automatically by "terraform init".
|
|
||||||
# Manual edits may be lost in future updates.
|
|
||||||
|
|
||||||
provider "registry.terraform.io/bpg/proxmox" {
|
|
||||||
version = "0.111.1"
|
|
||||||
constraints = "0.111.1"
|
|
||||||
hashes = [
|
|
||||||
"h1:ML2D3UUZTM99yrll/EBXj7wBYMb8xmQgomqFNybEoxY=",
|
|
||||||
"zh:18fb7c31a08dde6bffa1a4d4a211e604d6d17eec7092fd59331b3db3c6f3742c",
|
|
||||||
"zh:1cd60761538289d4dd2a1086b3ae62a7b0bdd4b1a2f824e9a44e243413168dba",
|
|
||||||
"zh:2eb76f6fc8299b6820ff678c8252332cc3366e226b5ae2e61748fd2449c1ed92",
|
|
||||||
"zh:45e6f7ebd0bf48911d37060359a4f359b5743b3092e985295733990e406d0416",
|
|
||||||
"zh:4aa8ba912eae37975d2e983394d173e595ca34fc76b5bf220b37d0e99d76e98c",
|
|
||||||
"zh:58e0789923103a77d502a0a9fc3eb920625e8eb935ec2d4ac0d006aebd1d186c",
|
|
||||||
"zh:6df8aa85fb8865915537e946c19b02538ad188018a629759c213c6f03730f642",
|
|
||||||
"zh:6ed47bc00d0913a1d0880618fa1376115e9edab6b4a658c081061a7f0e4ca360",
|
|
||||||
"zh:c5b10ff4f33df7e4c29e8f1127d49845b561b37b57517e844fb0954d7923d65e",
|
|
||||||
"zh:d016510e14b738499f0db9d9b3aafe82fc6877fb4ab4e9f831fb68a8d70a1385",
|
|
||||||
"zh:d941f394069bbf24351b363da1c64383f487067aaee0a84f9b96476d4912e212",
|
|
||||||
"zh:ddf271dbc2632ae8ffa8de3972f243ee47d260cb2ac90aa784f2746d98e21a0f",
|
|
||||||
"zh:ed0caa3501c42f611b7e9622c9b1df69fd85dc25a3cd88d3076381829688cd62",
|
|
||||||
"zh:f26e0763dbe6a6b2195c94b44696f2110f7f55433dc142839be16b9697fa5597",
|
|
||||||
]
|
|
||||||
}
|
|
||||||
@@ -1,34 +0,0 @@
|
|||||||
#cloud-config
|
|
||||||
hostname: ${hostname}
|
|
||||||
manage_etc_hosts: true
|
|
||||||
timezone: Etc/UTC
|
|
||||||
|
|
||||||
users:
|
|
||||||
- default
|
|
||||||
- name: ansible
|
|
||||||
groups: [adm, sudo]
|
|
||||||
shell: /bin/bash
|
|
||||||
lock_passwd: true
|
|
||||||
sudo: ALL=(ALL) NOPASSWD:ALL
|
|
||||||
ssh_authorized_keys:
|
|
||||||
- ${ssh_public_key}
|
|
||||||
|
|
||||||
package_update: true
|
|
||||||
package_upgrade: false
|
|
||||||
packages:
|
|
||||||
- ca-certificates
|
|
||||||
- curl
|
|
||||||
- jq
|
|
||||||
- qemu-guest-agent
|
|
||||||
|
|
||||||
runcmd:
|
|
||||||
- [systemctl, enable, --now, qemu-guest-agent]
|
|
||||||
- [modprobe, kvm_amd]
|
|
||||||
- [sh, -c, 'test -c /dev/kvm && echo available > /var/lib/cloud/nested-kvm.status || echo unavailable > /var/lib/cloud/nested-kvm.status']
|
|
||||||
- [sh, -c, 'curl -sfL https://get.k3s.io -o /tmp/install-k3s.sh']
|
|
||||||
- [chmod, '0755', /tmp/install-k3s.sh]
|
|
||||||
- [sh, -c, 'INSTALL_K3S_VERSION="${k3s_version}" INSTALL_K3S_EXEC="server --disable=traefik --write-kubeconfig-mode=0644" /tmp/install-k3s.sh']
|
|
||||||
- [sh, -c, 'kubectl wait --for=condition=Ready node/${hostname} --timeout=180s']
|
|
||||||
- [touch, /var/lib/cloud/kata-lab-bootstrap.done]
|
|
||||||
|
|
||||||
final_message: "kata-lab bootstrap completed after $UPTIME seconds"
|
|
||||||
@@ -1,3 +0,0 @@
|
|||||||
# Copy this file to credentials.auto.tfvars and replace the placeholder.
|
|
||||||
# Format: user@realm!token-id=token-secret
|
|
||||||
proxmox_api_token = "REPLACE_ME"
|
|
||||||
@@ -1,86 +0,0 @@
|
|||||||
locals {
|
|
||||||
ssh_public_key = trimspace(file(pathexpand(var.ssh_public_key_file)))
|
|
||||||
cloud_init = templatefile("${path.module}/cloud-init.yaml.tftpl", {
|
|
||||||
hostname = var.vm_name
|
|
||||||
ssh_public_key = local.ssh_public_key
|
|
||||||
k3s_version = var.k3s_version
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
data "proxmox_file" "ubuntu_noble" {
|
|
||||||
content_type = "import"
|
|
||||||
datastore_id = var.snippet_datastore_id
|
|
||||||
node_name = var.node_name
|
|
||||||
file_name = "noble-server-cloudimg-amd64.qcow2"
|
|
||||||
}
|
|
||||||
|
|
||||||
resource "proxmox_virtual_environment_file" "cloud_init" {
|
|
||||||
content_type = "snippets"
|
|
||||||
datastore_id = var.snippet_datastore_id
|
|
||||||
node_name = var.node_name
|
|
||||||
|
|
||||||
source_raw {
|
|
||||||
data = local.cloud_init
|
|
||||||
file_name = "${var.vm_name}-cloud-init.yaml"
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
resource "proxmox_virtual_environment_vm" "kata_lab" {
|
|
||||||
name = var.vm_name
|
|
||||||
description = "Disposable single-node k3s and Kata Containers validation environment."
|
|
||||||
tags = ["terraform", "disposable", "kata"]
|
|
||||||
|
|
||||||
node_name = var.node_name
|
|
||||||
vm_id = var.vm_id
|
|
||||||
|
|
||||||
started = true
|
|
||||||
on_boot = false
|
|
||||||
stop_on_destroy = true
|
|
||||||
|
|
||||||
agent {
|
|
||||||
enabled = true
|
|
||||||
timeout = "15m"
|
|
||||||
}
|
|
||||||
|
|
||||||
cpu {
|
|
||||||
cores = var.vm_cores
|
|
||||||
type = "host"
|
|
||||||
}
|
|
||||||
|
|
||||||
memory {
|
|
||||||
dedicated = var.vm_memory_mb
|
|
||||||
}
|
|
||||||
|
|
||||||
operating_system {
|
|
||||||
type = "l26"
|
|
||||||
}
|
|
||||||
|
|
||||||
scsi_hardware = "virtio-scsi-single"
|
|
||||||
|
|
||||||
disk {
|
|
||||||
datastore_id = var.disk_datastore_id
|
|
||||||
import_from = data.proxmox_file.ubuntu_noble.id
|
|
||||||
interface = "scsi0"
|
|
||||||
iothread = true
|
|
||||||
discard = "on"
|
|
||||||
size = var.vm_disk_gb
|
|
||||||
}
|
|
||||||
|
|
||||||
initialization {
|
|
||||||
datastore_id = var.disk_datastore_id
|
|
||||||
user_data_file_id = proxmox_virtual_environment_file.cloud_init.id
|
|
||||||
|
|
||||||
ip_config {
|
|
||||||
ipv4 {
|
|
||||||
address = "dhcp"
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
network_device {
|
|
||||||
bridge = var.network_bridge
|
|
||||||
model = "virtio"
|
|
||||||
}
|
|
||||||
|
|
||||||
serial_device {}
|
|
||||||
}
|
|
||||||
@@ -1,12 +0,0 @@
|
|||||||
output "vm_id" {
|
|
||||||
value = proxmox_virtual_environment_vm.kata_lab.vm_id
|
|
||||||
}
|
|
||||||
|
|
||||||
output "vm_ipv4_addresses" {
|
|
||||||
description = "QEMU guest agent 报告的地址；忽略 loopback 和 CNI 地址后再用于 SSH。"
|
|
||||||
value = proxmox_virtual_environment_vm.kata_lab.ipv4_addresses
|
|
||||||
}
|
|
||||||
|
|
||||||
output "ssh_user" {
|
|
||||||
value = "ansible"
|
|
||||||
}
|
|
||||||
@@ -1,17 +0,0 @@
|
|||||||
provider "proxmox" {
|
|
||||||
endpoint = var.proxmox_endpoint
|
|
||||||
api_token = var.proxmox_api_token
|
|
||||||
|
|
||||||
# Snippet uploads use SSH. The provider deliberately does not read
|
|
||||||
# ~/.ssh/config, so map the PVE node explicitly and use the current agent.
|
|
||||||
ssh {
|
|
||||||
agent = false
|
|
||||||
username = "root"
|
|
||||||
private_key = file(pathexpand(var.proxmox_ssh_private_key_file))
|
|
||||||
|
|
||||||
node {
|
|
||||||
name = "pve2"
|
|
||||||
address = "192.168.10.7"
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,83 +0,0 @@
|
|||||||
variable "proxmox_endpoint" {
|
|
||||||
description = "Proxmox VE API endpoint，不包含 /api2/json。"
|
|
||||||
type = string
|
|
||||||
default = "https://pve1.ad.ddupan.top:8006/"
|
|
||||||
}
|
|
||||||
|
|
||||||
variable "proxmox_api_token" {
|
|
||||||
description = "Proxmox API token，格式为 user@realm!token-id=secret。"
|
|
||||||
type = string
|
|
||||||
sensitive = true
|
|
||||||
}
|
|
||||||
|
|
||||||
variable "node_name" {
|
|
||||||
description = "承载 disposable kata-lab VM 的 PVE 节点。"
|
|
||||||
type = string
|
|
||||||
default = "pve2"
|
|
||||||
}
|
|
||||||
|
|
||||||
variable "vm_id" {
|
|
||||||
description = "kata-lab VMID。"
|
|
||||||
type = number
|
|
||||||
default = 147
|
|
||||||
}
|
|
||||||
|
|
||||||
variable "vm_name" {
|
|
||||||
description = "kata-lab VM 名称和 guest hostname。"
|
|
||||||
type = string
|
|
||||||
default = "kata-lab"
|
|
||||||
}
|
|
||||||
|
|
||||||
variable "vm_memory_mb" {
|
|
||||||
description = "VM 固定内存；pve2 只有约 7.4 GiB 可见内存。"
|
|
||||||
type = number
|
|
||||||
default = 4096
|
|
||||||
}
|
|
||||||
|
|
||||||
variable "vm_cores" {
|
|
||||||
description = "VM vCPU 数量。"
|
|
||||||
type = number
|
|
||||||
default = 4
|
|
||||||
}
|
|
||||||
|
|
||||||
variable "vm_disk_gb" {
|
|
||||||
description = "VM root disk 大小。"
|
|
||||||
type = number
|
|
||||||
default = 41
|
|
||||||
}
|
|
||||||
|
|
||||||
variable "disk_datastore_id" {
|
|
||||||
description = "VM root disk 所在 datastore。"
|
|
||||||
type = string
|
|
||||||
default = "pve-rg-hdd"
|
|
||||||
}
|
|
||||||
|
|
||||||
variable "snippet_datastore_id" {
|
|
||||||
description = "启用 snippets 的共享 datastore。"
|
|
||||||
type = string
|
|
||||||
default = "laptop"
|
|
||||||
}
|
|
||||||
|
|
||||||
variable "network_bridge" {
|
|
||||||
description = "连接 kata-lab VM 的 PVE bridge。"
|
|
||||||
type = string
|
|
||||||
default = "vmbr0"
|
|
||||||
}
|
|
||||||
|
|
||||||
variable "ssh_public_key_file" {
|
|
||||||
description = "注入 cloud-init 用户的 SSH 公钥路径。"
|
|
||||||
type = string
|
|
||||||
default = "~/.ssh/id_ed25519.pub"
|
|
||||||
}
|
|
||||||
|
|
||||||
variable "proxmox_ssh_private_key_file" {
|
|
||||||
description = "provider 上传 snippet 时登录 PVE 节点使用的私钥路径。"
|
|
||||||
type = string
|
|
||||||
default = "~/.ssh/id_ed25519"
|
|
||||||
}
|
|
||||||
|
|
||||||
variable "k3s_version" {
|
|
||||||
description = "可选的固定 k3s 版本；空值使用 stable channel。"
|
|
||||||
type = string
|
|
||||||
default = ""
|
|
||||||
}
|
|
||||||
@@ -1,10 +0,0 @@
|
|||||||
terraform {
|
|
||||||
required_version = ">= 1.10.0"
|
|
||||||
|
|
||||||
required_providers {
|
|
||||||
proxmox = {
|
|
||||||
source = "bpg/proxmox"
|
|
||||||
version = "0.111.1"
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,180 +0,0 @@
|
|||||||
# kata-lxc-lab
|
|
||||||
|
|
||||||
在 `pve2` 上验证 privileged LXC 内运行 k3s、Kata/QEMU，并直接使用 PVE host 的
|
|
||||||
`/dev/kvm`、`/dev/vhost-net` 与 `/dev/vhost-vsock`。这是可销毁 PoC，不承载持久数据。
|
|
||||||
|
|
||||||
Terraform 复用原 VM PoC 的 ignored credential 文件，不复制 token：
|
|
||||||
|
|
||||||
```bash
|
|
||||||
cd infrastructure/kata-lxc-lab/terraform
|
|
||||||
export SSL_CERT_FILE=/tmp/kata-lab-ddupan-ca.pem
|
|
||||||
terraform init
|
|
||||||
terraform plan \
|
|
||||||
-var-file=../../kata-lab/terraform/credentials.auto.tfvars
|
|
||||||
terraform apply \
|
|
||||||
-var-file=../../kata-lab/terraform/credentials.auto.tfvars
|
|
||||||
```
|
|
||||||
|
|
||||||
当前 PoC 使用 `pve2`、VMID 147、Ubuntu 24.04 LXC template 和 16 GiB
|
|
||||||
`local-lvm` rootfs。容器为 privileged，并开启 nesting、keyctl、FUSE、mknod；
|
|
||||||
Kata 所需的 KVM、vhost 与 `/dev/net/tun` 设备显式透传；另透传 `/dev/kmsg`，因为
|
|
||||||
kubelet 启动时会打开该设备。
|
|
||||||
PVE 9 的 `force_rw_sys=1` feature 用于开放 kubelet 所需的少量 `/proc/sys` 写操作；
|
|
||||||
provider 0.111.1 尚未暴露该 feature，因此同样由 `pct` 设置。
|
|
||||||
|
|
||||||
k3s/Kata worker 还使用以下 privileged LXC 原生配置；PVE 9 使用 cgroup v2，因此设备
|
|
||||||
规则采用 `lxc.cgroup2.devices.allow`：
|
|
||||||
|
|
||||||
```text
|
|
||||||
lxc.apparmor.profile: unconfined
|
|
||||||
lxc.cgroup2.devices.allow: a
|
|
||||||
lxc.cap.drop:
|
|
||||||
lxc.mount.auto: proc:rw sys:rw
|
|
||||||
```
|
|
||||||
|
|
||||||
这些设置让容器内 kubelet、containerd 和 Kata shim 能管理 mount、cgroup、设备与
|
|
||||||
共享宿主 sysctl。因此外层 LXC 不是安全边界；不可信 CI 的安全边界是内层 Kata VM。
|
|
||||||
`/dev/kmsg` 当前直接透传，没有使用 `/dev/console` symlink fallback。
|
|
||||||
|
|
||||||
PVE 将 privileged LXC 的创建限制为 `root@pam`（API token 即使拥有 `PVEAdmin` 也
|
|
||||||
缺少 root-only 的 `Sys.Modify`）。因此实际创建入口使用 root SSH 上的 Ansible/pct：
|
|
||||||
|
|
||||||
```bash
|
|
||||||
cd infrastructure/kata-lxc-lab/ansible
|
|
||||||
ansible-playbook create.yml
|
|
||||||
ansible-playbook bootstrap.yml
|
|
||||||
ansible-playbook kata.yml
|
|
||||||
ansible-playbook verify.yml
|
|
||||||
ansible-playbook verify-fuse.yml
|
|
||||||
```
|
|
||||||
|
|
||||||
Terraform 文件保留用于记录 provider 权限边界与声明式目标；当前 token 执行 apply 会
|
|
||||||
在创建前返回 403，不会产生资源或 state。
|
|
||||||
|
|
||||||
## 验证结果
|
|
||||||
|
|
||||||
当前管理地址为 `192.168.10.128`：2026-09-14 主 LAN DHCP 池调整后，从 `.11`
|
|
||||||
续租迁移到 `.128`，并同步 Ansible inventory。它仍是动态租约，不是固定地址。
|
|
||||||
|
|
||||||
2026-09-14，pve2 VMID 147 / `192.168.10.11` 验证通过：
|
|
||||||
|
|
||||||
- LXC 内 k3s `v1.36.4+k3s1` node Ready；
|
|
||||||
- Kata Containers 4.1.0 `qemu-runtime-rs` 安装成功；
|
|
||||||
- Kata guest kernel `6.18.35`，外层共享的 PVE kernel `7.0.2-6-pve`；
|
|
||||||
- Kata Pod 内的 Docker 27.5.1 daemon 成功运行 `busybox:1.37` 嵌套容器；
|
|
||||||
- `/dev/net/tun` 是 Kata 创建 TAP/link 的必要设备，未透传时 runtime-rs 在
|
|
||||||
`create link` 返回 `ENOENT`；
|
|
||||||
- `fuse-overlayfs` smoke test 成功：Docker 报告
|
|
||||||
`storage-driver=fuse-overlayfs`，并成功运行嵌套容器。
|
|
||||||
|
|
||||||
正式方案保持 Kata 默认 `emptydir_mode = "shared-fs"`，不向 LXC 暴露 host
|
|
||||||
`/dev/loop-control` 或 `/dev/loopN`。dockerd 镜像需要包含 `fuse-overlayfs`；启动前在
|
|
||||||
Kata guest 内创建临时设备节点 `mknod /dev/fuse c 10 229`，随后强制传入
|
|
||||||
`--storage-driver=fuse-overlayfs`。该 `/dev/fuse` 属于内层 guest kernel，不是 PVE
|
|
||||||
host 设备透传，并随 Kata Pod 一起销毁。
|
|
||||||
|
|
||||||
删除三个 smoke Pod 后，LXC cgroup 统计约 505 MB anonymous memory、1.34 GB file
|
|
||||||
cache；后者主要来自刚拉取的 Kata/Docker images，可由宿主回收。也就是说常驻 worker
|
|
||||||
的不可回收 working set 约 500 MB，8 GiB 配置是 cgroup 上限而非预分配。
|
|
||||||
|
|
||||||
## Gitea ephemeral runner PoC
|
|
||||||
|
|
||||||
`runner/` 定义一次性 Gitea runner Pod。整个 Pod 使用 `runtimeClassName: kata`；
|
|
||||||
runner 通过 guest 内同 Pod 的 dockerd 创建 job container，不连接 LXC/PVE host 的
|
|
||||||
Docker socket。dockerd 使用 zot 中预先发布的 `dind-fuse:29.7.1`，其
|
|
||||||
`/var/lib/docker` 是随 Kata Pod 删除的 `emptyDir`。
|
|
||||||
|
|
||||||
runner 以 `GITEA_RUNNER_EPHEMERAL=1` 注册，只接收一个 `runs-on: kata-poc` job；接单
|
|
||||||
后 Gitea 撤销其 runner credential，job 完成后进程与 Kata VM 一起退出。当前 PoC
|
|
||||||
手动创建一个 runner 等待 `.gitea/workflows/kata-runner-smoke.yml`，尚未部署
|
|
||||||
`workflow_job` webhook controller，因此不会保留常驻空闲 runner。
|
|
||||||
|
|
||||||
注册 token 只在运行时从现有 OpenBao/ESO 消费副本投递到 PoC 集群的临时 Secret，
|
|
||||||
不写入此目录。两个 Pod image 使用 `imagePullPolicy: Never`，部署前从可信工作站将
|
|
||||||
已验证镜像导入 PoC k3s containerd；这是当前 zot 拉取仍要求 SPIFFE 身份时的
|
|
||||||
bootstrap 边界，不是正式镜像分发方案。
|
|
||||||
|
|
||||||
正式接入 zot 前需要把这个 cluster 注册到 SPIRE，并为 runner Pod 创建专用
|
|
||||||
`ClusterSPIFFEID`。届时 runner 与 dockerd sidecar 在同一路径只读挂载 CSI Workload
|
|
||||||
API socket，runner 的 `container.options` 再把该路径 bind-mount 到 job container。
|
|
||||||
zot 只对最终 SPIFFE ID 的 `panxiao81/dind-fuse` 仓库授予
|
|
||||||
`read/create/update`；禁止授予 namespace 或整个 trust domain 写权限。
|
|
||||||
|
|
||||||
2026-09-14 已完成真实 Gitea job 验收：Actions run `242` / job `445` 使用
|
|
||||||
`alpine:3.22`，10 秒成功；runner container 退出码为 `0`，原生 dockerd sidecar 随后
|
|
||||||
由 kubelet 终止，Pod 最终为 `Succeeded`，没有保留空闲 Kata VM。guest 内核为
|
|
||||||
`6.18.35`，dockerd `29.7.1` 使用 `fuse-overlayfs`。
|
|
||||||
|
|
||||||
首次尝试暴露了两个部署陷阱：`COPY` 默认保留源文件 mode，入口脚本原为 `0664`，
|
|
||||||
必须在 Dockerfile 使用 `COPY --chmod=0755`；只配置 `ephemeral-storage: 20Gi` limit
|
|
||||||
会形成同值 request，使 12 GiB PoC 节点无法调度。当前 manifest 显式 request 1 GiB、
|
|
||||||
limit 8 GiB。完整 `ubuntu-latest` job image 也不适合这个小 rootfs，基础生命周期验收
|
|
||||||
改用 Alpine；正式 worker 必须扩容本地 image/cache 空间或使用更精简的 job image。
|
|
||||||
|
|
||||||
## Cloud Hypervisor 内存记账验证
|
|
||||||
|
|
||||||
2026-09-17 在 pve2 的一次性 VMID 148 中安装 k3s v1.36.4+k3s1 与 Kata 4.1.0，
|
|
||||||
只启用 `clh-runtime-rs`，验证 Cloud Hypervisor 在 privileged LXC 内不会重现早期
|
|
||||||
microVM PoC 的 private memfd 双重记账问题。测试完成后已删除 VMID 148。
|
|
||||||
|
|
||||||
Cloud Hypervisor `/api/v1/vm.info` 报告：
|
|
||||||
|
|
||||||
```json
|
|
||||||
{"size":3254779904,"shared":true,"hugepages":false,"prefault":false,"thp":true}
|
|
||||||
```
|
|
||||||
|
|
||||||
Kata guest 在 memory-backed `emptyDir` 中写入 1 GiB 后：
|
|
||||||
|
|
||||||
- VMM `Pss_Shmem` 从约 247 MiB 增至约 1269 MiB；
|
|
||||||
- VMM `Pss_Anon` 写入前后均约 1.2 MiB；
|
|
||||||
- 外层 LXC `shmem` 增加约 1 GiB，`anon` 基本不变；
|
|
||||||
- LXC `memory.events` 的 `max`、`oom` 与 `oom_kill` 均为 0；
|
|
||||||
- 删除 Pod 后 VMM 退出，LXC `shmem` 回落到约 1.6 MiB。
|
|
||||||
|
|
||||||
因此 Kata 的 Cloud Hypervisor handler 已使用 shared memfd，等价于独立 launcher 的
|
|
||||||
`--memory shared=on` 修复；后续 OpenSandbox/Kata 方案无需为 guest RAM 预留近似双倍
|
|
||||||
的 LXC cgroup 内存。部署验收仍应检查 `vm.info.config.memory.shared=true`，防止上游
|
|
||||||
配置或 runtime handler 变化造成回归。
|
|
||||||
|
|
||||||
测试期间还发现 pve2 `local-lvm` thin pool 已达到 100%；临时 rootfs 必须放到
|
|
||||||
`pve-rg-hdd`，不能根据容器内部 `df` 的剩余空间判断 thin pool 是否可写。
|
|
||||||
|
|
||||||
## Block-backed emptyDir 与 CI 构建验证
|
|
||||||
|
|
||||||
2026-09-17 又在 pve2 的一次性 VMID 148 中验证 Kata 4.1.0
|
|
||||||
`clh-runtime-rs` 的以下 drop-in：
|
|
||||||
|
|
||||||
```toml
|
|
||||||
[runtime]
|
|
||||||
emptydir_mode = "block-plain"
|
|
||||||
```
|
|
||||||
|
|
||||||
结果如下：
|
|
||||||
|
|
||||||
- 普通 Kubernetes `emptyDir` 被创建为稀疏 `disk.img`，由 Cloud Hypervisor 直接作为
|
|
||||||
block device 热插拔；guest 内 `/var/lib/docker` 是 `/dev/vdb` 上的 ext4。
|
|
||||||
- Docker 29.1.5 成功强制使用原生 `overlay2`，不再需要 `fuse-overlayfs`；整个过程
|
|
||||||
没有使用 PVE host 或 LXC 的 loop device。
|
|
||||||
- `emptyDir.sizeLimit: 8Gi` 没有决定虚拟磁盘容量。由于 kubelet 所在文件系统约
|
|
||||||
12 GiB，guest 看到的 ext4 约 11.7 GiB；这与 Kata 已知的 block-backed emptyDir
|
|
||||||
限制一致。
|
|
||||||
- 包含 256 MiB payload、2000 个小文件和一个额外复制层的冷 BuildKit 构建耗时
|
|
||||||
46 秒；同一 LXC、同一 DRBD HDD 存储上的 runc 对照为 34 秒，Kata 约慢 35%。
|
|
||||||
- 在 overlay2 容器层内写入并 fsync 512 MiB：Kata 为 12 秒（40.5 MB/s），runc 为
|
|
||||||
10 秒（50.3 MB/s）；创建一万个小文件两者均约 2 秒。
|
|
||||||
- kind v0.27.0 / Kubernetes v1.32.2 首次启动失败是因为 Kata guest 内的 dockerd
|
|
||||||
容器没有 `/dev/kmsg`，不是 block storage 或 cgroup 故障。在 privileged dockerd
|
|
||||||
启动前执行 `mknod /dev/kmsg c 1 11` 后，kind 创建成功：总计 122 秒，进入等待阶段
|
|
||||||
后 23 秒 control-plane Ready，随后可正常删除。
|
|
||||||
- 测试时稀疏 backing file 逻辑大小约 12 GiB，构建和 kind 后实际占用约 1.8 GiB。
|
|
||||||
删除 Pod 后 backing file、Cloud Hypervisor 进程和 kind 容器全部消失；host/LXC
|
|
||||||
均无 loop device 残留，LXC 没有 OOM 事件。
|
|
||||||
|
|
||||||
因此 `block-plain` 能解决 virtio-fs 不能作为 overlayfs upperdir 的问题，并满足
|
|
||||||
BuildKit 与 kind 的功能要求。正式 CI RuntimeClass 必须使用独立的 block-backed
|
|
||||||
配置，并在 dockerd bootstrap 中创建 guest `/dev/kmsg`。它仍比同机 runc 构建慢约
|
|
||||||
20–35%，且 `emptyDir.sizeLimit` 不能可靠限制虚拟磁盘容量。PoC 曾据此建议为 kubelet
|
|
||||||
目录提供独立文件系统；正式 sandbox 集群最终没有采用该设计：独立 volume 增加了
|
|
||||||
DRBD 与 LXC 挂载生命周期复杂度，却没有隔离 containerd 等其他节点数据。正式节点
|
|
||||||
改用单一、容量明确的 rootfs，并以磁盘占用监控、Pod 删除后的 backing-file 回收检查
|
|
||||||
和实际构建基准作为上线门槛。
|
|
||||||
@@ -1,10 +0,0 @@
|
|||||||
[defaults]
|
|
||||||
inventory = inventory/hosts.yml
|
|
||||||
host_key_checking = True
|
|
||||||
interpreter_python = auto_silent
|
|
||||||
retry_files_enabled = False
|
|
||||||
local_tmp = /tmp/ansible-kata-lxc-local
|
|
||||||
remote_tmp = /tmp/ansible-kata-lxc-remote
|
|
||||||
|
|
||||||
[ssh_connection]
|
|
||||||
pipelining = True
|
|
||||||
@@ -1,58 +0,0 @@
|
|||||||
---
|
|
||||||
- name: Bootstrap k3s in the Kata LXC worker
|
|
||||||
hosts: pve2
|
|
||||||
gather_facts: false
|
|
||||||
vars:
|
|
||||||
kata_lxc_id: 147
|
|
||||||
tasks:
|
|
||||||
- name: Check base packages in LXC
|
|
||||||
ansible.builtin.command:
|
|
||||||
cmd: >-
|
|
||||||
pct exec {{ kata_lxc_id }} -- dpkg-query -W
|
|
||||||
ca-certificates curl jq openssh-server
|
|
||||||
register: kata_lxc_packages
|
|
||||||
changed_when: false
|
|
||||||
failed_when: false
|
|
||||||
|
|
||||||
- name: Install base packages and SSH server in LXC
|
|
||||||
ansible.builtin.command:
|
|
||||||
cmd: >-
|
|
||||||
pct exec {{ kata_lxc_id }} -- bash -lc
|
|
||||||
'apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y
|
|
||||||
ca-certificates curl jq openssh-server'
|
|
||||||
when: kata_lxc_packages.rc != 0
|
|
||||||
|
|
||||||
- name: Install k3s in LXC
|
|
||||||
ansible.builtin.shell:
|
|
||||||
cmd: |
|
|
||||||
set -euo pipefail
|
|
||||||
pct exec {{ kata_lxc_id }} -- bash -lc '
|
|
||||||
if ! command -v k3s >/dev/null; then
|
|
||||||
curl -sfL https://get.k3s.io -o /tmp/install-k3s.sh
|
|
||||||
chmod 0755 /tmp/install-k3s.sh
|
|
||||||
INSTALL_K3S_EXEC="server --disable=traefik --write-kubeconfig-mode=0644" /tmp/install-k3s.sh
|
|
||||||
fi
|
|
||||||
'
|
|
||||||
executable: /bin/bash
|
|
||||||
register: kata_lxc_k3s_install
|
|
||||||
changed_when: "'No change detected' not in kata_lxc_k3s_install.stdout"
|
|
||||||
|
|
||||||
- name: Wait for k3s node readiness
|
|
||||||
ansible.builtin.command:
|
|
||||||
cmd: >-
|
|
||||||
pct exec {{ kata_lxc_id }} -- /usr/local/bin/k3s kubectl wait
|
|
||||||
--for=condition=Ready node/kata-lxc-lab --timeout=180s
|
|
||||||
changed_when: false
|
|
||||||
|
|
||||||
- name: Check LXC virtualization and devices
|
|
||||||
ansible.builtin.command:
|
|
||||||
cmd: >-
|
|
||||||
pct exec {{ kata_lxc_id }} -- bash -lc
|
|
||||||
'systemd-detect-virt; ls -l /dev/kvm /dev/vhost-net /dev/vhost-vsock;
|
|
||||||
/usr/local/bin/k3s kubectl get node -o wide'
|
|
||||||
register: kata_lxc_ready
|
|
||||||
changed_when: false
|
|
||||||
|
|
||||||
- name: Report k3s readiness
|
|
||||||
ansible.builtin.debug:
|
|
||||||
var: kata_lxc_ready.stdout_lines
|
|
||||||
@@ -1,137 +0,0 @@
|
|||||||
---
|
|
||||||
- name: Create the disposable Kata LXC worker
|
|
||||||
hosts: pve2
|
|
||||||
gather_facts: false
|
|
||||||
vars:
|
|
||||||
kata_lxc_id: 147
|
|
||||||
kata_lxc_template: laptop:vztmpl/ubuntu-24.04-standard_24.04-2_amd64.tar.zst
|
|
||||||
tasks:
|
|
||||||
- name: Check whether the LXC already exists
|
|
||||||
ansible.builtin.stat:
|
|
||||||
path: /etc/pve/lxc/{{ kata_lxc_id }}.conf
|
|
||||||
register: kata_lxc_config
|
|
||||||
|
|
||||||
- name: Create privileged LXC with Kata host devices
|
|
||||||
ansible.builtin.command:
|
|
||||||
argv:
|
|
||||||
- pct
|
|
||||||
- create
|
|
||||||
- "{{ kata_lxc_id }}"
|
|
||||||
- "{{ kata_lxc_template }}"
|
|
||||||
- --hostname
|
|
||||||
- kata-lxc-lab
|
|
||||||
- --ostype
|
|
||||||
- ubuntu
|
|
||||||
- --rootfs
|
|
||||||
- local-lvm:12
|
|
||||||
- --cores
|
|
||||||
- "4"
|
|
||||||
- --memory
|
|
||||||
- "8192"
|
|
||||||
- --swap
|
|
||||||
- "0"
|
|
||||||
- --net0
|
|
||||||
- name=eth0,bridge=vmbr0,ip=dhcp
|
|
||||||
- --features
|
|
||||||
- nesting=1,keyctl=1,fuse=1,mknod=1,force_rw_sys=1
|
|
||||||
- --unprivileged
|
|
||||||
- "0"
|
|
||||||
- --onboot
|
|
||||||
- "0"
|
|
||||||
- --ssh-public-keys
|
|
||||||
- /root/.ssh/authorized_keys
|
|
||||||
- --dev0
|
|
||||||
- path=/dev/kvm,mode=0660
|
|
||||||
- --dev1
|
|
||||||
- path=/dev/vhost-net,mode=0660
|
|
||||||
- --dev2
|
|
||||||
- path=/dev/vhost-vsock,mode=0660
|
|
||||||
- --dev3
|
|
||||||
- path=/dev/kmsg,mode=0660
|
|
||||||
- --dev4
|
|
||||||
- path=/dev/net/tun,mode=0666
|
|
||||||
- --tags
|
|
||||||
- disposable;kata;lxc;ansible
|
|
||||||
when: not kata_lxc_config.stat.exists
|
|
||||||
|
|
||||||
- name: Read LXC status
|
|
||||||
ansible.builtin.command:
|
|
||||||
cmd: pct status {{ kata_lxc_id }}
|
|
||||||
register: kata_lxc_status
|
|
||||||
changed_when: false
|
|
||||||
|
|
||||||
- name: Read current LXC configuration
|
|
||||||
ansible.builtin.command:
|
|
||||||
cmd: pct config {{ kata_lxc_id }}
|
|
||||||
register: kata_lxc_current_config
|
|
||||||
changed_when: false
|
|
||||||
|
|
||||||
- name: Ensure kubelet kernel log device is present
|
|
||||||
ansible.builtin.command:
|
|
||||||
cmd: pct set {{ kata_lxc_id }} --dev3 path=/dev/kmsg,mode=0660
|
|
||||||
register: kata_lxc_kmsg
|
|
||||||
when: "'/dev/kmsg' not in kata_lxc_current_config.stdout"
|
|
||||||
|
|
||||||
- name: Enable writable sysctls required by kubelet
|
|
||||||
ansible.builtin.command:
|
|
||||||
cmd: >-
|
|
||||||
pct set {{ kata_lxc_id }} --features
|
|
||||||
nesting=1,keyctl=1,fuse=1,mknod=1,force_rw_sys=1
|
|
||||||
register: kata_lxc_rw_sys
|
|
||||||
when: "'force_rw_sys=1' not in kata_lxc_current_config.stdout"
|
|
||||||
|
|
||||||
- name: Ensure TUN device required by Kata networking is present
|
|
||||||
ansible.builtin.command:
|
|
||||||
cmd: pct set {{ kata_lxc_id }} --dev4 path=/dev/net/tun,mode=0666
|
|
||||||
register: kata_lxc_tun
|
|
||||||
when: "'/dev/net/tun' not in kata_lxc_current_config.stdout"
|
|
||||||
|
|
||||||
- name: Configure privileged nested-runtime LXC directives
|
|
||||||
ansible.builtin.lineinfile:
|
|
||||||
path: /etc/pve/lxc/{{ kata_lxc_id }}.conf
|
|
||||||
regexp: "^{{ item.key | regex_escape }}:"
|
|
||||||
line: "{{ item.key }}: {{ item.value }}"
|
|
||||||
loop:
|
|
||||||
- key: lxc.apparmor.profile
|
|
||||||
value: unconfined
|
|
||||||
- key: lxc.cgroup2.devices.allow
|
|
||||||
value: a
|
|
||||||
- key: lxc.cap.drop
|
|
||||||
value: ""
|
|
||||||
- key: lxc.mount.auto
|
|
||||||
value: proc:rw sys:rw
|
|
||||||
register: kata_lxc_raw_config
|
|
||||||
|
|
||||||
- name: Restart LXC after device configuration change
|
|
||||||
ansible.builtin.command:
|
|
||||||
cmd: pct reboot {{ kata_lxc_id }}
|
|
||||||
when: >-
|
|
||||||
kata_lxc_kmsg.changed or kata_lxc_rw_sys.changed or kata_lxc_tun.changed or
|
|
||||||
kata_lxc_raw_config.changed
|
|
||||||
|
|
||||||
- name: Start LXC
|
|
||||||
ansible.builtin.command:
|
|
||||||
cmd: pct start {{ kata_lxc_id }}
|
|
||||||
when: "'status: stopped' in kata_lxc_status.stdout"
|
|
||||||
|
|
||||||
- name: Wait for systemd in LXC
|
|
||||||
ansible.builtin.command:
|
|
||||||
cmd: pct exec {{ kata_lxc_id }} -- systemctl is-system-running --wait
|
|
||||||
register: kata_lxc_systemd
|
|
||||||
retries: 30
|
|
||||||
delay: 2
|
|
||||||
until: kata_lxc_systemd.rc in [0, 1]
|
|
||||||
changed_when: false
|
|
||||||
failed_when: kata_lxc_systemd.rc not in [0, 1]
|
|
||||||
|
|
||||||
- name: Show LXC address and Kata devices
|
|
||||||
ansible.builtin.command:
|
|
||||||
cmd: >-
|
|
||||||
pct exec {{ kata_lxc_id }} -- sh -c
|
|
||||||
'hostname -I; ls -l /dev/kvm /dev/vhost-net /dev/vhost-vsock /dev/net/tun'
|
|
||||||
register: kata_lxc_facts
|
|
||||||
changed_when: false
|
|
||||||
|
|
||||||
- name: Report LXC address and devices
|
|
||||||
ansible.builtin.debug:
|
|
||||||
var: kata_lxc_facts.stdout_lines
|
|
||||||
@@ -1,9 +0,0 @@
|
|||||||
---
|
|
||||||
all:
|
|
||||||
hosts:
|
|
||||||
pve2:
|
|
||||||
ansible_host: 192.168.10.7
|
|
||||||
ansible_user: root
|
|
||||||
kata-lab:
|
|
||||||
ansible_host: 192.168.10.128
|
|
||||||
ansible_user: root
|
|
||||||
@@ -1,3 +0,0 @@
|
|||||||
---
|
|
||||||
# Reuse the proven Kata 4.1.0 installation playbook against the LXC inventory.
|
|
||||||
- import_playbook: ../../kata-lab/ansible/kata.yml
|
|
||||||
@@ -1,73 +0,0 @@
|
|||||||
---
|
|
||||||
- name: Verify fuse-overlayfs for Docker inside Kata
|
|
||||||
hosts: kata-lab
|
|
||||||
gather_facts: false
|
|
||||||
tasks:
|
|
||||||
- name: Remove an earlier fuse-overlayfs smoke Pod
|
|
||||||
ansible.builtin.command:
|
|
||||||
cmd: k3s kubectl delete pod kata-fuse-smoke --ignore-not-found --wait=true
|
|
||||||
changed_when: false
|
|
||||||
|
|
||||||
- name: Create Kata Docker Pod using fuse-overlayfs
|
|
||||||
ansible.builtin.shell:
|
|
||||||
cmd: |
|
|
||||||
set -o pipefail
|
|
||||||
k3s kubectl apply -f - <<'EOF'
|
|
||||||
apiVersion: v1
|
|
||||||
kind: Pod
|
|
||||||
metadata:
|
|
||||||
name: kata-fuse-smoke
|
|
||||||
spec:
|
|
||||||
runtimeClassName: kata
|
|
||||||
restartPolicy: Never
|
|
||||||
containers:
|
|
||||||
- name: dockerd
|
|
||||||
image: docker.io/library/docker:27-dind
|
|
||||||
securityContext:
|
|
||||||
privileged: true
|
|
||||||
env:
|
|
||||||
- name: DOCKER_TLS_CERTDIR
|
|
||||||
value: ""
|
|
||||||
command:
|
|
||||||
- sh
|
|
||||||
- -c
|
|
||||||
- |
|
|
||||||
apk add --no-cache fuse-overlayfs
|
|
||||||
test -e /dev/fuse || mknod /dev/fuse c 10 229
|
|
||||||
chmod 0666 /dev/fuse
|
|
||||||
exec dockerd-entrypoint.sh --storage-driver=fuse-overlayfs
|
|
||||||
- name: client
|
|
||||||
image: docker.io/library/docker:27-cli
|
|
||||||
env:
|
|
||||||
- name: DOCKER_HOST
|
|
||||||
value: tcp://127.0.0.1:2375
|
|
||||||
command:
|
|
||||||
- sh
|
|
||||||
- -c
|
|
||||||
- |
|
|
||||||
until docker info >/dev/null 2>&1; do sleep 1; done
|
|
||||||
docker info --format 'storage-driver={{ "{{" }}.Driver{{ "}}" }}'
|
|
||||||
docker run --rm docker.io/library/busybox:1.37 echo fuse-nested-docker-ok
|
|
||||||
EOF
|
|
||||||
executable: /bin/bash
|
|
||||||
changed_when: true
|
|
||||||
|
|
||||||
- name: Wait for the Docker client to finish
|
|
||||||
ansible.builtin.command:
|
|
||||||
cmd: >-
|
|
||||||
k3s kubectl wait --for=jsonpath='{.status.containerStatuses[?(@.name=="client")].state.terminated.exitCode}'=0
|
|
||||||
pod/kata-fuse-smoke --timeout=10m
|
|
||||||
changed_when: false
|
|
||||||
|
|
||||||
- name: Read Docker client result
|
|
||||||
ansible.builtin.command:
|
|
||||||
cmd: k3s kubectl logs kata-fuse-smoke -c client
|
|
||||||
register: kata_fuse_smoke_log
|
|
||||||
changed_when: false
|
|
||||||
|
|
||||||
- name: Require fuse-overlayfs and nested Docker
|
|
||||||
ansible.builtin.assert:
|
|
||||||
that:
|
|
||||||
- "'storage-driver=fuse-overlayfs' in kata_fuse_smoke_log.stdout"
|
|
||||||
- "'fuse-nested-docker-ok' in kata_fuse_smoke_log.stdout"
|
|
||||||
success_msg: Docker used fuse-overlayfs and completed a nested container inside Kata
|
|
||||||
@@ -1,3 +0,0 @@
|
|||||||
---
|
|
||||||
# Reuse the guest-kernel and Docker sidecar smoke tests against the LXC worker.
|
|
||||||
- import_playbook: ../../kata-lab/ansible/verify.yml
|
|
||||||
@@ -1,35 +0,0 @@
|
|||||||
apiVersion: v1
|
|
||||||
kind: Namespace
|
|
||||||
metadata:
|
|
||||||
name: gitea-actions
|
|
||||||
labels:
|
|
||||||
pod-security.kubernetes.io/enforce: privileged
|
|
||||||
---
|
|
||||||
apiVersion: v1
|
|
||||||
kind: ConfigMap
|
|
||||||
metadata:
|
|
||||||
name: kata-ephemeral-runner-config
|
|
||||||
namespace: gitea-actions
|
|
||||||
data:
|
|
||||||
config.yaml: |
|
|
||||||
log:
|
|
||||||
level: info
|
|
||||||
runner:
|
|
||||||
file: /runner-state/.runner
|
|
||||||
capacity: 1
|
|
||||||
envs:
|
|
||||||
DOCKER_HOST: tcp://host.docker.internal:2375
|
|
||||||
timeout: 3h
|
|
||||||
shutdown_timeout: 5m
|
|
||||||
labels:
|
|
||||||
- kata-poc:docker://docker.io/library/alpine:3.22
|
|
||||||
cache:
|
|
||||||
enabled: false
|
|
||||||
container:
|
|
||||||
network: ""
|
|
||||||
options: --add-host=host.docker.internal:host-gateway
|
|
||||||
docker_host: tcp://127.0.0.1:2375
|
|
||||||
require_docker: true
|
|
||||||
docker_timeout: 5m
|
|
||||||
force_pull: true
|
|
||||||
bind_workdir: false
|
|
||||||
@@ -1,111 +0,0 @@
|
|||||||
apiVersion: v1
|
|
||||||
kind: Pod
|
|
||||||
metadata:
|
|
||||||
generateName: kata-ephemeral-runner-
|
|
||||||
namespace: gitea-actions
|
|
||||||
labels:
|
|
||||||
app.kubernetes.io/name: kata-ephemeral-runner
|
|
||||||
spec:
|
|
||||||
runtimeClassName: kata
|
|
||||||
restartPolicy: Never
|
|
||||||
terminationGracePeriodSeconds: 330
|
|
||||||
automountServiceAccountToken: false
|
|
||||||
initContainers:
|
|
||||||
# Native sidecar: startupProbe gates the runner, and kubelet terminates this
|
|
||||||
# container after the ephemeral runner exits so the Kata sandbox can end.
|
|
||||||
- name: docker
|
|
||||||
image: zot.ad.ddupan.top/panxiao81/dind-fuse:29.7.1
|
|
||||||
imagePullPolicy: Never
|
|
||||||
restartPolicy: Always
|
|
||||||
env:
|
|
||||||
- name: DOCKER_TLS_CERTDIR
|
|
||||||
value: ""
|
|
||||||
- name: DIND_LOOPBACK_SIZE
|
|
||||||
value: 6G
|
|
||||||
- name: DIND_LOOPBACK_FILE
|
|
||||||
value: /loopback/docker.img
|
|
||||||
args:
|
|
||||||
- --host=tcp://0.0.0.0:2375
|
|
||||||
- --tls=false
|
|
||||||
- --mtu=1450
|
|
||||||
- --default-network-opt=bridge=com.docker.network.driver.mtu=1450
|
|
||||||
securityContext:
|
|
||||||
privileged: true
|
|
||||||
resources:
|
|
||||||
requests:
|
|
||||||
cpu: 250m
|
|
||||||
memory: 512Mi
|
|
||||||
ephemeral-storage: 1Gi
|
|
||||||
limits:
|
|
||||||
cpu: "4"
|
|
||||||
memory: 6Gi
|
|
||||||
ephemeral-storage: 8Gi
|
|
||||||
volumeMounts:
|
|
||||||
- name: docker-data
|
|
||||||
mountPath: /var/lib/docker
|
|
||||||
- name: loopback-data
|
|
||||||
mountPath: /loopback
|
|
||||||
startupProbe:
|
|
||||||
tcpSocket:
|
|
||||||
port: 2375
|
|
||||||
periodSeconds: 2
|
|
||||||
failureThreshold: 150
|
|
||||||
readinessProbe:
|
|
||||||
tcpSocket:
|
|
||||||
port: 2375
|
|
||||||
periodSeconds: 5
|
|
||||||
containers:
|
|
||||||
- name: runner
|
|
||||||
image: docker.io/gitea/runner:2.3.0
|
|
||||||
imagePullPolicy: Never
|
|
||||||
env:
|
|
||||||
- name: CONFIG_FILE
|
|
||||||
value: /config/config.yaml
|
|
||||||
- name: DOCKER_HOST
|
|
||||||
value: tcp://127.0.0.1:2375
|
|
||||||
- name: GITEA_INSTANCE_URL
|
|
||||||
value: https://git.ddupan.top
|
|
||||||
- name: GITEA_RUNNER_EPHEMERAL
|
|
||||||
value: "1"
|
|
||||||
- name: GITEA_RUNNER_NAME
|
|
||||||
valueFrom:
|
|
||||||
fieldRef:
|
|
||||||
fieldPath: metadata.name
|
|
||||||
- name: GITEA_RUNNER_LABELS
|
|
||||||
value: kata-poc:docker://docker.io/library/alpine:3.22
|
|
||||||
- name: GITEA_RUNNER_REGISTRATION_TOKEN_FILE
|
|
||||||
value: /registration/token
|
|
||||||
resources:
|
|
||||||
requests:
|
|
||||||
cpu: 100m
|
|
||||||
memory: 128Mi
|
|
||||||
limits:
|
|
||||||
cpu: "1"
|
|
||||||
memory: 1Gi
|
|
||||||
volumeMounts:
|
|
||||||
- name: config
|
|
||||||
mountPath: /config
|
|
||||||
readOnly: true
|
|
||||||
- name: registration
|
|
||||||
mountPath: /registration
|
|
||||||
readOnly: true
|
|
||||||
- name: runner-state
|
|
||||||
mountPath: /runner-state
|
|
||||||
volumes:
|
|
||||||
- name: config
|
|
||||||
configMap:
|
|
||||||
name: kata-ephemeral-runner-config
|
|
||||||
- name: registration
|
|
||||||
secret:
|
|
||||||
secretName: gitea-runner-registration
|
|
||||||
- name: runner-state
|
|
||||||
emptyDir:
|
|
||||||
medium: Memory
|
|
||||||
sizeLimit: 16Mi
|
|
||||||
- name: docker-data
|
|
||||||
emptyDir:
|
|
||||||
sizeLimit: 8Gi
|
|
||||||
- name: loopback-data
|
|
||||||
emptyDir:
|
|
||||||
medium: Memory
|
|
||||||
sizeLimit: 6Gi
|
|
||||||
@@ -1,24 +0,0 @@
|
|||||||
# This file is maintained automatically by "terraform init".
|
|
||||||
# Manual edits may be lost in future updates.
|
|
||||||
|
|
||||||
provider "registry.terraform.io/bpg/proxmox" {
|
|
||||||
version = "0.111.1"
|
|
||||||
constraints = "0.111.1"
|
|
||||||
hashes = [
|
|
||||||
"h1:ML2D3UUZTM99yrll/EBXj7wBYMb8xmQgomqFNybEoxY=",
|
|
||||||
"zh:18fb7c31a08dde6bffa1a4d4a211e604d6d17eec7092fd59331b3db3c6f3742c",
|
|
||||||
"zh:1cd60761538289d4dd2a1086b3ae62a7b0bdd4b1a2f824e9a44e243413168dba",
|
|
||||||
"zh:2eb76f6fc8299b6820ff678c8252332cc3366e226b5ae2e61748fd2449c1ed92",
|
|
||||||
"zh:45e6f7ebd0bf48911d37060359a4f359b5743b3092e985295733990e406d0416",
|
|
||||||
"zh:4aa8ba912eae37975d2e983394d173e595ca34fc76b5bf220b37d0e99d76e98c",
|
|
||||||
"zh:58e0789923103a77d502a0a9fc3eb920625e8eb935ec2d4ac0d006aebd1d186c",
|
|
||||||
"zh:6df8aa85fb8865915537e946c19b02538ad188018a629759c213c6f03730f642",
|
|
||||||
"zh:6ed47bc00d0913a1d0880618fa1376115e9edab6b4a658c081061a7f0e4ca360",
|
|
||||||
"zh:c5b10ff4f33df7e4c29e8f1127d49845b561b37b57517e844fb0954d7923d65e",
|
|
||||||
"zh:d016510e14b738499f0db9d9b3aafe82fc6877fb4ab4e9f831fb68a8d70a1385",
|
|
||||||
"zh:d941f394069bbf24351b363da1c64383f487067aaee0a84f9b96476d4912e212",
|
|
||||||
"zh:ddf271dbc2632ae8ffa8de3972f243ee47d260cb2ac90aa784f2746d98e21a0f",
|
|
||||||
"zh:ed0caa3501c42f611b7e9622c9b1df69fd85dc25a3cd88d3076381829688cd62",
|
|
||||||
"zh:f26e0763dbe6a6b2195c94b44696f2110f7f55433dc142839be16b9697fa5597",
|
|
||||||
]
|
|
||||||
}
|
|
||||||
@@ -1,95 +0,0 @@
|
|||||||
locals {
|
|
||||||
ssh_public_key = trimspace(file(pathexpand(var.ssh_public_key_file)))
|
|
||||||
}
|
|
||||||
|
|
||||||
data "proxmox_file" "ubuntu_noble" {
|
|
||||||
content_type = "vztmpl"
|
|
||||||
datastore_id = var.template_datastore
|
|
||||||
node_name = var.proxmox_node
|
|
||||||
file_name = var.template_file_name
|
|
||||||
}
|
|
||||||
|
|
||||||
resource "proxmox_virtual_environment_container" "kata_lxc_lab" {
|
|
||||||
node_name = var.proxmox_node
|
|
||||||
vm_id = var.container_id
|
|
||||||
|
|
||||||
description = "Disposable privileged LXC for validating k3s and Kata with direct host KVM access."
|
|
||||||
unprivileged = false
|
|
||||||
started = true
|
|
||||||
start_on_boot = false
|
|
||||||
tags = ["disposable", "kata", "lxc", "terraform"]
|
|
||||||
|
|
||||||
cpu {
|
|
||||||
cores = 4
|
|
||||||
}
|
|
||||||
|
|
||||||
memory {
|
|
||||||
dedicated = 8192
|
|
||||||
swap = 0
|
|
||||||
}
|
|
||||||
|
|
||||||
disk {
|
|
||||||
datastore_id = var.root_datastore
|
|
||||||
size = 16
|
|
||||||
}
|
|
||||||
|
|
||||||
features {
|
|
||||||
fuse = true
|
|
||||||
keyctl = true
|
|
||||||
mknod = true
|
|
||||||
nesting = true
|
|
||||||
}
|
|
||||||
|
|
||||||
device_passthrough {
|
|
||||||
path = "/dev/kvm"
|
|
||||||
mode = "0660"
|
|
||||||
}
|
|
||||||
|
|
||||||
device_passthrough {
|
|
||||||
path = "/dev/vhost-net"
|
|
||||||
mode = "0660"
|
|
||||||
}
|
|
||||||
|
|
||||||
device_passthrough {
|
|
||||||
path = "/dev/vhost-vsock"
|
|
||||||
mode = "0660"
|
|
||||||
}
|
|
||||||
|
|
||||||
device_passthrough {
|
|
||||||
path = "/dev/kmsg"
|
|
||||||
mode = "0660"
|
|
||||||
}
|
|
||||||
|
|
||||||
device_passthrough {
|
|
||||||
path = "/dev/net/tun"
|
|
||||||
mode = "0666"
|
|
||||||
}
|
|
||||||
|
|
||||||
initialization {
|
|
||||||
hostname = var.container_name
|
|
||||||
|
|
||||||
ip_config {
|
|
||||||
ipv4 {
|
|
||||||
address = "dhcp"
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
user_account {
|
|
||||||
keys = [local.ssh_public_key]
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
network_interface {
|
|
||||||
name = "eth0"
|
|
||||||
bridge = "vmbr0"
|
|
||||||
}
|
|
||||||
|
|
||||||
operating_system {
|
|
||||||
template_file_id = data.proxmox_file.ubuntu_noble.id
|
|
||||||
type = "ubuntu"
|
|
||||||
}
|
|
||||||
|
|
||||||
wait_for_ip {
|
|
||||||
ipv4 = true
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,7 +0,0 @@
|
|||||||
output "container_id" {
|
|
||||||
value = proxmox_virtual_environment_container.kata_lxc_lab.vm_id
|
|
||||||
}
|
|
||||||
|
|
||||||
output "ipv4" {
|
|
||||||
value = proxmox_virtual_environment_container.kata_lxc_lab.ipv4
|
|
||||||
}
|
|
||||||
@@ -1,15 +0,0 @@
|
|||||||
provider "proxmox" {
|
|
||||||
endpoint = var.proxmox_endpoint
|
|
||||||
api_token = var.proxmox_api_token
|
|
||||||
|
|
||||||
ssh {
|
|
||||||
agent = false
|
|
||||||
username = "root"
|
|
||||||
private_key = file(pathexpand(var.proxmox_ssh_private_key_file))
|
|
||||||
|
|
||||||
node {
|
|
||||||
name = var.proxmox_node
|
|
||||||
address = var.proxmox_node_address
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,54 +0,0 @@
|
|||||||
variable "proxmox_endpoint" {
|
|
||||||
type = string
|
|
||||||
default = "https://pve1.ad.ddupan.top:8006"
|
|
||||||
}
|
|
||||||
|
|
||||||
variable "proxmox_api_token" {
|
|
||||||
type = string
|
|
||||||
sensitive = true
|
|
||||||
}
|
|
||||||
|
|
||||||
variable "proxmox_node" {
|
|
||||||
type = string
|
|
||||||
default = "pve2"
|
|
||||||
}
|
|
||||||
|
|
||||||
variable "proxmox_node_address" {
|
|
||||||
type = string
|
|
||||||
default = "192.168.10.7"
|
|
||||||
}
|
|
||||||
|
|
||||||
variable "proxmox_ssh_private_key_file" {
|
|
||||||
type = string
|
|
||||||
default = "~/.ssh/id_ed25519"
|
|
||||||
}
|
|
||||||
|
|
||||||
variable "ssh_public_key_file" {
|
|
||||||
type = string
|
|
||||||
default = "~/.ssh/id_ed25519.pub"
|
|
||||||
}
|
|
||||||
|
|
||||||
variable "container_id" {
|
|
||||||
type = number
|
|
||||||
default = 147
|
|
||||||
}
|
|
||||||
|
|
||||||
variable "container_name" {
|
|
||||||
type = string
|
|
||||||
default = "kata-lxc-lab"
|
|
||||||
}
|
|
||||||
|
|
||||||
variable "template_datastore" {
|
|
||||||
type = string
|
|
||||||
default = "laptop"
|
|
||||||
}
|
|
||||||
|
|
||||||
variable "template_file_name" {
|
|
||||||
type = string
|
|
||||||
default = "ubuntu-24.04-standard_24.04-2_amd64.tar.zst"
|
|
||||||
}
|
|
||||||
|
|
||||||
variable "root_datastore" {
|
|
||||||
type = string
|
|
||||||
default = "local-lvm"
|
|
||||||
}
|
|
||||||
@@ -1,10 +0,0 @@
|
|||||||
terraform {
|
|
||||||
required_version = ">= 1.10"
|
|
||||||
|
|
||||||
required_providers {
|
|
||||||
proxmox = {
|
|
||||||
source = "bpg/proxmox"
|
|
||||||
version = "0.111.1"
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,53 +0,0 @@
|
|||||||
# Gitea kind microVM runner
|
|
||||||
|
|
||||||
`kind-microvm` 是专门运行 kind / nested Kubernetes CI 的 runner label。每个 job
|
|
||||||
独占一台 Cloud Hypervisor VM；guest 内直接运行 Docker 与 ephemeral Gitea Runner，
|
|
||||||
不经过 Kata。VM 完成一个 job 后关机，临时 COW 磁盘随即删除。
|
|
||||||
|
|
||||||
集群内的 `controller.py` 消费 Gitea `workflow_job` webhook。仅当
|
|
||||||
`action=queued` 且 `workflow_job.labels` 包含 `kind-microvm` 时，向 NATS JetStream
|
|
||||||
的 `ci.runner.kind-microvm` subject 发布消息。`workflow_job.id` 写入
|
|
||||||
`Nats-Msg-Id`，重复 webhook 不会生成第二个任务。
|
|
||||||
|
|
||||||
pve2 LXC 内的 `worker.py` 使用 durable pull consumer 领取消息。领取后启动一台
|
|
||||||
microVM，运行期间每分钟发送 `InProgress`；VM/ephemeral runner 正常退出才 ACK，
|
|
||||||
启动失败则 NAK 并延迟重试。当前 `RUNNER_CAPACITY=1` 只是 pve2 的资源配置；以后可
|
|
||||||
提高单 worker capacity，或在其他宿主机增加共享同一 durable consumer 的 worker，
|
|
||||||
而无需修改 webhook/controller。相同 runner 类型必须共享 durable；WorkQueue stream
|
|
||||||
不允许多个 consumer 使用重叠的 subject filter。
|
|
||||||
|
|
||||||
worker 的凭据位于：
|
|
||||||
|
|
||||||
```text
|
|
||||||
/etc/microvm-runner/registration-token
|
|
||||||
/etc/microvm-runner/nats-password
|
|
||||||
```
|
|
||||||
|
|
||||||
二者必须为 root-only 文件，不能写入镜像、cloud-init seed 或仓库。worker 为每次
|
|
||||||
启动生成一次性 nonce；guest 只能从 TAP 网段请求一次 registration token，请求后
|
|
||||||
立即从 worker 内存删除。runner 注册成功后删除 guest 内的 token 文件，再启动
|
|
||||||
daemon。这样 job 无法从 seed disk 或 Docker inspect 取回可复用 registration token。
|
|
||||||
|
|
||||||
NATS stream 使用 `WorkQueuePolicy`，ACK 后立即删除消息；同时限制为 24 小时、
|
|
||||||
10000 条和 256 MiB，异常积压也不会无限增长。NATS 平台配置见
|
|
||||||
`../../platform/nats/`。
|
|
||||||
|
|
||||||
2026-09-16 PoC 已验证：Cloud Hypervisor v52.0、Ubuntu 24.04、4 vCPU、3 GiB RAM、
|
|
||||||
10 GiB COW disk 中，Docker 29.1.3 成功创建 kind v0.27.0 / Kubernetes v1.32.2
|
|
||||||
集群，并运行 `busybox:1.37` smoke Pod。冷启动约 20 秒。pve2 资源只允许一台：
|
|
||||||
VMM cgroup（guest RAM 与 page cache）峰值约 6.2 GiB。
|
|
||||||
|
|
||||||
正式启用前还需要安装 launcher、制作不含凭据的基础镜像、部署 controller，并在
|
|
||||||
Gitea 配置只发送 `workflow_job` 的 instance/organization webhook。workflow 使用：
|
|
||||||
|
|
||||||
```yaml
|
|
||||||
runs-on: kind-microvm
|
|
||||||
```
|
|
||||||
|
|
||||||
当前 pve2 配置：
|
|
||||||
|
|
||||||
```ini
|
|
||||||
RUNNER_CAPACITY=1
|
|
||||||
NATS_DURABLE=kind-microvm
|
|
||||||
RUNNER_MAX_INFLIGHT=64
|
|
||||||
```
|
|
||||||
@@ -1,113 +0,0 @@
|
|||||||
#!/usr/bin/env python3
|
|
||||||
"""Gitea workflow_job webhook to NATS JetStream producer."""
|
|
||||||
|
|
||||||
import hashlib
|
|
||||||
import hmac
|
|
||||||
import json
|
|
||||||
import os
|
|
||||||
import ssl
|
|
||||||
from pathlib import Path
|
|
||||||
|
|
||||||
import nats
|
|
||||||
from aiohttp import web
|
|
||||||
from nats.js.api import DiscardPolicy, RetentionPolicy, StorageType, StreamConfig
|
|
||||||
from nats.js.errors import NotFoundError
|
|
||||||
|
|
||||||
LABEL = os.environ.get("RUNNER_LABEL", "kind-microvm")
|
|
||||||
SUBJECT = os.environ.get("NATS_SUBJECT", f"ci.runner.{LABEL}")
|
|
||||||
STREAM = os.environ.get("NATS_STREAM", "CI_RUNNER")
|
|
||||||
NATS_URL = os.environ.get("NATS_URL", "tls://nats.nats.svc.cluster.local:4222")
|
|
||||||
NATS_USER = os.environ.get("NATS_USER", "ci-producer")
|
|
||||||
NATS_PASSWORD_FILE = Path(os.environ.get("NATS_PASSWORD_FILE", "/run/secrets/nats/password"))
|
|
||||||
NATS_CA_FILE = os.environ.get("NATS_CA_FILE", "/etc/ssl/certs/ca-certificates.crt")
|
|
||||||
WEBHOOK_SECRET_FILE = Path(os.environ.get("WEBHOOK_SECRET_FILE", "/run/secrets/gitea/webhook-secret"))
|
|
||||||
|
|
||||||
|
|
||||||
def accepts(payload: object) -> tuple[bool, str | None]:
|
|
||||||
if not isinstance(payload, dict) or payload.get("action") != "queued":
|
|
||||||
return False, None
|
|
||||||
job = payload.get("workflow_job")
|
|
||||||
if not isinstance(job, dict) or LABEL not in job.get("labels", []):
|
|
||||||
return False, None
|
|
||||||
job_id = job.get("id")
|
|
||||||
if not isinstance(job_id, int):
|
|
||||||
return False, None
|
|
||||||
return True, str(job_id)
|
|
||||||
|
|
||||||
|
|
||||||
def valid_signature(body: bytes, signature: str) -> bool:
|
|
||||||
expected = hmac.new(WEBHOOK_SECRET_FILE.read_bytes().strip(), body, hashlib.sha256).hexdigest()
|
|
||||||
return hmac.compare_digest(signature, expected)
|
|
||||||
|
|
||||||
|
|
||||||
async def ensure_stream(js: object) -> None:
|
|
||||||
config = StreamConfig(
|
|
||||||
name=STREAM,
|
|
||||||
subjects=["ci.runner.*"],
|
|
||||||
retention=RetentionPolicy.WORK_QUEUE,
|
|
||||||
storage=StorageType.FILE,
|
|
||||||
discard=DiscardPolicy.OLD,
|
|
||||||
max_age=24 * 60 * 60,
|
|
||||||
max_msgs=10_000,
|
|
||||||
max_bytes=256 * 1024 * 1024,
|
|
||||||
duplicate_window=24 * 60 * 60,
|
|
||||||
)
|
|
||||||
try:
|
|
||||||
await js.stream_info(STREAM)
|
|
||||||
except NotFoundError:
|
|
||||||
await js.add_stream(config=config)
|
|
||||||
else:
|
|
||||||
await js.update_stream(config=config)
|
|
||||||
|
|
||||||
|
|
||||||
async def webhook(request: web.Request) -> web.Response:
|
|
||||||
body = await request.read()
|
|
||||||
if not valid_signature(body, request.headers.get("X-Gitea-Signature", "")):
|
|
||||||
raise web.HTTPUnauthorized()
|
|
||||||
try:
|
|
||||||
payload = json.loads(body)
|
|
||||||
except json.JSONDecodeError as error:
|
|
||||||
raise web.HTTPBadRequest(text="invalid JSON\n") from error
|
|
||||||
accepted, job_id = accepts(payload)
|
|
||||||
if not accepted:
|
|
||||||
return web.Response(status=204)
|
|
||||||
await request.app["js"].publish(
|
|
||||||
SUBJECT,
|
|
||||||
body,
|
|
||||||
headers={"Nats-Msg-Id": f"gitea-workflow-job-{job_id}"},
|
|
||||||
)
|
|
||||||
return web.Response(status=202, text="queued\n")
|
|
||||||
|
|
||||||
|
|
||||||
async def health(request: web.Request) -> web.Response:
|
|
||||||
return web.Response(text="ok\n" if request.app["nc"].is_connected else "disconnected\n",
|
|
||||||
status=200 if request.app["nc"].is_connected else 503)
|
|
||||||
|
|
||||||
|
|
||||||
async def nats_context(app: web.Application):
|
|
||||||
tls = ssl.create_default_context(cafile=NATS_CA_FILE)
|
|
||||||
nc = await nats.connect(
|
|
||||||
NATS_URL,
|
|
||||||
user=NATS_USER,
|
|
||||||
password=NATS_PASSWORD_FILE.read_text().strip(),
|
|
||||||
tls=tls,
|
|
||||||
name="microvm-runner-controller",
|
|
||||||
)
|
|
||||||
app["nc"] = nc
|
|
||||||
app["js"] = nc.jetstream()
|
|
||||||
await ensure_stream(app["js"])
|
|
||||||
yield
|
|
||||||
await nc.drain()
|
|
||||||
|
|
||||||
|
|
||||||
def create_app() -> web.Application:
|
|
||||||
app = web.Application(client_max_size=1024 * 1024)
|
|
||||||
app.cleanup_ctx.append(nats_context)
|
|
||||||
app.router.add_post("/webhook", webhook)
|
|
||||||
app.router.add_get("/healthz", health)
|
|
||||||
return app
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
|
||||||
web.run_app(create_app(), host=os.environ.get("LISTEN", "0.0.0.0"),
|
|
||||||
port=int(os.environ.get("PORT", "8787")))
|
|
||||||
@@ -1,19 +0,0 @@
|
|||||||
[Unit]
|
|
||||||
Description=Gitea kind microVM runner worker
|
|
||||||
After=network-online.target
|
|
||||||
Wants=network-online.target
|
|
||||||
|
|
||||||
[Service]
|
|
||||||
Type=simple
|
|
||||||
EnvironmentFile=-/etc/microvm-runner/worker.env
|
|
||||||
ExecStart=/opt/microvm-runner/venv/bin/python /opt/microvm-runner/worker.py
|
|
||||||
Restart=on-failure
|
|
||||||
RestartSec=5s
|
|
||||||
SupplementaryGroups=kvm
|
|
||||||
PrivateTmp=yes
|
|
||||||
ProtectHome=yes
|
|
||||||
ProtectSystem=strict
|
|
||||||
ReadWritePaths=/var/lib/microvm-runner /run/microvm-runner
|
|
||||||
|
|
||||||
[Install]
|
|
||||||
WantedBy=multi-user.target
|
|
||||||
@@ -1,2 +0,0 @@
|
|||||||
aiohttp==3.12.15
|
|
||||||
nats-py==2.11.0
|
|
||||||
@@ -1,138 +0,0 @@
|
|||||||
#!/usr/bin/env python3
|
|
||||||
"""Capacity-bounded JetStream consumer that launches one ephemeral VM per job."""
|
|
||||||
|
|
||||||
import asyncio
|
|
||||||
import os
|
|
||||||
import secrets
|
|
||||||
import ssl
|
|
||||||
import uuid
|
|
||||||
from pathlib import Path
|
|
||||||
|
|
||||||
import nats
|
|
||||||
from aiohttp import web
|
|
||||||
from nats.errors import TimeoutError
|
|
||||||
from nats.js.api import AckPolicy, ConsumerConfig
|
|
||||||
|
|
||||||
CAPACITY = int(os.environ.get("RUNNER_CAPACITY", "1"))
|
|
||||||
SUBJECT = os.environ.get("NATS_SUBJECT", "ci.runner.kind-microvm")
|
|
||||||
STREAM = os.environ.get("NATS_STREAM", "CI_RUNNER")
|
|
||||||
DURABLE = os.environ.get("NATS_DURABLE", "kind-microvm")
|
|
||||||
MAX_INFLIGHT = int(os.environ.get("RUNNER_MAX_INFLIGHT", "64"))
|
|
||||||
NATS_URL = os.environ.get("NATS_URL", "tls://nats.ad.ddupan.top:4222")
|
|
||||||
NATS_USER = os.environ.get("NATS_USER", "ci-worker")
|
|
||||||
NATS_PASSWORD_FILE = Path(os.environ.get("NATS_PASSWORD_FILE", "/etc/microvm-runner/nats-password"))
|
|
||||||
NATS_CA_FILE = os.environ.get("NATS_CA_FILE", "/etc/ssl/certs/ca-certificates.crt")
|
|
||||||
REGISTRATION_TOKEN_FILE = Path(os.environ.get("REGISTRATION_TOKEN_FILE", "/etc/microvm-runner/registration-token"))
|
|
||||||
LAUNCHER = os.environ.get("LAUNCHER", "/usr/local/libexec/microvm-runner-launch")
|
|
||||||
TOKEN_LISTEN = os.environ.get("TOKEN_LISTEN", "172.30.0.1")
|
|
||||||
TOKEN_PORT = int(os.environ.get("TOKEN_PORT", "8787"))
|
|
||||||
|
|
||||||
tokens: dict[str, bytes] = {}
|
|
||||||
token_lock = asyncio.Lock()
|
|
||||||
|
|
||||||
|
|
||||||
async def token(request: web.Request) -> web.Response:
|
|
||||||
nonce = request.match_info["nonce"]
|
|
||||||
async with token_lock:
|
|
||||||
value = tokens.pop(nonce, None)
|
|
||||||
if value is None:
|
|
||||||
raise web.HTTPNotFound()
|
|
||||||
return web.Response(body=value, headers={"Cache-Control": "no-store"})
|
|
||||||
|
|
||||||
|
|
||||||
async def heartbeat(message: object, stop: asyncio.Event) -> None:
|
|
||||||
while True:
|
|
||||||
try:
|
|
||||||
await asyncio.wait_for(stop.wait(), timeout=60)
|
|
||||||
return
|
|
||||||
except asyncio.TimeoutError:
|
|
||||||
await message.in_progress()
|
|
||||||
|
|
||||||
|
|
||||||
async def run_one(message: object) -> None:
|
|
||||||
instance_id = str(uuid.uuid4())
|
|
||||||
nonce = secrets.token_urlsafe(32)
|
|
||||||
async with token_lock:
|
|
||||||
tokens[nonce] = REGISTRATION_TOKEN_FILE.read_bytes().strip()
|
|
||||||
stop = asyncio.Event()
|
|
||||||
pulse = asyncio.create_task(heartbeat(message, stop))
|
|
||||||
try:
|
|
||||||
process = await asyncio.create_subprocess_exec(LAUNCHER, instance_id, nonce)
|
|
||||||
return_code = await process.wait()
|
|
||||||
if return_code == 0:
|
|
||||||
await message.ack()
|
|
||||||
else:
|
|
||||||
await message.nak(delay=30)
|
|
||||||
except Exception:
|
|
||||||
await message.nak(delay=30)
|
|
||||||
raise
|
|
||||||
finally:
|
|
||||||
stop.set()
|
|
||||||
await pulse
|
|
||||||
async with token_lock:
|
|
||||||
tokens.pop(nonce, None)
|
|
||||||
|
|
||||||
|
|
||||||
async def consume() -> None:
|
|
||||||
if CAPACITY < 1:
|
|
||||||
raise ValueError("RUNNER_CAPACITY must be at least 1")
|
|
||||||
tls = ssl.create_default_context(cafile=NATS_CA_FILE)
|
|
||||||
nc = await nats.connect(
|
|
||||||
NATS_URL,
|
|
||||||
user=NATS_USER,
|
|
||||||
password=NATS_PASSWORD_FILE.read_text().strip(),
|
|
||||||
tls=tls,
|
|
||||||
name=DURABLE,
|
|
||||||
)
|
|
||||||
js = nc.jetstream()
|
|
||||||
subscription = await js.pull_subscribe(
|
|
||||||
SUBJECT,
|
|
||||||
durable=DURABLE,
|
|
||||||
stream=STREAM,
|
|
||||||
config=ConsumerConfig(
|
|
||||||
durable_name=DURABLE,
|
|
||||||
filter_subject=SUBJECT,
|
|
||||||
ack_policy=AckPolicy.EXPLICIT,
|
|
||||||
ack_wait=5 * 60,
|
|
||||||
# This durable consumer is shared by every host of this runner type.
|
|
||||||
# Local CAPACITY controls each host; this is only a global safety cap.
|
|
||||||
max_ack_pending=MAX_INFLIGHT,
|
|
||||||
max_deliver=5,
|
|
||||||
),
|
|
||||||
)
|
|
||||||
active: set[asyncio.Task[None]] = set()
|
|
||||||
try:
|
|
||||||
while True:
|
|
||||||
active = {task for task in active if not task.done()}
|
|
||||||
free = CAPACITY - len(active)
|
|
||||||
if free == 0:
|
|
||||||
await asyncio.wait(active, return_when=asyncio.FIRST_COMPLETED)
|
|
||||||
continue
|
|
||||||
try:
|
|
||||||
messages = await subscription.fetch(batch=free, timeout=5)
|
|
||||||
except TimeoutError:
|
|
||||||
continue
|
|
||||||
for message in messages:
|
|
||||||
task = asyncio.create_task(run_one(message))
|
|
||||||
task.add_done_callback(lambda done: done.exception())
|
|
||||||
active.add(task)
|
|
||||||
finally:
|
|
||||||
if active:
|
|
||||||
await asyncio.gather(*active, return_exceptions=True)
|
|
||||||
await nc.drain()
|
|
||||||
|
|
||||||
|
|
||||||
async def main() -> None:
|
|
||||||
app = web.Application()
|
|
||||||
app.router.add_get("/token/{nonce}", token)
|
|
||||||
runner = web.AppRunner(app)
|
|
||||||
await runner.setup()
|
|
||||||
await web.TCPSite(runner, TOKEN_LISTEN, TOKEN_PORT).start()
|
|
||||||
try:
|
|
||||||
await consume()
|
|
||||||
finally:
|
|
||||||
await runner.cleanup()
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
|
||||||
asyncio.run(main())
|
|
||||||
@@ -74,6 +74,6 @@ sudo journalctl -u homelab-data-collect.service --since '2 hours ago'
|
|||||||
|
|
||||||
备份年龄不能证明恢复能力或 WAL 连续性；定期隔离恢复演练仍必要。证书指标读取磁盘文件，
|
备份年龄不能证明恢复能力或 WAL 连续性；定期隔离恢复演练仍必要。证书指标读取磁盘文件，
|
||||||
不证明运行进程已重载该证书。dev 检查不验证外部 DNS/TLS 路径、不执行写入。
|
不证明运行进程已重载该证书。dev 检查不验证外部 DNS/TLS 路径、不执行写入。
|
||||||
SQL 级指标见 [共享 PG](../shared-postgresql/README.md#sql-级指标2026-10-01-主机端上线)；异地备份及应用迁移不在本次范围。
|
SQL 级 exporter、异地备份及应用迁移不在本次范围。
|
||||||
|
|
||||||
跨服务状态见 [共享 PG wiki](https://git.ddupan.top/panxiao81/homelab-wiki/src/branch/main/services/shared-postgresql.md)。
|
跨服务状态见 [共享 PG wiki](https://git.ddupan.top/panxiao81/homelab-wiki/src/branch/main/services/shared-postgresql.md)。
|
||||||
|
|||||||
@@ -69,10 +69,7 @@ Ayatori 单独使用 `kv/infra/postgresql/ayatori/{prod,dev}`，仅有 username/
|
|||||||
6. `bootstrap-backup.yml` 验证 WAL 与首备成功后启用每日 timer；`proxy.yml` 要求唯一 primary 才发布入口。
|
6. `bootstrap-backup.yml` 验证 WAL 与首备成功后启用每日 timer；`proxy.yml` 要求唯一 primary 才发布入口。
|
||||||
7. DNS 由 `infrastructure/dns/records.yml` 与 Samba `provision-dc.yml --tags dns` 管理。
|
7. DNS 由 `infrastructure/dns/records.yml` 与 Samba `provision-dc.yml --tags dns` 管理。
|
||||||
8. `controller.yml` 安装 PG 证书每日续签调度，预检通过才启用。
|
8. `controller.yml` 安装 PG 证书每日续签调度，预检通过才启用。
|
||||||
9. `monitoring.yml` 激活 SQL 指标：先由 `site.yml` 写入模板，再执行本 play。它只在配置未生效时
|
9. Ayatori 独立控制面就绪后按 [接入说明](ayatori/README.md) 注册；CNPG 迁移另行按应用验收。
|
||||||
向 Patroni 发 HUP、对 pending 成员 `patronictl restart --pending`（primary 原位重启，不 failover，
|
|
||||||
写入短暂中断）或重启 dev；随后在可写实例创建监控角色与扩展并安装 exporter。不需要 Bao。
|
|
||||||
10. Ayatori 独立控制面就绪后按 [接入说明](ayatori/README.md) 注册；CNPG 迁移另行按应用验收。
|
|
||||||
|
|
||||||
普通 `site.yml` 不代表所有配置已在线激活；尚未实现通用参数变更的 rolling restart。
|
普通 `site.yml` 不代表所有配置已在线激活；尚未实现通用参数变更的 rolling restart。
|
||||||
专用 `renew-certificates.yml` 只处理证书，60 天有效期、剩余不足 14 天续签；逐主机检查服务，
|
专用 `renew-certificates.yml` 只处理证书，60 天有效期、剩余不足 14 天续签；逐主机检查服务，
|
||||||
@@ -100,30 +97,10 @@ Ayatori 角色，随后停库清理；不覆盖生产 PGDATA。全站恢复及
|
|||||||
|
|
||||||
现有 VictoriaMetrics/Alertmanager 已采集生产两节点 Patroni 原生指标，规则位于
|
现有 VictoriaMetrics/Alertmanager 已采集生产两节点 Patroni 原生指标，规则位于
|
||||||
`platform/observability/metrics/{scrapes,rules}/shared-postgresql.yaml`。覆盖成员不可采集、无可见主库、
|
`platform/observability/metrics/{scrapes,rules}/shared-postgresql.yaml`。覆盖成员不可采集、无可见主库、
|
||||||
多主、replica 未流复制及 PG 进程停止。没有部署另一套监控栈。dev SQL、备份年龄、证书和续签检查的主机采集已于 2026-09-27 部署，
|
多主、replica 未流复制及 PG 进程停止。没有部署另一套监控栈；数据库 SQL 级 exporter 仍后置。dev SQL、备份年龄、证书和续签检查的主机采集已于 2026-09-27 部署，
|
||||||
新增规则已随 [PR #166](https://git.ddupan.top/panxiao81/homelab-infra/pulls/166) 合并并由 Flux 接管；
|
新增规则已随 [PR #166](https://git.ddupan.top/panxiao81/homelab-infra/pulls/166) 合并并由 Flux 接管；
|
||||||
阈值、验证边界和排障见 [维护监控](../shared-data-monitoring/README.md)。
|
阈值、验证边界和排障见 [维护监控](../shared-data-monitoring/README.md)。
|
||||||
|
|
||||||
### SQL 级指标（2026-10-01 主机端上线）
|
|
||||||
|
|
||||||
每个实例在自身主机上运行 [prometheus-community/postgres_exporter](https://github.com/prometheus-community/postgres_exporter)
|
|
||||||
（`pg_exporter_version`，安装包与官方 `sha256sums.txt` 同源校验）：prod 为 laptop/pve1 的 `:9187`，
|
|
||||||
dev 为 laptop `:9188`，仅监听实例内网地址。exporter 以实例 OS 用户走本地 socket，由
|
|
||||||
`local all homelab_monitor peer map=monitor` 与 ident 映射登录到 `homelab_monitor`
|
|
||||||
（仅 `pg_monitor`、无密码、`CONNECTION LIMIT 3`）。这只约束 exporter 自己的查询，**不是隔离边界**：
|
|
||||||
同一 OS 用户仍可经 peer 以 superuser 登录，exporter 被攻破等同实例属主；systemd
|
|
||||||
`InaccessiblePaths` 只让它读不到数据目录、私钥与 pgpass。prod/dev 启用 `pg_stat_statements`。
|
|
||||||
|
|
||||||
规则 `platform/observability/metrics/rules/shared-postgresql-sql.yaml`（PrometheusRule）以上游
|
|
||||||
`postgres_mixin` 为底改写：采集/SQL 不通、复制回放落后 > 1 MiB（与 failover 阈值一致）、主库复制槽
|
|
||||||
保留 > 1 GiB、主库落在 pve1（HDD）超过 15 分钟、WAL 归档失败、XID/MXID 年龄、客户端连接 > 80%、空闲事务 > 10 分钟、事务 > 1 小时、
|
|
||||||
死锁、频繁请求 checkpoint。目标带 `env` 标签，Alertmanager 把 `env="dev"` 送到独立 dev 频道。
|
|
||||||
|
|
||||||
2026-10-01 验证：隔离集成测试覆盖 peer+ident 登录（主/备/dev）、读取 `pg_stat_statements`、
|
|
||||||
监控角色不可写；线上激活时两个 prod 成员按 pending 重启，时间线未变（未切换）；三个端点 `pg_up 1`、
|
|
||||||
无采集失败，exporter 各约 8 MiB；重跑 `changed=0`；vmagent Pod 可达三个端点。规则 promtool 场景通过。
|
|
||||||
`monitoring.yml --check` 不会显示 HUP、restart 与建角色（均为 command 任务），只能作参考。
|
|
||||||
|
|
||||||
2026-09-25 已验证：
|
2026-09-25 已验证：
|
||||||
|
|
||||||
- 真实 prod 主从、dev 初始化；Ayatori 管理账号 verify-full TLS/权限校验；生产稳定入口 SQL 写入。
|
- 真实 prod 主从、dev 初始化；Ayatori 管理账号 verify-full TLS/权限校验；生产稳定入口 SQL 写入。
|
||||||
|
|||||||
@@ -19,7 +19,6 @@ pg_profiles:
|
|||||||
user: pgprod
|
user: pgprod
|
||||||
port: 5432
|
port: 5432
|
||||||
api_port: 8008
|
api_port: 8008
|
||||||
exporter_port: 9187
|
|
||||||
dns: pg-prod.ad.ddupan.top
|
dns: pg-prod.ad.ddupan.top
|
||||||
memory_high: 768M
|
memory_high: 768M
|
||||||
memory_max: 1G
|
memory_max: 1G
|
||||||
@@ -29,7 +28,6 @@ pg_profiles:
|
|||||||
user: pgdev
|
user: pgdev
|
||||||
port: 5433
|
port: 5433
|
||||||
dns: pg-dev.ad.ddupan.top
|
dns: pg-dev.ad.ddupan.top
|
||||||
exporter_port: 9188
|
|
||||||
memory_high: 256M
|
memory_high: 256M
|
||||||
memory_max: 384M
|
memory_max: 384M
|
||||||
shared_buffers: 32MB
|
shared_buffers: 32MB
|
||||||
@@ -40,8 +38,3 @@ pg_replication_addresses: ['192.168.10.127/32', '10.60.0.20/32']
|
|||||||
pg_repo_host: 10.60.0.21
|
pg_repo_host: 10.60.0.21
|
||||||
pg_repo_user: pgbackup
|
pg_repo_user: pgbackup
|
||||||
pg_repo_path: /var/lib/homelab-pgbackrest
|
pg_repo_path: /var/lib/homelab-pgbackrest
|
||||||
# SQL 指标：exporter 以实例 OS 用户经本地 socket 连接，peer + ident 映射到只读监控角色，
|
|
||||||
# 因此无需密码或 Bao；角色仅有 pg_monitor，不复用实例管理账号。
|
|
||||||
pg_monitor_user: homelab_monitor
|
|
||||||
# 升级只改版本；安装包与同版本官方 sha256sums.txt 一起下载校验，不写死摘要。
|
|
||||||
pg_exporter_version: 0.20.1
|
|
||||||
|
|||||||
@@ -1,81 +0,0 @@
|
|||||||
---
|
|
||||||
# SQL 指标：激活监控所需的 HBA/ident 与 pg_stat_statements，创建只读监控角色并安装 exporter。
|
|
||||||
# 不需要 Bao：模板先由 site.yml 写入；本 play 只做激活与验证。会重启实例（仅在预加载库未生效时），
|
|
||||||
# 因此不放进 site.yml。重复执行应为 changed=0。
|
|
||||||
- name: 激活实例监控配置
|
|
||||||
hosts: pg_hosts
|
|
||||||
become: true
|
|
||||||
gather_facts: false
|
|
||||||
serial: 1
|
|
||||||
any_errors_fatal: true
|
|
||||||
tasks:
|
|
||||||
- name: 逐实例激活
|
|
||||||
ansible.builtin.include_tasks: tasks/monitoring-activate.yml
|
|
||||||
loop: "{{ pg_instances }}"
|
|
||||||
loop_control:
|
|
||||||
loop_var: pg_instance
|
|
||||||
|
|
||||||
- name: 重启 pending 的生产成员以加载 pg_stat_statements
|
|
||||||
hosts: pg-laptop
|
|
||||||
become: true
|
|
||||||
become_user: "{{ pg_profiles.prod.user }}"
|
|
||||||
gather_facts: false
|
|
||||||
vars:
|
|
||||||
pg_patronictl: [/opt/homelab-patroni/bin/patronictl, -c, "{{ pg_config_root }}/prod/patroni.yml"]
|
|
||||||
tasks:
|
|
||||||
# JSON 输出会省略空值字段：没有 pending 的成员不带 "Pending restart" 键。
|
|
||||||
- name: 读取集群成员状态
|
|
||||||
ansible.builtin.command:
|
|
||||||
argv: "{{ pg_patronictl + ['list', '-f', 'json'] }}"
|
|
||||||
register: pg_members
|
|
||||||
changed_when: false
|
|
||||||
check_mode: false
|
|
||||||
# Patroni 在原位重启 primary，不触发 failover；写入会短暂中断，客户端需重试。
|
|
||||||
- name: 仅重启 pending restart 的成员
|
|
||||||
ansible.builtin.command:
|
|
||||||
argv: "{{ pg_patronictl + ['restart', 'pg-prod', '--pending', '--force'] }}"
|
|
||||||
when: pg_members.stdout | from_json | selectattr('Pending restart', 'defined') | selectattr('Pending restart', 'equalto', '*') | list | length > 0
|
|
||||||
changed_when: true
|
|
||||||
- name: 等待两个成员恢复且无 pending
|
|
||||||
ansible.builtin.command:
|
|
||||||
argv: "{{ pg_patronictl + ['list', '-f', 'json'] }}"
|
|
||||||
register: pg_members_after
|
|
||||||
changed_when: false
|
|
||||||
when: not ansible_check_mode
|
|
||||||
retries: 30
|
|
||||||
delay: 4
|
|
||||||
until: >-
|
|
||||||
(pg_members_after.stdout | from_json | length) == 2
|
|
||||||
and (pg_members_after.stdout | from_json | selectattr('Pending restart', 'defined') | selectattr('Pending restart', 'equalto', '*') | list | length) == 0
|
|
||||||
and (pg_members_after.stdout | from_json | selectattr('State', 'in', ['running', 'streaming']) | list | length) == 2
|
|
||||||
|
|
||||||
- name: 创建监控角色并安装 exporter
|
|
||||||
hosts: pg_hosts
|
|
||||||
become: true
|
|
||||||
gather_facts: false
|
|
||||||
tasks:
|
|
||||||
- name: 下载 postgres_exporter 并校验同版本官方摘要
|
|
||||||
ansible.builtin.get_url:
|
|
||||||
url: "{{ pg_exporter_base }}/postgres_exporter-{{ pg_exporter_version }}.linux-amd64.tar.gz"
|
|
||||||
dest: /opt/homelab-postgres-exporter.tar.gz
|
|
||||||
checksum: "sha256:{{ pg_exporter_base }}/sha256sums.txt"
|
|
||||||
mode: '0644'
|
|
||||||
vars:
|
|
||||||
pg_exporter_base: https://github.com/prometheus-community/postgres_exporter/releases/download/v{{ pg_exporter_version }}
|
|
||||||
register: pg_exporter_download
|
|
||||||
retries: 5
|
|
||||||
delay: 10
|
|
||||||
until: pg_exporter_download is succeeded
|
|
||||||
- name: 解压 exporter（保留上游许可证）
|
|
||||||
ansible.builtin.unarchive:
|
|
||||||
src: /opt/homelab-postgres-exporter.tar.gz
|
|
||||||
dest: /opt
|
|
||||||
remote_src: true
|
|
||||||
creates: /opt/postgres_exporter-{{ pg_exporter_version }}.linux-amd64/postgres_exporter
|
|
||||||
# check 模式不下载，首次时没有可解压的文件。
|
|
||||||
when: not (ansible_check_mode and pg_exporter_download is changed)
|
|
||||||
- name: 逐实例配置
|
|
||||||
ansible.builtin.include_tasks: tasks/monitoring-exporter.yml
|
|
||||||
loop: "{{ pg_instances }}"
|
|
||||||
loop_control:
|
|
||||||
loop_var: pg_instance
|
|
||||||
@@ -71,7 +71,7 @@
|
|||||||
owner: root
|
owner: root
|
||||||
group: "{{ pg_profile.user }}"
|
group: "{{ pg_profile.user }}"
|
||||||
mode: '0640'
|
mode: '0640'
|
||||||
loop: "{{ ['patroni.yml', 'pgbackrest.conf'] if pg_instance == 'prod' else ['postgresql.conf', 'pg_hba.conf', 'pg_ident.conf'] }}"
|
loop: "{{ ['patroni.yml', 'pgbackrest.conf'] if pg_instance == 'prod' else ['postgresql.conf', 'pg_hba.conf'] }}"
|
||||||
no_log: true
|
no_log: true
|
||||||
- name: 安装实例专用服务（不自动启动或重启）
|
- name: 安装实例专用服务（不自动启动或重启）
|
||||||
ansible.builtin.template:
|
ansible.builtin.template:
|
||||||
|
|||||||
@@ -66,8 +66,6 @@ postgresql:
|
|||||||
unix_socket_directories: {{ pg_socket_dir }}
|
unix_socket_directories: {{ pg_socket_dir }}
|
||||||
unix_socket_permissions: '0700'
|
unix_socket_permissions: '0700'
|
||||||
password_encryption: scram-sha-256
|
password_encryption: scram-sha-256
|
||||||
# postmaster 级参数：变更后需由 monitoring.yml 执行 Patroni pending restart。
|
|
||||||
shared_preload_libraries: pg_stat_statements
|
|
||||||
shared_buffers: {{ pg_profile.shared_buffers }}
|
shared_buffers: {{ pg_profile.shared_buffers }}
|
||||||
work_mem: 4MB
|
work_mem: 4MB
|
||||||
maintenance_work_mem: 64MB
|
maintenance_work_mem: 64MB
|
||||||
@@ -78,7 +76,6 @@ postgresql:
|
|||||||
log_statement: none
|
log_statement: none
|
||||||
log_min_error_statement: panic
|
log_min_error_statement: panic
|
||||||
pg_hba:
|
pg_hba:
|
||||||
- local all {{ pg_monitor_user }} peer map=monitor
|
|
||||||
- local all {{ pg_profile.user }} peer
|
- local all {{ pg_profile.user }} peer
|
||||||
- local replication {{ pg_profile.user }} peer
|
- local replication {{ pg_profile.user }} peer
|
||||||
- local replication replicator scram-sha-256
|
- local replication replicator scram-sha-256
|
||||||
@@ -91,9 +88,6 @@ postgresql:
|
|||||||
{% endfor %}
|
{% endfor %}
|
||||||
- host all all 0.0.0.0/0 reject
|
- host all all 0.0.0.0/0 reject
|
||||||
- host all all ::/0 reject
|
- host all all ::/0 reject
|
||||||
# 只有实例 OS 用户（exporter 运行身份）可映射为监控角色。
|
|
||||||
pg_ident:
|
|
||||||
- monitor {{ pg_profile.user }} {{ pg_monitor_user }}
|
|
||||||
remove_data_directory_on_rewind_failure: false
|
remove_data_directory_on_rewind_failure: false
|
||||||
remove_data_directory_on_diverged_timelines: false
|
remove_data_directory_on_diverged_timelines: false
|
||||||
watchdog:
|
watchdog:
|
||||||
|
|||||||
@@ -1,4 +1,3 @@
|
|||||||
local all {{ pg_monitor_user }} peer map=monitor
|
|
||||||
local all {{ pg_profile.user }} peer
|
local all {{ pg_profile.user }} peer
|
||||||
{% for cidr in pg_client_cidrs %}
|
{% for cidr in pg_client_cidrs %}
|
||||||
hostssl all all {{ cidr }} scram-sha-256
|
hostssl all all {{ cidr }} scram-sha-256
|
||||||
|
|||||||
@@ -1,2 +0,0 @@
|
|||||||
# 只有实例 OS 用户（exporter 运行身份）可映射为监控角色。
|
|
||||||
monitor {{ pg_profile.user }} {{ pg_monitor_user }}
|
|
||||||
-3
@@ -3,7 +3,6 @@ port = {{ pg_profile.port }}
|
|||||||
unix_socket_directories = '{{ pg_socket_dir }}'
|
unix_socket_directories = '{{ pg_socket_dir }}'
|
||||||
unix_socket_permissions = 0700
|
unix_socket_permissions = 0700
|
||||||
hba_file = '{{ pg_config_dir }}/pg_hba.conf'
|
hba_file = '{{ pg_config_dir }}/pg_hba.conf'
|
||||||
ident_file = '{{ pg_config_dir }}/pg_ident.conf'
|
|
||||||
password_encryption = 'scram-sha-256'
|
password_encryption = 'scram-sha-256'
|
||||||
shared_buffers = '{{ pg_profile.shared_buffers }}'
|
shared_buffers = '{{ pg_profile.shared_buffers }}'
|
||||||
max_connections = {{ pg_profile.max_connections }}
|
max_connections = {{ pg_profile.max_connections }}
|
||||||
@@ -17,5 +16,3 @@ ssl_ca_file = '{{ pg_config_dir }}/ca.crt'
|
|||||||
archive_mode = off
|
archive_mode = off
|
||||||
log_statement = none
|
log_statement = none
|
||||||
log_min_error_statement = panic
|
log_min_error_statement = panic
|
||||||
# postmaster 级参数：变更后需由 monitoring.yml 重启 dev 实例。
|
|
||||||
shared_preload_libraries = 'pg_stat_statements'
|
|
||||||
|
|||||||
@@ -1,73 +0,0 @@
|
|||||||
---
|
|
||||||
- name: 设置现有实例路径
|
|
||||||
ansible.builtin.set_fact:
|
|
||||||
pg_profile: "{{ pg_profiles[pg_instance] }}"
|
|
||||||
pg_config_dir: "{{ pg_config_root }}/{{ pg_instance }}"
|
|
||||||
pg_socket_dir: "/run/homelab-postgresql-{{ pg_instance }}"
|
|
||||||
- name: 激活前必须存在运行中的受管服务
|
|
||||||
ansible.builtin.command:
|
|
||||||
argv: [systemctl, is-active, "homelab-postgresql-{{ pg_instance }}.service"]
|
|
||||||
changed_when: false
|
|
||||||
check_mode: false
|
|
||||||
# 生产：Patroni 只在自身 reload 时把本地配置写成数据目录里的 postgresql.conf/pg_ident.conf，
|
|
||||||
# 所以文件视图能说明 Patroni 是否已接收新配置；预加载库是否已生效另看 current_setting。
|
|
||||||
# 重启前 pg_file_settings 对该行报 "setting could not be applied"，属预期，不能按 error 过滤。
|
|
||||||
# dev：ident_file 是 postmaster 级参数，视图读取的是“当前生效”的那个文件，
|
|
||||||
# 所以启动时仍指向旧路径就会读不到映射；两列都为真才算已加载。
|
|
||||||
- name: 读取监控配置的激活状态
|
|
||||||
ansible.builtin.command:
|
|
||||||
argv:
|
|
||||||
- "{{ pg_bin_dir }}/psql"
|
|
||||||
- -X
|
|
||||||
- -At
|
|
||||||
- -h
|
|
||||||
- "{{ pg_socket_dir }}"
|
|
||||||
- -p
|
|
||||||
- "{{ pg_profile.port | string }}"
|
|
||||||
- -d
|
|
||||||
- postgres
|
|
||||||
- -c
|
|
||||||
- >-
|
|
||||||
SELECT
|
|
||||||
EXISTS (SELECT 1 FROM pg_ident_file_mappings WHERE map_name = 'monitor' AND error IS NULL)
|
|
||||||
AND EXISTS (SELECT 1 FROM pg_file_settings WHERE name = 'shared_preload_libraries'
|
|
||||||
AND setting ~ 'pg_stat_statements'),
|
|
||||||
current_setting('shared_preload_libraries') ~ 'pg_stat_statements'
|
|
||||||
become: true
|
|
||||||
become_user: "{{ pg_profile.user }}"
|
|
||||||
changed_when: false
|
|
||||||
check_mode: false
|
|
||||||
register: pg_monitor_state
|
|
||||||
- name: 让 Patroni 接收新的本地配置（reload，不重启）
|
|
||||||
ansible.builtin.command:
|
|
||||||
argv: [systemctl, kill, --kill-whom=main, --signal=HUP, homelab-postgresql-prod.service]
|
|
||||||
when: pg_instance == 'prod' and pg_monitor_state.stdout.split('|')[0] != 't'
|
|
||||||
changed_when: true
|
|
||||||
- name: 等待 Patroni 写出新配置
|
|
||||||
ansible.builtin.command:
|
|
||||||
argv: "{{ pg_monitor_state.cmd }}"
|
|
||||||
become: true
|
|
||||||
become_user: "{{ pg_profile.user }}"
|
|
||||||
when: pg_instance == 'prod' and pg_monitor_state.stdout.split('|')[0] != 't' and not ansible_check_mode
|
|
||||||
changed_when: false
|
|
||||||
register: pg_monitor_reloaded
|
|
||||||
retries: 10
|
|
||||||
delay: 3
|
|
||||||
until: pg_monitor_reloaded.stdout.split('|')[0] == 't'
|
|
||||||
# dev 没有 Patroni；预加载库与 ident_file 都是 postmaster 级参数，只能重启。
|
|
||||||
- name: 重启 dev 以加载监控配置
|
|
||||||
ansible.builtin.systemd_service:
|
|
||||||
name: homelab-postgresql-dev.service
|
|
||||||
state: restarted
|
|
||||||
when: pg_instance == 'dev' and pg_monitor_state.stdout != 't|t'
|
|
||||||
- name: 检查 dev 重启后监控配置已生效
|
|
||||||
ansible.builtin.command:
|
|
||||||
argv: "{{ pg_monitor_state.cmd }}"
|
|
||||||
become: true
|
|
||||||
become_user: "{{ pg_profile.user }}"
|
|
||||||
when: pg_instance == 'dev' and not ansible_check_mode
|
|
||||||
changed_when: false
|
|
||||||
register: pg_monitor_dev
|
|
||||||
retries: 10
|
|
||||||
delay: 3
|
|
||||||
until: pg_monitor_dev.rc == 0 and pg_monitor_dev.stdout == 't|t'
|
|
||||||
@@ -1,65 +0,0 @@
|
|||||||
---
|
|
||||||
- name: 设置现有实例路径
|
|
||||||
ansible.builtin.set_fact:
|
|
||||||
pg_profile: "{{ pg_profiles[pg_instance] }}"
|
|
||||||
pg_config_dir: "{{ pg_config_root }}/{{ pg_instance }}"
|
|
||||||
pg_parent_dir: "{{ pg_data_root }}/{{ pg_instance }}"
|
|
||||||
pg_socket_dir: "/run/homelab-postgresql-{{ pg_instance }}"
|
|
||||||
pg_psql: ["{{ pg_bin_dir }}/psql", -X, -At, -v, ON_ERROR_STOP=1, -h, "/run/homelab-postgresql-{{ pg_instance }}",
|
|
||||||
-p, "{{ pg_profiles[pg_instance].port | string }}", -d, postgres]
|
|
||||||
# 角色与扩展只在可写实例上创建，standby 经复制获得。
|
|
||||||
- name: 检查监控角色与扩展是否符合声明
|
|
||||||
ansible.builtin.command:
|
|
||||||
argv: "{{ pg_psql + ['-c', pg_monitor_check] }}"
|
|
||||||
vars:
|
|
||||||
pg_monitor_check: >-
|
|
||||||
SELECT pg_is_in_recovery(),
|
|
||||||
COALESCE((SELECT rolcanlogin AND NOT rolsuper AND NOT rolcreaterole AND NOT rolcreatedb
|
|
||||||
AND NOT rolreplication AND NOT rolbypassrls AND rolconnlimit = 3 AND rolpassword IS NULL
|
|
||||||
AND pg_has_role(oid, 'pg_monitor', 'MEMBER')
|
|
||||||
FROM pg_authid WHERE rolname = '{{ pg_monitor_user }}'), false)
|
|
||||||
AND EXISTS (SELECT 1 FROM pg_extension WHERE extname = 'pg_stat_statements')
|
|
||||||
become: true
|
|
||||||
become_user: "{{ pg_profile.user }}"
|
|
||||||
changed_when: false
|
|
||||||
check_mode: false
|
|
||||||
register: pg_monitor_role
|
|
||||||
- name: 创建或修正监控角色（无密码，仅能经本地 ident 映射登录）
|
|
||||||
ansible.builtin.command:
|
|
||||||
argv: "{{ pg_psql }}"
|
|
||||||
stdin: |
|
|
||||||
SELECT 'CREATE ROLE {{ pg_monitor_user }}'
|
|
||||||
WHERE NOT EXISTS (SELECT 1 FROM pg_roles WHERE rolname = '{{ pg_monitor_user }}') \gexec
|
|
||||||
ALTER ROLE {{ pg_monitor_user }} LOGIN NOSUPERUSER NOCREATEDB NOCREATEROLE NOREPLICATION NOBYPASSRLS
|
|
||||||
CONNECTION LIMIT 3 PASSWORD NULL;
|
|
||||||
GRANT pg_monitor TO {{ pg_monitor_user }};
|
|
||||||
CREATE EXTENSION IF NOT EXISTS pg_stat_statements;
|
|
||||||
become: true
|
|
||||||
become_user: "{{ pg_profile.user }}"
|
|
||||||
when: pg_monitor_role.stdout == 'f|f'
|
|
||||||
changed_when: true
|
|
||||||
- name: 安装实例 exporter 服务
|
|
||||||
ansible.builtin.template:
|
|
||||||
src: postgres-exporter.service.j2
|
|
||||||
dest: "/etc/systemd/system/homelab-postgresql-{{ pg_instance }}-exporter.service"
|
|
||||||
mode: '0644'
|
|
||||||
register: pg_exporter_unit
|
|
||||||
- name: 启用 exporter
|
|
||||||
ansible.builtin.systemd_service:
|
|
||||||
name: "homelab-postgresql-{{ pg_instance }}-exporter.service"
|
|
||||||
daemon_reload: "{{ pg_exporter_unit is changed }}"
|
|
||||||
enabled: true
|
|
||||||
state: "{{ 'restarted' if pg_exporter_unit is changed else 'started' }}"
|
|
||||||
# check 模式下 unit 文件尚未写入，systemd 找不到服务。
|
|
||||||
when: not (ansible_check_mode and pg_exporter_unit is changed)
|
|
||||||
# 同时证明 HBA/ident 已加载、角色可登录、exporter 能查询。
|
|
||||||
- name: 检查 exporter 已连上数据库
|
|
||||||
ansible.builtin.uri:
|
|
||||||
url: "http://{{ pg_address }}:{{ pg_profile.exporter_port }}/metrics"
|
|
||||||
return_content: true
|
|
||||||
register: pg_exporter_metrics
|
|
||||||
changed_when: false
|
|
||||||
when: not ansible_check_mode
|
|
||||||
retries: 10
|
|
||||||
delay: 3
|
|
||||||
until: pg_exporter_metrics.status == 200 and pg_exporter_metrics.content is search('(?m)^pg_up 1$')
|
|
||||||
@@ -1,27 +0,0 @@
|
|||||||
[Unit]
|
|
||||||
Description=PostgreSQL {{ pg_instance }} SQL metrics
|
|
||||||
After=homelab-postgresql-{{ pg_instance }}.service
|
|
||||||
[Service]
|
|
||||||
# 本地 socket 上 TLS 无意义，sslmode=disable 只是避免驱动默认尝试 TLS。
|
|
||||||
# 以实例 OS 用户运行才能进入 0700 的 socket 目录；peer + ident 把它映射成只读监控角色。
|
|
||||||
# 监控角色只约束 exporter 自己发出的查询；这不是隔离边界——同一 OS 用户仍可经 peer 以
|
|
||||||
# superuser 登录，exporter 被攻破即等同实例属主。InaccessiblePaths 只是不让它直接读写
|
|
||||||
# 数据目录、私钥与 pgpass。暴露面因此仅限内网监听地址。
|
|
||||||
User={{ pg_profile.user }}
|
|
||||||
Group={{ pg_profile.user }}
|
|
||||||
Environment="DATA_SOURCE_NAME=host={{ pg_socket_dir }} port={{ pg_profile.port }} user={{ pg_monitor_user }} dbname=postgres sslmode=disable application_name=postgres_exporter"
|
|
||||||
ExecStart=/opt/postgres_exporter-{{ pg_exporter_version }}.linux-amd64/postgres_exporter \
|
|
||||||
--web.listen-address={{ pg_address }}:{{ pg_profile.exporter_port }} \
|
|
||||||
--collector.database_wraparound --collector.long_running_transactions \
|
|
||||||
--collector.stat_checkpointer --collector.stat_wal_receiver --collector.stat_statements
|
|
||||||
Restart=on-failure
|
|
||||||
RestartSec=5
|
|
||||||
InaccessiblePaths={{ pg_parent_dir }} {{ pg_config_dir }}
|
|
||||||
NoNewPrivileges=true
|
|
||||||
ProtectSystem=strict
|
|
||||||
ProtectHome=true
|
|
||||||
PrivateTmp=true
|
|
||||||
PrivateDevices=true
|
|
||||||
MemoryMax=64M
|
|
||||||
[Install]
|
|
||||||
WantedBy=multi-user.target
|
|
||||||
@@ -1,45 +0,0 @@
|
|||||||
# SecretStore 由独立控制面的部署管理，必须只读下述两个管理凭据路径。
|
|
||||||
apiVersion: external-secrets.io/v1
|
|
||||||
kind: ExternalSecret
|
|
||||||
metadata:
|
|
||||||
name: homelab-postgresql-prod-admin
|
|
||||||
spec:
|
|
||||||
refreshInterval: 1h
|
|
||||||
secretStoreRef:
|
|
||||||
name: homelab-postgresql-admin
|
|
||||||
kind: SecretStore
|
|
||||||
target:
|
|
||||||
name: homelab-postgresql-prod-admin
|
|
||||||
creationPolicy: Owner
|
|
||||||
data:
|
|
||||||
- secretKey: username
|
|
||||||
remoteRef:
|
|
||||||
key: infra/postgresql/ayatori/prod
|
|
||||||
property: username
|
|
||||||
- secretKey: password
|
|
||||||
remoteRef:
|
|
||||||
key: infra/postgresql/ayatori/prod
|
|
||||||
property: password
|
|
||||||
---
|
|
||||||
# SecretStore 由独立控制面的部署管理，必须只读下述两个管理凭据路径。
|
|
||||||
apiVersion: external-secrets.io/v1
|
|
||||||
kind: ExternalSecret
|
|
||||||
metadata:
|
|
||||||
name: homelab-postgresql-dev-admin
|
|
||||||
spec:
|
|
||||||
refreshInterval: 1h
|
|
||||||
secretStoreRef:
|
|
||||||
name: homelab-postgresql-admin
|
|
||||||
kind: SecretStore
|
|
||||||
target:
|
|
||||||
name: homelab-postgresql-dev-admin
|
|
||||||
creationPolicy: Owner
|
|
||||||
data:
|
|
||||||
- secretKey: username
|
|
||||||
remoteRef:
|
|
||||||
key: infra/postgresql/ayatori/dev
|
|
||||||
property: username
|
|
||||||
- secretKey: password
|
|
||||||
remoteRef:
|
|
||||||
key: infra/postgresql/ayatori/dev
|
|
||||||
property: password
|
|
||||||
@@ -1,20 +0,0 @@
|
|||||||
-----BEGIN CERTIFICATE-----
|
|
||||||
MIIDMzCCAhugAwIBAgIUMs0iV657yC9UhA2p2vomLIbFnzgwDQYJKoZIhvcNAQEL
|
|
||||||
BQAwITEfMB0GA1UEAxMWZGR1cGFuLnRvcCBJbnRlcm5hbCBDQTAeFw0yNjA3MjQy
|
|
||||||
MDE1MDFaFw0zNjA3MjEyMDE1MzFaMCExHzAdBgNVBAMTFmRkdXBhbi50b3AgSW50
|
|
||||||
ZXJuYWwgQ0EwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQC6QWlwBe6f
|
|
||||||
t7Ca3KCTvr4Pz+jVO60WrMBoEDYYM8Mp04btBHzhAQHf9Pp8+15aEW9iUcQhqqm+
|
|
||||||
2vT6H0JEhIbplyCWY6Guv0mTu8f+lvFknJIl2b3JqnMLHJKjh/rBrsE12XZ3i17M
|
|
||||||
2tCr34BWcei85IZyQl5HMW6dB8lAE6bdom+YynK4oLJdej9DD6bSyM8WcL0OsneZ
|
|
||||||
NsjwOlNMy3zjbtaH6mH71SgbFinxLp3AAAuLVe1DIKhFxuTQeVr/WaPum5y/oOsc
|
|
||||||
0gJp9If6nsC33lpRGcPLiZE9kfFZa4fPe8laCaN8q1K253qZ0rjRiDhbTAppW4Fy
|
|
||||||
r5P67h+2D+TbAgMBAAGjYzBhMA4GA1UdDwEB/wQEAwIBBjAPBgNVHRMBAf8EBTAD
|
|
||||||
AQH/MB0GA1UdDgQWBBSOgk1fR0qhz/Bo4wD9g2BnOAzDXzAfBgNVHSMEGDAWgBSO
|
|
||||||
gk1fR0qhz/Bo4wD9g2BnOAzDXzANBgkqhkiG9w0BAQsFAAOCAQEANm5kKkts1Ar2
|
|
||||||
7IlS+TxLFrZ/C9yhIdGcBk2SL5E+5E8S3skQWLEPGLRwvV4RmiB8gQ2V6UyGLrCx
|
|
||||||
1MuuSmCDaSYL9G66sGX1MIHlQ0F0bHIOxxtsTwIYzb5Sl8h3MfsARabmOhE3xUkn
|
|
||||||
jaAT9YUweHhjF4vi0U1Q4F8oOSvu4eJp5dMx1r7b2bLN90A1xh9sfdkEenSBX0tm
|
|
||||||
xK82ROYXI2Ejv/EO+lPUIn3jfqbqrS2itw75Xz/ECHjIfSxvW98puP69U54a1gf6
|
|
||||||
gWdXslr0pGkyMHqxw4dmaecpK0QK3jvqCNycNwNBfMdCypS2QRy03adcUosEAP3O
|
|
||||||
LZU7Kd8aeg==
|
|
||||||
-----END CERTIFICATE-----
|
|
||||||
@@ -0,0 +1 @@
|
|||||||
|
common/ca.crt
|
||||||
@@ -0,0 +1,20 @@
|
|||||||
|
-----BEGIN CERTIFICATE-----
|
||||||
|
MIIDMzCCAhugAwIBAgIUMs0iV657yC9UhA2p2vomLIbFnzgwDQYJKoZIhvcNAQEL
|
||||||
|
BQAwITEfMB0GA1UEAxMWZGR1cGFuLnRvcCBJbnRlcm5hbCBDQTAeFw0yNjA3MjQy
|
||||||
|
MDE1MDFaFw0zNjA3MjEyMDE1MzFaMCExHzAdBgNVBAMTFmRkdXBhbi50b3AgSW50
|
||||||
|
ZXJuYWwgQ0EwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQC6QWlwBe6f
|
||||||
|
t7Ca3KCTvr4Pz+jVO60WrMBoEDYYM8Mp04btBHzhAQHf9Pp8+15aEW9iUcQhqqm+
|
||||||
|
2vT6H0JEhIbplyCWY6Guv0mTu8f+lvFknJIl2b3JqnMLHJKjh/rBrsE12XZ3i17M
|
||||||
|
2tCr34BWcei85IZyQl5HMW6dB8lAE6bdom+YynK4oLJdej9DD6bSyM8WcL0OsneZ
|
||||||
|
NsjwOlNMy3zjbtaH6mH71SgbFinxLp3AAAuLVe1DIKhFxuTQeVr/WaPum5y/oOsc
|
||||||
|
0gJp9If6nsC33lpRGcPLiZE9kfFZa4fPe8laCaN8q1K253qZ0rjRiDhbTAppW4Fy
|
||||||
|
r5P67h+2D+TbAgMBAAGjYzBhMA4GA1UdDwEB/wQEAwIBBjAPBgNVHRMBAf8EBTAD
|
||||||
|
AQH/MB0GA1UdDgQWBBSOgk1fR0qhz/Bo4wD9g2BnOAzDXzAfBgNVHSMEGDAWgBSO
|
||||||
|
gk1fR0qhz/Bo4wD9g2BnOAzDXzANBgkqhkiG9w0BAQsFAAOCAQEANm5kKkts1Ar2
|
||||||
|
7IlS+TxLFrZ/C9yhIdGcBk2SL5E+5E8S3skQWLEPGLRwvV4RmiB8gQ2V6UyGLrCx
|
||||||
|
1MuuSmCDaSYL9G66sGX1MIHlQ0F0bHIOxxtsTwIYzb5Sl8h3MfsARabmOhE3xUkn
|
||||||
|
jaAT9YUweHhjF4vi0U1Q4F8oOSvu4eJp5dMx1r7b2bLN90A1xh9sfdkEenSBX0tm
|
||||||
|
xK82ROYXI2Ejv/EO+lPUIn3jfqbqrS2itw75Xz/ECHjIfSxvW98puP69U54a1gf6
|
||||||
|
gWdXslr0pGkyMHqxw4dmaecpK0QK3jvqCNycNwNBfMdCypS2QRy03adcUosEAP3O
|
||||||
|
LZU7Kd8aeg==
|
||||||
|
-----END CERTIFICATE-----
|
||||||
@@ -0,0 +1,9 @@
|
|||||||
|
apiVersion: kustomize.config.k8s.io/v1beta1
|
||||||
|
kind: Kustomization
|
||||||
|
# 公共部分只含公开 CA；接入时必须选择 dev/ 或 prod/，不提供跨环境聚合入口。
|
||||||
|
configMapGenerator:
|
||||||
|
- name: homelab-postgresql-ca
|
||||||
|
namespace: ayatori-system
|
||||||
|
files: [ca.crt]
|
||||||
|
generatorOptions:
|
||||||
|
disableNameSuffixHash: true
|
||||||
@@ -0,0 +1,22 @@
|
|||||||
|
apiVersion: external-secrets.io/v1
|
||||||
|
kind: ExternalSecret
|
||||||
|
metadata:
|
||||||
|
name: homelab-postgresql-dev-admin
|
||||||
|
namespace: ayatori-system
|
||||||
|
spec:
|
||||||
|
refreshInterval: 1h
|
||||||
|
secretStoreRef:
|
||||||
|
name: homelab-postgresql-dev-admin
|
||||||
|
kind: SecretStore
|
||||||
|
target:
|
||||||
|
name: homelab-postgresql-dev-admin
|
||||||
|
creationPolicy: Owner
|
||||||
|
data:
|
||||||
|
- secretKey: username
|
||||||
|
remoteRef:
|
||||||
|
key: infra/postgresql/ayatori/dev
|
||||||
|
property: username
|
||||||
|
- secretKey: password
|
||||||
|
remoteRef:
|
||||||
|
key: infra/postgresql/ayatori/dev
|
||||||
|
property: password
|
||||||
@@ -0,0 +1,2 @@
|
|||||||
|
# 仅交付本环境管理凭据；由控制面部署流程绑定专用 ESO 身份。
|
||||||
|
path "kv/data/infra/postgresql/ayatori/dev" { capabilities = ["read"] }
|
||||||
@@ -0,0 +1,13 @@
|
|||||||
|
apiVersion: database.ayatori.ddupan.top/v1alpha1
|
||||||
|
kind: PostgreSQLInstance
|
||||||
|
metadata:
|
||||||
|
name: homelab-dev
|
||||||
|
spec:
|
||||||
|
endpoint:
|
||||||
|
host: pg-dev.ad.ddupan.top
|
||||||
|
hostaddr: 192.168.10.127
|
||||||
|
port: 5433
|
||||||
|
database: postgres
|
||||||
|
sslMode: verify-full
|
||||||
|
adminCredentialRef:
|
||||||
|
name: homelab-postgresql-dev-admin
|
||||||
@@ -0,0 +1,6 @@
|
|||||||
|
apiVersion: kustomize.config.k8s.io/v1beta1
|
||||||
|
kind: Kustomization
|
||||||
|
resources:
|
||||||
|
- ../common
|
||||||
|
- instance.yaml
|
||||||
|
- admin-credentials.yaml
|
||||||
@@ -1,3 +0,0 @@
|
|||||||
# 供未来独立控制面管理 SecretStore 的 ESO 身份绑定；本文件尚未应用。
|
|
||||||
path "kv/data/infra/postgresql/ayatori/prod" { capabilities = ["read"] }
|
|
||||||
path "kv/data/infra/postgresql/ayatori/dev" { capabilities = ["read"] }
|
|
||||||
@@ -1,28 +0,0 @@
|
|||||||
# 尚未 apply：须先准备管理 Secret、CA bundle 和角色感知的生产稳定入口。
|
|
||||||
apiVersion: database.ayatori.ddupan.top/v1alpha1
|
|
||||||
kind: PostgreSQLInstance
|
|
||||||
metadata:
|
|
||||||
name: homelab-prod
|
|
||||||
spec:
|
|
||||||
endpoint:
|
|
||||||
host: pg-prod.ad.ddupan.top
|
|
||||||
hostaddr: 192.168.10.2
|
|
||||||
port: 5432
|
|
||||||
database: postgres
|
|
||||||
sslMode: verify-full
|
|
||||||
adminCredentialRef:
|
|
||||||
name: homelab-postgresql-prod-admin
|
|
||||||
---
|
|
||||||
apiVersion: database.ayatori.ddupan.top/v1alpha1
|
|
||||||
kind: PostgreSQLInstance
|
|
||||||
metadata:
|
|
||||||
name: homelab-dev
|
|
||||||
spec:
|
|
||||||
endpoint:
|
|
||||||
host: pg-dev.ad.ddupan.top
|
|
||||||
hostaddr: 192.168.10.127
|
|
||||||
port: 5433
|
|
||||||
database: postgres
|
|
||||||
sslMode: verify-full
|
|
||||||
adminCredentialRef:
|
|
||||||
name: homelab-postgresql-dev-admin
|
|
||||||
@@ -1,12 +1,5 @@
|
|||||||
apiVersion: kustomize.config.k8s.io/v1beta1
|
apiVersion: kustomize.config.k8s.io/v1beta1
|
||||||
kind: Kustomization
|
kind: Kustomization
|
||||||
# 仅供未来独立 Ayatori 控制面；此 namespace 必须与 manager 的 Secret namespace 一致。
|
# 公共入口不注册实例或同步凭据；必须显式选择 dev/ 或 prod/。
|
||||||
namespace: ayatori-system
|
|
||||||
resources:
|
resources:
|
||||||
- instances.yaml
|
- common
|
||||||
- admin-credentials.yaml
|
|
||||||
configMapGenerator:
|
|
||||||
- name: homelab-postgresql-ca
|
|
||||||
files: [ca.crt]
|
|
||||||
generatorOptions:
|
|
||||||
disableNameSuffixHash: true
|
|
||||||
|
|||||||
@@ -0,0 +1,22 @@
|
|||||||
|
apiVersion: external-secrets.io/v1
|
||||||
|
kind: ExternalSecret
|
||||||
|
metadata:
|
||||||
|
name: homelab-postgresql-prod-admin
|
||||||
|
namespace: ayatori-system
|
||||||
|
spec:
|
||||||
|
refreshInterval: 1h
|
||||||
|
secretStoreRef:
|
||||||
|
name: homelab-postgresql-prod-admin
|
||||||
|
kind: SecretStore
|
||||||
|
target:
|
||||||
|
name: homelab-postgresql-prod-admin
|
||||||
|
creationPolicy: Owner
|
||||||
|
data:
|
||||||
|
- secretKey: username
|
||||||
|
remoteRef:
|
||||||
|
key: infra/postgresql/ayatori/prod
|
||||||
|
property: username
|
||||||
|
- secretKey: password
|
||||||
|
remoteRef:
|
||||||
|
key: infra/postgresql/ayatori/prod
|
||||||
|
property: password
|
||||||
@@ -0,0 +1,2 @@
|
|||||||
|
# 仅交付本环境管理凭据；由控制面部署流程绑定专用 ESO 身份。
|
||||||
|
path "kv/data/infra/postgresql/ayatori/prod" { capabilities = ["read"] }
|
||||||
@@ -0,0 +1,13 @@
|
|||||||
|
apiVersion: database.ayatori.ddupan.top/v1alpha1
|
||||||
|
kind: PostgreSQLInstance
|
||||||
|
metadata:
|
||||||
|
name: homelab-prod
|
||||||
|
spec:
|
||||||
|
endpoint:
|
||||||
|
host: pg-prod.ad.ddupan.top
|
||||||
|
hostaddr: 192.168.10.2
|
||||||
|
port: 5432
|
||||||
|
database: postgres
|
||||||
|
sslMode: verify-full
|
||||||
|
adminCredentialRef:
|
||||||
|
name: homelab-postgresql-prod-admin
|
||||||
@@ -0,0 +1,6 @@
|
|||||||
|
apiVersion: kustomize.config.k8s.io/v1beta1
|
||||||
|
kind: Kustomization
|
||||||
|
resources:
|
||||||
|
- ../common
|
||||||
|
- instance.yaml
|
||||||
|
- admin-credentials.yaml
|
||||||
@@ -88,9 +88,9 @@ def main():
|
|||||||
pg_etcd_endpoints=[f'{h["etcd_address"]}:23379' for h in hosts.values()],
|
pg_etcd_endpoints=[f'{h["etcd_address"]}:23379' for h in hosts.values()],
|
||||||
pg_config_dir='/node/config',pg_parent_dir='/node',pg_data_dir='/node/data',pg_socket_dir='/node/socket',
|
pg_config_dir='/node/config',pg_parent_dir='/node',pg_data_dir='/node/data',pg_socket_dir='/node/socket',
|
||||||
pg_bin_dir='/usr/lib/postgresql/18/bin',pg_replication_addresses=['127.0.0.0/8'],pg_client_cidrs=['127.0.0.0/8'],
|
pg_bin_dir='/usr/lib/postgresql/18/bin',pg_replication_addresses=['127.0.0.0/8'],pg_client_cidrs=['127.0.0.0/8'],
|
||||||
pg_repo_remote=False,pg_repo_path='/repo',pg_monitor_user='homelab_monitor')
|
pg_repo_remote=False,pg_repo_path='/repo')
|
||||||
values[name]=v
|
values[name]=v
|
||||||
for template in (['postgresql.conf','pg_hba.conf','pg_ident.conf'] if name=='dev' else ['patroni.yml','pgbackrest.conf']):
|
for template in (['postgresql.conf','pg_hba.conf'] if name=='dev' else ['patroni.yml','pgbackrest.conf']):
|
||||||
(cfg/template).write_text(env.get_template(template+'.j2').render(**v))
|
(cfg/template).write_text(env.get_template(template+'.j2').render(**v))
|
||||||
for folder in ['socket','backup-lock','backup-spool']:(d/folder).mkdir()
|
for folder in ['socket','backup-lock','backup-spool']:(d/folder).mkdir()
|
||||||
(pg/'repo').mkdir();(pg/'restore').mkdir();(pg/'pitr').mkdir()
|
(pg/'repo').mkdir();(pg/'restore').mkdir();(pg/'pitr').mkdir()
|
||||||
@@ -131,20 +131,6 @@ def main():
|
|||||||
initial_backup=json.loads(backrest('p1','--output=json','info').stdout)[0]['backup'][-1]['label']
|
initial_backup=json.loads(backrest('p1','--output=json','info').stdout)[0]['backup'][-1]['label']
|
||||||
recovery_target=sql('p1','SELECT clock_timestamp()::text')
|
recovery_target=sql('p1','SELECT clock_timestamp()::text')
|
||||||
print('PASS: separate dev, TLS verify-full, non-superuser Ayatori native privileges, initial backup',flush=True)
|
print('PASS: separate dev, TLS verify-full, non-superuser Ayatori native privileges, initial backup',flush=True)
|
||||||
# 与 monitoring.yml 相同的声明：无密码，只能由实例 OS 用户经 peer + ident 映射登录。
|
|
||||||
monitor_role="CREATE ROLE homelab_monitor LOGIN CONNECTION LIMIT 3 PASSWORD NULL; GRANT pg_monitor TO homelab_monitor; CREATE EXTENSION pg_stat_statements;"
|
|
||||||
sql('p1',monitor_role);sql('dev',monitor_role)
|
|
||||||
def monitor(name,query):
|
|
||||||
return run(['docker','exec','-i',prefix+'-'+name,'psql','-X','-v','ON_ERROR_STOP=1','-At','-h','/node/socket','-p','25432','-U','homelab_monitor','postgres'],input=query).stdout.strip()
|
|
||||||
for name in ['p1','p2','dev']:
|
|
||||||
wait_for(lambda: monitor(name,"SELECT current_setting('shared_preload_libraries')||(SELECT count(*)>=0 FROM pg_stat_statements)")=='pg_stat_statementstrue')
|
|
||||||
for name in ['p1','dev']:
|
|
||||||
try:
|
|
||||||
monitor(name,'CREATE TABLE public.monitor_write(id int)')
|
|
||||||
raise AssertionError(f'monitor role could write on {name}')
|
|
||||||
except subprocess.CalledProcessError:
|
|
||||||
pass
|
|
||||||
print('PASS: monitor role logs in via peer+ident on primary/standby/dev, reads pg_stat_statements, cannot write',flush=True)
|
|
||||||
proxycfg=w/'proxy.cfg'
|
proxycfg=w/'proxy.cfg'
|
||||||
# 文件父目录仍由调用者拥有，新增代理配置不触碰 PG 文件。
|
# 文件父目录仍由调用者拥有，新增代理配置不触碰 PG 文件。
|
||||||
proxycfg.write_text(env.get_template('haproxy.cfg.j2').render(pg_proxy_bind='127.0.2.10:26432',pg_proxy_members=[dict(name=n,address=values[n]['pg_address'],port=25432,api_port=28008) for n in ['p1','p2']]))
|
proxycfg.write_text(env.get_template('haproxy.cfg.j2').render(pg_proxy_bind='127.0.2.10:26432',pg_proxy_members=[dict(name=n,address=values[n]['pg_address'],port=25432,api_port=28008) for n in ['p1','p2']]))
|
||||||
|
|||||||
@@ -0,0 +1,51 @@
|
|||||||
|
"""校验渲染后的环境隔离与引用关系，无网络、集群写入或凭据访问。"""
|
||||||
|
from pathlib import Path
|
||||||
|
import re
|
||||||
|
import subprocess
|
||||||
|
import unittest
|
||||||
|
|
||||||
|
import yaml
|
||||||
|
|
||||||
|
ROOT = Path(__file__).resolve().parents[1] / 'ayatori'
|
||||||
|
|
||||||
|
|
||||||
|
def render(environment):
|
||||||
|
return list(yaml.safe_load_all(subprocess.check_output(
|
||||||
|
['kubectl', 'kustomize', str(ROOT / environment)], text=True)))
|
||||||
|
|
||||||
|
|
||||||
|
class AyatoriManifestTests(unittest.TestCase):
|
||||||
|
def test_common_entry_cannot_register_instances_or_sync_credentials(self):
|
||||||
|
objects = render('.')
|
||||||
|
self.assertEqual([x['kind'] for x in objects], ['ConfigMap'])
|
||||||
|
self.assertEqual(objects[0]['metadata']['namespace'], 'ayatori-system')
|
||||||
|
|
||||||
|
def test_environment_isolation_scope_and_references(self):
|
||||||
|
for env, address, port in [('dev', '192.168.10.127', 5433), ('prod', '192.168.10.2', 5432)]:
|
||||||
|
with self.subTest(environment=env):
|
||||||
|
objects = render(env)
|
||||||
|
self.assertEqual(sorted(x['kind'] for x in objects),
|
||||||
|
['ConfigMap', 'ExternalSecret', 'PostgreSQLInstance'])
|
||||||
|
instance = next(x for x in objects if x['kind'] == 'PostgreSQLInstance')
|
||||||
|
secret = next(x for x in objects if x['kind'] == 'ExternalSecret')
|
||||||
|
self.assertNotIn('namespace', instance['metadata'])
|
||||||
|
self.assertEqual(instance['metadata']['name'], 'homelab-' + env)
|
||||||
|
self.assertEqual(instance['spec']['endpoint'], {
|
||||||
|
'host': 'pg-' + env + '.ad.ddupan.top', 'hostaddr': address,
|
||||||
|
'port': port, 'database': 'postgres', 'sslMode': 'verify-full'})
|
||||||
|
self.assertEqual(secret['metadata']['namespace'], 'ayatori-system')
|
||||||
|
self.assertEqual(instance['spec']['adminCredentialRef']['name'], secret['spec']['target']['name'])
|
||||||
|
self.assertEqual(secret['spec']['secretStoreRef'],
|
||||||
|
{'name': 'homelab-postgresql-' + env + '-admin', 'kind': 'SecretStore'})
|
||||||
|
self.assertEqual({x['remoteRef']['key'] for x in secret['spec']['data']},
|
||||||
|
{'infra/postgresql/ayatori/' + env})
|
||||||
|
self.assertEqual({x['secretKey'] for x in secret['spec']['data']}, {'username', 'password'})
|
||||||
|
self.assertEqual({x['remoteRef']['property'] for x in secret['spec']['data']}, {'username', 'password'})
|
||||||
|
policy = (ROOT / env / 'eso-policy.hcl').read_text()
|
||||||
|
self.assertEqual(re.findall(r'path\s+"([^"]+)"', policy),
|
||||||
|
['kv/data/infra/postgresql/ayatori/' + env])
|
||||||
|
self.assertEqual(re.findall(r'capabilities\s*=\s*\[([^]]+)\]', policy), ['"read"'])
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == '__main__':
|
||||||
|
unittest.main()
|
||||||
@@ -1,26 +0,0 @@
|
|||||||
# CI pushes only :latest from main; follow the digest behind it. Old digests
|
|
||||||
# lose their tag and are collected by zot GC, so the deployment must keep
|
|
||||||
# tracking the current one.
|
|
||||||
apiVersion: image.toolkit.fluxcd.io/v1
|
|
||||||
kind: ImageRepository
|
|
||||||
metadata:
|
|
||||||
name: backstage
|
|
||||||
namespace: flux-system
|
|
||||||
spec:
|
|
||||||
image: zot.ad.ddupan.top/panxiao81/backstage
|
|
||||||
interval: 1m
|
|
||||||
---
|
|
||||||
apiVersion: image.toolkit.fluxcd.io/v1
|
|
||||||
kind: ImagePolicy
|
|
||||||
metadata:
|
|
||||||
name: backstage
|
|
||||||
namespace: flux-system
|
|
||||||
spec:
|
|
||||||
imageRepositoryRef:
|
|
||||||
name: backstage
|
|
||||||
filterTags:
|
|
||||||
pattern: '^latest$'
|
|
||||||
policy:
|
|
||||||
alphabetical: {}
|
|
||||||
digestReflectionPolicy: Always
|
|
||||||
interval: 1m
|
|
||||||
@@ -1,19 +0,0 @@
|
|||||||
# Gitea token of the private `flux-bot` user: collaborator with write on
|
|
||||||
# homelab-infra only, token scoped to write:repository. Stored in OpenBao as
|
|
||||||
# username/password because that is the basic-auth Secret shape Flux reads.
|
|
||||||
apiVersion: external-secrets.io/v1
|
|
||||||
kind: ExternalSecret
|
|
||||||
metadata:
|
|
||||||
name: homelab-infra-write
|
|
||||||
namespace: flux-system
|
|
||||||
spec:
|
|
||||||
refreshInterval: 1h
|
|
||||||
secretStoreRef:
|
|
||||||
kind: ClusterSecretStore
|
|
||||||
name: openbao
|
|
||||||
target:
|
|
||||||
creationPolicy: Owner
|
|
||||||
name: homelab-infra-write
|
|
||||||
dataFrom:
|
|
||||||
- extract:
|
|
||||||
key: k8s/flux-image-automation
|
|
||||||
@@ -1,15 +0,0 @@
|
|||||||
# Write-capable source used only by ImageUpdateAutomation. The flux-system
|
|
||||||
# GitRepository stays anonymous and read-only.
|
|
||||||
apiVersion: source.toolkit.fluxcd.io/v1
|
|
||||||
kind: GitRepository
|
|
||||||
metadata:
|
|
||||||
name: homelab-infra-write
|
|
||||||
namespace: flux-system
|
|
||||||
spec:
|
|
||||||
interval: 10m
|
|
||||||
ref:
|
|
||||||
branch: main
|
|
||||||
secretRef:
|
|
||||||
name: homelab-infra-write
|
|
||||||
timeout: 60s
|
|
||||||
url: http://gitea-http.gitea.svc.cluster.local:3000/panxiao81/homelab-infra.git
|
|
||||||
@@ -1,28 +0,0 @@
|
|||||||
# Commits digest changes straight to main (no PR): dev images are expected to
|
|
||||||
# deploy as soon as CI pushes them. Scoped to apps/backstage so a stray setter
|
|
||||||
# elsewhere cannot be rewritten.
|
|
||||||
apiVersion: image.toolkit.fluxcd.io/v1
|
|
||||||
kind: ImageUpdateAutomation
|
|
||||||
metadata:
|
|
||||||
name: homelab-infra
|
|
||||||
namespace: flux-system
|
|
||||||
spec:
|
|
||||||
interval: 1m
|
|
||||||
sourceRef:
|
|
||||||
kind: GitRepository
|
|
||||||
name: homelab-infra-write
|
|
||||||
git:
|
|
||||||
checkout:
|
|
||||||
ref:
|
|
||||||
branch: main
|
|
||||||
commit:
|
|
||||||
author:
|
|
||||||
name: flux-bot
|
|
||||||
email: [email protected]
|
|
||||||
messageTemplate: |
|
|
||||||
chore(image): update {{ range .Changed.Changes }}{{ .OldValue }} -> {{ .NewValue }} {{ end }}
|
|
||||||
push:
|
|
||||||
branch: main
|
|
||||||
update:
|
|
||||||
path: ./apps/backstage
|
|
||||||
strategy: Setters
|
|
||||||
@@ -1,7 +0,0 @@
|
|||||||
apiVersion: kustomize.config.k8s.io/v1beta1
|
|
||||||
kind: Kustomization
|
|
||||||
resources:
|
|
||||||
- external-secret.yaml
|
|
||||||
- gitrepository.yaml
|
|
||||||
- imageupdateautomation.yaml
|
|
||||||
- backstage.yaml
|
|
||||||
@@ -1,9 +0,0 @@
|
|||||||
ARG DOCKER_VERSION=29.7.1
|
|
||||||
FROM docker:${DOCKER_VERSION}-dind
|
|
||||||
|
|
||||||
RUN apk add --no-cache fuse-overlayfs
|
|
||||||
|
|
||||||
COPY --chmod=0755 entrypoint.sh /usr/local/bin/dind-fuse-entrypoint
|
|
||||||
|
|
||||||
ENTRYPOINT ["/usr/local/bin/dind-fuse-entrypoint"]
|
|
||||||
CMD []
|
|
||||||
@@ -1,42 +0,0 @@
|
|||||||
#!/bin/sh
|
|
||||||
set -eu
|
|
||||||
|
|
||||||
# /dev belongs to the Kata guest. A privileged dockerd container can create
|
|
||||||
# this standard FUSE node without exposing the PVE host's /dev/fuse to the LXC.
|
|
||||||
if [ ! -e /dev/fuse ]; then
|
|
||||||
mknod /dev/fuse c 10 229
|
|
||||||
fi
|
|
||||||
chmod 0666 /dev/fuse
|
|
||||||
|
|
||||||
# kind's nested kubelet opens /dev/kmsg. This node belongs to the Kata guest;
|
|
||||||
# exposing it to a kind node does not expose the LXC or PVE host kernel log.
|
|
||||||
if [ ! -e /dev/kmsg ]; then
|
|
||||||
mknod /dev/kmsg c 1 11
|
|
||||||
fi
|
|
||||||
chmod 0600 /dev/kmsg
|
|
||||||
|
|
||||||
storage_driver=fuse-overlayfs
|
|
||||||
|
|
||||||
# kind's kubelet/cAdvisor cannot map a virtiofs-backed fuse-overlayfs root to a
|
|
||||||
# block device. When requested, create an ext4 filesystem on a guest-local loop
|
|
||||||
# device. The backing file, loop device and filesystem all die with the Kata VM.
|
|
||||||
if [ -n "${DIND_LOOPBACK_SIZE:-}" ]; then
|
|
||||||
[ -e /dev/loop-control ] || mknod /dev/loop-control c 10 237
|
|
||||||
i=0
|
|
||||||
while [ "$i" -lt 8 ]; do
|
|
||||||
[ -e "/dev/loop$i" ] || mknod "/dev/loop$i" b 7 "$i"
|
|
||||||
i=$((i + 1))
|
|
||||||
done
|
|
||||||
|
|
||||||
backing_file="${DIND_LOOPBACK_FILE:-/var/lib/docker-loopback.img}"
|
|
||||||
truncate -s "$DIND_LOOPBACK_SIZE" "$backing_file"
|
|
||||||
# Alpine's BusyBox losetup supports -f, but not util-linux's
|
|
||||||
# --find/--show long options.
|
|
||||||
loop_device="$(losetup -f)"
|
|
||||||
losetup "$loop_device" "$backing_file"
|
|
||||||
mkfs.ext4 -q -F -m 0 "$loop_device"
|
|
||||||
mount "$loop_device" /var/lib/docker
|
|
||||||
storage_driver=overlay2
|
|
||||||
fi
|
|
||||||
|
|
||||||
exec dockerd-entrypoint.sh --storage-driver="$storage_driver" "$@"
|
|
||||||
@@ -25,8 +25,6 @@ resources:
|
|||||||
- scrapes/shared-etcd.yaml
|
- scrapes/shared-etcd.yaml
|
||||||
- rules/shared-postgresql.yaml
|
- rules/shared-postgresql.yaml
|
||||||
- scrapes/shared-postgresql.yaml
|
- scrapes/shared-postgresql.yaml
|
||||||
- rules/shared-postgresql-sql.yaml
|
|
||||||
- scrapes/shared-postgresql-sql.yaml
|
|
||||||
- scrapes/platform-controllers.yaml
|
- scrapes/platform-controllers.yaml
|
||||||
- rules/platform-controllers.yaml
|
- rules/platform-controllers.yaml
|
||||||
- scrapes/cnpg.yaml
|
- scrapes/cnpg.yaml
|
||||||
|
|||||||
@@ -58,7 +58,6 @@ spec:
|
|||||||
- alert: SharedPostgresDevUnavailable
|
- alert: SharedPostgresDevUnavailable
|
||||||
expr: homelab_data_dev_sql_ready{job="shared-data-maintenance"} == 0
|
expr: homelab_data_dev_sql_ready{job="shared-data-maintenance"} == 0
|
||||||
for: 3m
|
for: 3m
|
||||||
# env 供 Alertmanager 把 dev 分流到独立 channel。
|
labels: {severity: warning, service: shared-postgresql}
|
||||||
labels: {severity: warning, service: shared-postgresql, env: dev}
|
|
||||||
annotations:
|
annotations:
|
||||||
summary: 共享 PG 开发实例本地 SQL 不可用或处于只读状态
|
summary: 共享 PG 开发实例本地 SQL 不可用或处于只读状态
|
||||||
|
|||||||
@@ -1,127 +0,0 @@
|
|||||||
# 以 postgres_exporter 上游 postgres_mixin 为底按本环境改写：实例停止、多主、角色变化已由
|
|
||||||
# Patroni 规则（shared-postgresql.yaml）覆盖；QPS/缓存命中率/回滚数不是故障信号，未采用；
|
|
||||||
# 上游 PostgresHasReplicationSlotUsed 会被 Patroni 在 standby 保留的非活跃槽持续触发，未采用。
|
|
||||||
# 每条告警都带抓取目标的 env 标签，Alertmanager 按它把 dev 分流到独立 channel。
|
|
||||||
apiVersion: monitoring.coreos.com/v1
|
|
||||||
kind: PrometheusRule
|
|
||||||
metadata:
|
|
||||||
name: shared-postgresql-sql
|
|
||||||
namespace: monitoring
|
|
||||||
spec:
|
|
||||||
groups:
|
|
||||||
- name: shared-postgresql-sql
|
|
||||||
rules:
|
|
||||||
- alert: SharedPostgresSqlExporterDown
|
|
||||||
expr: up{job="shared-postgresql-sql"} == 0
|
|
||||||
for: 3m
|
|
||||||
labels: {severity: warning, service: shared-postgresql}
|
|
||||||
annotations:
|
|
||||||
summary: '共享 PG {{ $labels.env }} {{ $labels.member }} 的 SQL exporter 无法采集'
|
|
||||||
# 进程停止已有 Patroni critical；这里覆盖进程在但 SQL 不通（HBA、连接耗尽、dev 实例）。
|
|
||||||
- alert: SharedPostgresSqlUnreachable
|
|
||||||
expr: pg_up{job="shared-postgresql-sql"} == 0
|
|
||||||
for: 2m
|
|
||||||
labels: {severity: warning, service: shared-postgresql}
|
|
||||||
annotations:
|
|
||||||
summary: '共享 PG {{ $labels.env }} {{ $labels.member }} 无法以监控角色执行 SQL'
|
|
||||||
# 正常 primary 在 laptop ZFS SSD；pve1 的数据盘是 HDD。自动切换后主库安全落到 pve1
|
|
||||||
# 不会触发任何“无主/多主”规则（2026-09-30 的切换整整一天无人察觉），所以单独提示回切。
|
|
||||||
# 15m 留给计划内切换演练。
|
|
||||||
- alert: SharedPostgresPrimaryOnStandbyHost
|
|
||||||
expr: pg_replication_is_replica{job="shared-postgresql-sql", env="prod", member="pg-pve1"} == 0
|
|
||||||
for: 15m
|
|
||||||
labels: {severity: warning, service: shared-postgresql}
|
|
||||||
annotations:
|
|
||||||
summary: 共享 PG 生产主库运行在 pg-pve1（HDD），请确认切换原因并计划回切 laptop
|
|
||||||
- alert: SharedPostgresSqlCollectorFailing
|
|
||||||
expr: pg_scrape_collector_success{job="shared-postgresql-sql"} == 0 or pg_exporter_last_scrape_error{job="shared-postgresql-sql"} == 1
|
|
||||||
for: 10m
|
|
||||||
labels: {severity: warning, service: shared-postgresql}
|
|
||||||
annotations:
|
|
||||||
summary: '共享 PG {{ $labels.env }} {{ $labels.member }} 部分 SQL 指标采集失败 {{ $labels.collector }}'
|
|
||||||
# 与 Patroni maximum_lag_on_failover（1 MiB）一致：持续超过即该 standby 不再是合格的切换候选。
|
|
||||||
- alert: SharedPostgresReplicationLagging
|
|
||||||
expr: pg_stat_replication_pg_wal_lsn_diff{job="shared-postgresql-sql"} > 1048576
|
|
||||||
for: 5m
|
|
||||||
labels: {severity: warning, service: shared-postgresql}
|
|
||||||
annotations:
|
|
||||||
summary: '共享 PG {{ $labels.env }} 副本 {{ $labels.application_name }} 回放落后超过 1 MiB，已不满足自动切换条件'
|
|
||||||
# 只看可写成员：standby 上 Patroni 维护的槽副本不反映真实保留压力。
|
|
||||||
# 1 GiB 是 max_slot_wal_keep_size（2GB）的一半，留出处理时间。
|
|
||||||
- alert: SharedPostgresReplicationSlotRetention
|
|
||||||
expr: >-
|
|
||||||
pg_replication_slots_pg_wal_lsn_diff{job="shared-postgresql-sql"} > 1073741824
|
|
||||||
and on(instance) pg_replication_is_replica{job="shared-postgresql-sql"} == 0
|
|
||||||
for: 10m
|
|
||||||
labels: {severity: warning, service: shared-postgresql}
|
|
||||||
annotations:
|
|
||||||
summary: '共享 PG {{ $labels.env }} 复制槽 {{ $labels.slot_name }} 保留 WAL 超过 1 GiB'
|
|
||||||
# 计数器规则不用 increase()：MetricsQL 把新序列首个样本视为从 0 增长，exporter 新上线或
|
|
||||||
# 角色变化时，历史累计值（如 2026-09-25 搭建期的 9 次归档失败）会被当成刚发生而误报。
|
|
||||||
# 与 offset 相减对新序列返回空；计数器重置（重启/stats reset）得负值，同样不告警。
|
|
||||||
- alert: SharedPostgresWalArchiveFailing
|
|
||||||
expr: >-
|
|
||||||
pg_stat_archiver_failed_count{job="shared-postgresql-sql"}
|
|
||||||
- pg_stat_archiver_failed_count{job="shared-postgresql-sql"} offset 15m > 0
|
|
||||||
labels: {severity: warning, service: shared-postgresql}
|
|
||||||
annotations:
|
|
||||||
summary: '共享 PG {{ $labels.env }} {{ $labels.member }} WAL 归档失败，PITR 链可能中断'
|
|
||||||
# XID/MXID 上限约 2^31；超过一半说明 freeze 跟不上，超过 75% 已接近强制停写；critical 时不重复 warning。
|
|
||||||
- alert: SharedPostgresWraparoundRisk
|
|
||||||
expr: >-
|
|
||||||
(max without(datname) (pg_database_wraparound_age_datfrozenxid_seconds{job="shared-postgresql-sql"}) > 1e9
|
|
||||||
or max without(datname) (pg_database_wraparound_age_datminmxid_seconds{job="shared-postgresql-sql"}) > 1e9)
|
|
||||||
unless (max without(datname) (pg_database_wraparound_age_datfrozenxid_seconds{job="shared-postgresql-sql"}) > 1.6e9
|
|
||||||
or max without(datname) (pg_database_wraparound_age_datminmxid_seconds{job="shared-postgresql-sql"}) > 1.6e9)
|
|
||||||
for: 10m
|
|
||||||
labels: {severity: warning, service: shared-postgresql}
|
|
||||||
annotations:
|
|
||||||
summary: '共享 PG {{ $labels.env }} {{ $labels.member }} XID/MXID 年龄超过 10 亿'
|
|
||||||
- alert: SharedPostgresWraparoundImminent
|
|
||||||
expr: >-
|
|
||||||
max without(datname) (pg_database_wraparound_age_datfrozenxid_seconds{job="shared-postgresql-sql"}) > 1.6e9
|
|
||||||
or max without(datname) (pg_database_wraparound_age_datminmxid_seconds{job="shared-postgresql-sql"}) > 1.6e9
|
|
||||||
for: 5m
|
|
||||||
labels: {severity: critical, service: shared-postgresql}
|
|
||||||
annotations:
|
|
||||||
summary: '共享 PG {{ $labels.env }} {{ $labels.member }} XID/MXID 接近回卷停写，需立即 VACUUM FREEZE'
|
|
||||||
- alert: SharedPostgresConnectionsHigh
|
|
||||||
expr: >-
|
|
||||||
sum without(datname, state, usename, application_name, backend_type, wait_event_type, wait_event)
|
|
||||||
(pg_stat_activity_count{job="shared-postgresql-sql", backend_type="client backend"})
|
|
||||||
/ pg_settings_max_connections{job="shared-postgresql-sql"} > 0.8
|
|
||||||
for: 5m
|
|
||||||
labels: {severity: warning, service: shared-postgresql}
|
|
||||||
annotations:
|
|
||||||
summary: '共享 PG {{ $labels.env }} {{ $labels.member }} 客户端连接超过 max_connections 的 80%'
|
|
||||||
# 长事务阻止 vacuum 回收并持有锁；idle in transaction 通常是应用忘记提交。
|
|
||||||
- alert: SharedPostgresIdleInTransaction
|
|
||||||
expr: >-
|
|
||||||
max without(datname, state, usename, application_name, backend_type, wait_event_type, wait_event)
|
|
||||||
(pg_stat_activity_max_tx_duration{job="shared-postgresql-sql", state=~"idle in transaction.*"}) > 600
|
|
||||||
for: 1m
|
|
||||||
labels: {severity: warning, service: shared-postgresql}
|
|
||||||
annotations:
|
|
||||||
summary: '共享 PG {{ $labels.env }} {{ $labels.member }} 有事务空闲未提交超过 10 分钟'
|
|
||||||
- alert: SharedPostgresLongTransaction
|
|
||||||
expr: pg_long_running_transactions_oldest_timestamp_seconds{job="shared-postgresql-sql"} > 3600
|
|
||||||
for: 1m
|
|
||||||
labels: {severity: warning, service: shared-postgresql}
|
|
||||||
annotations:
|
|
||||||
summary: '共享 PG {{ $labels.env }} {{ $labels.member }} 最老事务已运行超过 1 小时'
|
|
||||||
- alert: SharedPostgresDeadlocks
|
|
||||||
expr: >-
|
|
||||||
sum without(datname) (pg_stat_database_deadlocks{job="shared-postgresql-sql"}
|
|
||||||
- pg_stat_database_deadlocks{job="shared-postgresql-sql"} offset 15m) > 0
|
|
||||||
labels: {severity: warning, service: shared-postgresql}
|
|
||||||
annotations:
|
|
||||||
summary: '共享 PG {{ $labels.env }} {{ $labels.member }} 15 分钟内出现死锁'
|
|
||||||
# 每日 pgBackRest full 会请求一次 checkpoint；每小时多于 4 次通常意味着 max_wal_size 偏小。
|
|
||||||
- alert: SharedPostgresCheckpointsRequested
|
|
||||||
expr: >-
|
|
||||||
pg_stat_checkpointer_num_requested_total{job="shared-postgresql-sql"}
|
|
||||||
- pg_stat_checkpointer_num_requested_total{job="shared-postgresql-sql"} offset 1h > 4
|
|
||||||
for: 15m
|
|
||||||
labels: {severity: warning, service: shared-postgresql}
|
|
||||||
annotations:
|
|
||||||
summary: '共享 PG {{ $labels.env }} {{ $labels.member }} 频繁请求 checkpoint，max_wal_size 可能偏小'
|
|
||||||
@@ -1,18 +0,0 @@
|
|||||||
# postgres_exporter 跑在数据库主机上，按实例一个端口；env 标签供 Alertmanager 分流 dev。
|
|
||||||
apiVersion: operator.victoriametrics.com/v1beta1
|
|
||||||
kind: VMStaticScrape
|
|
||||||
metadata:
|
|
||||||
name: shared-postgresql-sql
|
|
||||||
namespace: monitoring
|
|
||||||
spec:
|
|
||||||
jobName: shared-postgresql-sql
|
|
||||||
targetEndpoints:
|
|
||||||
- targets: ['192.168.10.127:9187']
|
|
||||||
labels: {cluster: homelab-pg-prod, member: pg-laptop, env: prod}
|
|
||||||
interval: 30s
|
|
||||||
- targets: ['10.60.0.20:9187']
|
|
||||||
labels: {cluster: homelab-pg-prod, member: pg-pve1, env: prod}
|
|
||||||
interval: 30s
|
|
||||||
- targets: ['192.168.10.127:9188']
|
|
||||||
labels: {cluster: homelab-pg-dev, member: pg-laptop, env: dev}
|
|
||||||
interval: 30s
|
|
||||||
@@ -10,7 +10,7 @@ import sys
|
|||||||
import yaml
|
import yaml
|
||||||
|
|
||||||
groups = []
|
groups = []
|
||||||
for name in ("kubernetes-health", "host-health", "monitoring-delivery", "platform-controllers", "data-services", "openbao", "shared-data-maintenance", "shared-postgresql-sql"):
|
for name in ("kubernetes-health", "host-health", "monitoring-delivery", "platform-controllers", "data-services", "openbao", "shared-data-maintenance"):
|
||||||
path = pathlib.Path(sys.argv[1]) / f"{name}.yaml"
|
path = pathlib.Path(sys.argv[1]) / f"{name}.yaml"
|
||||||
groups.extend(yaml.safe_load(path.read_text())["spec"]["groups"])
|
groups.extend(yaml.safe_load(path.read_text())["spec"]["groups"])
|
||||||
pathlib.Path(sys.argv[2]).write_text(yaml.safe_dump({"groups": groups}, allow_unicode=True))
|
pathlib.Path(sys.argv[2]).write_text(yaml.safe_dump({"groups": groups}, allow_unicode=True))
|
||||||
|
|||||||
@@ -125,7 +125,6 @@ tests:
|
|||||||
instance: host:9109
|
instance: host:9109
|
||||||
severity: warning
|
severity: warning
|
||||||
service: shared-postgresql
|
service: shared-postgresql
|
||||||
env: dev
|
|
||||||
exp_annotations:
|
exp_annotations:
|
||||||
summary: 共享 PG 开发实例本地 SQL 不可用或处于只读状态
|
summary: 共享 PG 开发实例本地 SQL 不可用或处于只读状态
|
||||||
- name: dev 恢复后解除
|
- name: dev 恢复后解除
|
||||||
|
|||||||
@@ -1,195 +0,0 @@
|
|||||||
rule_files:
|
|
||||||
- rules.yaml
|
|
||||||
evaluation_interval: 1m
|
|
||||||
tests:
|
|
||||||
- name: dev SQL 不通时告警带 env=dev 以便分流
|
|
||||||
interval: 1m
|
|
||||||
input_series:
|
|
||||||
- series: pg_up{job="shared-postgresql-sql",instance="h:9188",cluster="homelab-pg-dev",member="pg-laptop",env="dev"}
|
|
||||||
values: '0x5'
|
|
||||||
alert_rule_test:
|
|
||||||
- eval_time: 5m
|
|
||||||
alertname: SharedPostgresSqlUnreachable
|
|
||||||
exp_alerts:
|
|
||||||
- exp_labels: {job: shared-postgresql-sql, instance: 'h:9188', cluster: homelab-pg-dev, member: pg-laptop, env: dev, severity: warning, service: shared-postgresql}
|
|
||||||
exp_annotations:
|
|
||||||
summary: 共享 PG dev pg-laptop 无法以监控角色执行 SQL
|
|
||||||
- name: 回放落后持续超过 1 MiB 告警
|
|
||||||
interval: 1m
|
|
||||||
input_series:
|
|
||||||
- series: pg_stat_replication_pg_wal_lsn_diff{job="shared-postgresql-sql",instance="p:9187",member="pg-pve1",env="prod",application_name="pg-laptop"}
|
|
||||||
values: '2097152x10'
|
|
||||||
alert_rule_test:
|
|
||||||
- eval_time: 8m
|
|
||||||
alertname: SharedPostgresReplicationLagging
|
|
||||||
exp_alerts:
|
|
||||||
- exp_labels: {job: shared-postgresql-sql, instance: 'p:9187', member: pg-pve1, env: prod, application_name: pg-laptop, severity: warning, service: shared-postgresql}
|
|
||||||
exp_annotations:
|
|
||||||
summary: 共享 PG prod 副本 pg-laptop 回放落后超过 1 MiB，已不满足自动切换条件
|
|
||||||
- name: 短暂落后不告警
|
|
||||||
interval: 1m
|
|
||||||
input_series:
|
|
||||||
- series: pg_stat_replication_pg_wal_lsn_diff{job="shared-postgresql-sql",instance="p:9187",env="prod",application_name="pg-laptop"}
|
|
||||||
values: '2097152x2 0x8'
|
|
||||||
alert_rule_test:
|
|
||||||
- eval_time: 8m
|
|
||||||
alertname: SharedPostgresReplicationLagging
|
|
||||||
exp_alerts: []
|
|
||||||
- name: 复制槽积压只在主库告警
|
|
||||||
interval: 1m
|
|
||||||
input_series:
|
|
||||||
- series: pg_replication_slots_pg_wal_lsn_diff{job="shared-postgresql-sql",instance="p:9187",env="prod",slot_name="pg_laptop"}
|
|
||||||
values: '2147483648x15'
|
|
||||||
- series: pg_replication_is_replica{job="shared-postgresql-sql",instance="p:9187",env="prod"}
|
|
||||||
values: '0x15'
|
|
||||||
- series: pg_replication_slots_pg_wal_lsn_diff{job="shared-postgresql-sql",instance="r:9187",env="prod",slot_name="pg_pve1"}
|
|
||||||
values: '2147483648x15'
|
|
||||||
- series: pg_replication_is_replica{job="shared-postgresql-sql",instance="r:9187",env="prod"}
|
|
||||||
values: '1x15'
|
|
||||||
alert_rule_test:
|
|
||||||
- eval_time: 12m
|
|
||||||
alertname: SharedPostgresReplicationSlotRetention
|
|
||||||
exp_alerts:
|
|
||||||
- exp_labels: {job: shared-postgresql-sql, instance: 'p:9187', env: prod, slot_name: pg_laptop, severity: warning, service: shared-postgresql}
|
|
||||||
exp_annotations:
|
|
||||||
summary: 共享 PG prod 复制槽 pg_laptop 保留 WAL 超过 1 GiB
|
|
||||||
- name: 回卷 critical 时不重复 warning
|
|
||||||
interval: 1m
|
|
||||||
input_series:
|
|
||||||
- series: pg_database_wraparound_age_datfrozenxid_seconds{job="shared-postgresql-sql",instance="p:9187",member="pg-pve1",env="prod",datname="app"}
|
|
||||||
values: '1700000000x15'
|
|
||||||
- series: pg_database_wraparound_age_datminmxid_seconds{job="shared-postgresql-sql",instance="p:9187",member="pg-pve1",env="prod",datname="app"}
|
|
||||||
values: '1200000000x15'
|
|
||||||
alert_rule_test:
|
|
||||||
- eval_time: 12m
|
|
||||||
alertname: SharedPostgresWraparoundRisk
|
|
||||||
exp_alerts: []
|
|
||||||
- eval_time: 12m
|
|
||||||
alertname: SharedPostgresWraparoundImminent
|
|
||||||
exp_alerts:
|
|
||||||
- exp_labels: {job: shared-postgresql-sql, instance: 'p:9187', member: pg-pve1, env: prod, severity: critical, service: shared-postgresql}
|
|
||||||
exp_annotations:
|
|
||||||
summary: 共享 PG prod pg-pve1 XID/MXID 接近回卷停写，需立即 VACUUM FREEZE
|
|
||||||
- name: 回卷 warning
|
|
||||||
interval: 1m
|
|
||||||
input_series:
|
|
||||||
- series: pg_database_wraparound_age_datfrozenxid_seconds{job="shared-postgresql-sql",instance="p:9187",member="pg-pve1",env="prod",datname="app"}
|
|
||||||
values: '1200000000x15'
|
|
||||||
alert_rule_test:
|
|
||||||
- eval_time: 12m
|
|
||||||
alertname: SharedPostgresWraparoundRisk
|
|
||||||
exp_alerts:
|
|
||||||
- exp_labels: {job: shared-postgresql-sql, instance: 'p:9187', member: pg-pve1, env: prod, severity: warning, service: shared-postgresql}
|
|
||||||
exp_annotations:
|
|
||||||
summary: 共享 PG prod pg-pve1 XID/MXID 年龄超过 10 亿
|
|
||||||
- name: 只计客户端连接
|
|
||||||
interval: 1m
|
|
||||||
input_series:
|
|
||||||
- series: pg_stat_activity_count{job="shared-postgresql-sql",instance="p:9187",member="pg-pve1",env="prod",datname="app",state="active",usename="a",application_name="x",backend_type="client backend",wait_event_type="",wait_event=""}
|
|
||||||
values: '50x10'
|
|
||||||
- series: pg_stat_activity_count{job="shared-postgresql-sql",instance="p:9187",member="pg-pve1",env="prod",datname="app",state="idle",usename="b",application_name="y",backend_type="client backend",wait_event_type="",wait_event=""}
|
|
||||||
values: '20x10'
|
|
||||||
- series: pg_stat_activity_count{job="shared-postgresql-sql",instance="p:9187",member="pg-pve1",env="prod",datname="",state="",usename="",application_name="",backend_type="autovacuum worker",wait_event_type="",wait_event=""}
|
|
||||||
values: '30x10'
|
|
||||||
- series: pg_settings_max_connections{job="shared-postgresql-sql",instance="p:9187",member="pg-pve1",env="prod"}
|
|
||||||
values: '80x10'
|
|
||||||
alert_rule_test:
|
|
||||||
- eval_time: 8m
|
|
||||||
alertname: SharedPostgresConnectionsHigh
|
|
||||||
exp_alerts:
|
|
||||||
- exp_labels: {job: shared-postgresql-sql, instance: 'p:9187', member: pg-pve1, env: prod, severity: warning, service: shared-postgresql}
|
|
||||||
exp_annotations:
|
|
||||||
summary: 共享 PG prod pg-pve1 客户端连接超过 max_connections 的 80%
|
|
||||||
- name: 后台进程不计入连接使用率
|
|
||||||
interval: 1m
|
|
||||||
input_series:
|
|
||||||
- series: pg_stat_activity_count{job="shared-postgresql-sql",instance="p:9187",env="prod",datname="app",state="active",usename="a",application_name="x",backend_type="client backend",wait_event_type="",wait_event=""}
|
|
||||||
values: '40x10'
|
|
||||||
- series: pg_stat_activity_count{job="shared-postgresql-sql",instance="p:9187",env="prod",datname="",state="",usename="",application_name="",backend_type="autovacuum worker",wait_event_type="",wait_event=""}
|
|
||||||
values: '40x10'
|
|
||||||
- series: pg_settings_max_connections{job="shared-postgresql-sql",instance="p:9187",env="prod"}
|
|
||||||
values: '80x10'
|
|
||||||
alert_rule_test:
|
|
||||||
- eval_time: 8m
|
|
||||||
alertname: SharedPostgresConnectionsHigh
|
|
||||||
exp_alerts: []
|
|
||||||
- name: 死锁计数增长告警
|
|
||||||
interval: 1m
|
|
||||||
input_series:
|
|
||||||
- series: pg_stat_database_deadlocks{job="shared-postgresql-sql",instance="p:9187",member="pg-pve1",env="prod",datname="app"}
|
|
||||||
values: '0x20 1x5'
|
|
||||||
alert_rule_test:
|
|
||||||
- eval_time: 22m
|
|
||||||
alertname: SharedPostgresDeadlocks
|
|
||||||
exp_alerts:
|
|
||||||
- exp_labels: {job: shared-postgresql-sql, instance: 'p:9187', member: pg-pve1, env: prod, severity: warning, service: shared-postgresql}
|
|
||||||
exp_annotations:
|
|
||||||
summary: 共享 PG prod pg-pve1 15 分钟内出现死锁
|
|
||||||
- name: 空闲事务超过 10 分钟
|
|
||||||
interval: 1m
|
|
||||||
input_series:
|
|
||||||
- series: pg_stat_activity_max_tx_duration{job="shared-postgresql-sql",instance="p:9187",member="pg-pve1",env="prod",datname="app",state="idle in transaction",usename="a",application_name="x",backend_type="client backend",wait_event_type="Client",wait_event="ClientRead"}
|
|
||||||
values: '700x5'
|
|
||||||
alert_rule_test:
|
|
||||||
- eval_time: 3m
|
|
||||||
alertname: SharedPostgresIdleInTransaction
|
|
||||||
exp_alerts:
|
|
||||||
- exp_labels: {job: shared-postgresql-sql, instance: 'p:9187', member: pg-pve1, env: prod, severity: warning, service: shared-postgresql}
|
|
||||||
exp_annotations:
|
|
||||||
summary: 共享 PG prod pg-pve1 有事务空闲未提交超过 10 分钟
|
|
||||||
- name: 主库落在 pve1 持续 15 分钟提示回切
|
|
||||||
interval: 1m
|
|
||||||
input_series:
|
|
||||||
- series: pg_replication_is_replica{job="shared-postgresql-sql",instance="10.60.0.20:9187",cluster="homelab-pg-prod",member="pg-pve1",env="prod"}
|
|
||||||
values: '0x20'
|
|
||||||
- series: pg_replication_is_replica{job="shared-postgresql-sql",instance="192.168.10.127:9187",cluster="homelab-pg-prod",member="pg-laptop",env="prod"}
|
|
||||||
values: '1x20'
|
|
||||||
alert_rule_test:
|
|
||||||
- eval_time: 10m
|
|
||||||
alertname: SharedPostgresPrimaryOnStandbyHost
|
|
||||||
exp_alerts: []
|
|
||||||
- eval_time: 16m
|
|
||||||
alertname: SharedPostgresPrimaryOnStandbyHost
|
|
||||||
exp_alerts:
|
|
||||||
- exp_labels: {job: shared-postgresql-sql, instance: '10.60.0.20:9187', cluster: homelab-pg-prod, member: pg-pve1, env: prod, severity: warning, service: shared-postgresql}
|
|
||||||
exp_annotations:
|
|
||||||
summary: 共享 PG 生产主库运行在 pg-pve1（HDD），请确认切换原因并计划回切 laptop
|
|
||||||
- name: 主库在 laptop 不提示
|
|
||||||
interval: 1m
|
|
||||||
input_series:
|
|
||||||
- series: pg_replication_is_replica{job="shared-postgresql-sql",instance="10.60.0.20:9187",member="pg-pve1",env="prod"}
|
|
||||||
values: '1x20'
|
|
||||||
alert_rule_test:
|
|
||||||
- eval_time: 16m
|
|
||||||
alertname: SharedPostgresPrimaryOnStandbyHost
|
|
||||||
exp_alerts: []
|
|
||||||
- name: 新出现的计数器序列带历史累计值不误报归档失败
|
|
||||||
interval: 1m
|
|
||||||
input_series:
|
|
||||||
- series: pg_stat_archiver_failed_count{job="shared-postgresql-sql",instance="l:9187",member="pg-laptop",env="prod"}
|
|
||||||
values: '_x20 9x20'
|
|
||||||
alert_rule_test:
|
|
||||||
- eval_time: 25m
|
|
||||||
alertname: SharedPostgresWalArchiveFailing
|
|
||||||
exp_alerts: []
|
|
||||||
- name: 归档失败计数增长告警
|
|
||||||
interval: 1m
|
|
||||||
input_series:
|
|
||||||
- series: pg_stat_archiver_failed_count{job="shared-postgresql-sql",instance="l:9187",member="pg-laptop",env="prod"}
|
|
||||||
values: '9x20 10x5'
|
|
||||||
alert_rule_test:
|
|
||||||
- eval_time: 22m
|
|
||||||
alertname: SharedPostgresWalArchiveFailing
|
|
||||||
exp_alerts:
|
|
||||||
- exp_labels: {job: shared-postgresql-sql, instance: 'l:9187', member: pg-laptop, env: prod, severity: warning, service: shared-postgresql}
|
|
||||||
exp_annotations:
|
|
||||||
summary: 共享 PG prod pg-laptop WAL 归档失败，PITR 链可能中断
|
|
||||||
- name: 计数器重置不误报 checkpoint
|
|
||||||
interval: 1m
|
|
||||||
input_series:
|
|
||||||
- series: pg_stat_checkpointer_num_requested_total{job="shared-postgresql-sql",instance="l:9187",member="pg-laptop",env="prod"}
|
|
||||||
values: '16x60 0x30'
|
|
||||||
alert_rule_test:
|
|
||||||
- eval_time: 85m
|
|
||||||
alertname: SharedPostgresCheckpointsRequested
|
|
||||||
exp_alerts: []
|
|
||||||
@@ -16,10 +16,6 @@ spec:
|
|||||||
group_interval: 5m
|
group_interval: 5m
|
||||||
repeat_interval: 4h
|
repeat_interval: 4h
|
||||||
routes:
|
routes:
|
||||||
# dev 必须排第一且不 continue：否则 dev 告警会被后面的 severity 路由送进生产频道。
|
|
||||||
- receiver: telegram-dev
|
|
||||||
matchers:
|
|
||||||
- env="dev"
|
|
||||||
- receiver: telegram
|
- receiver: telegram
|
||||||
matchers:
|
matchers:
|
||||||
- severity="critical"
|
- severity="critical"
|
||||||
@@ -41,11 +37,6 @@ spec:
|
|||||||
- bot_token_file: /etc/vm/secrets/alertmanager-telegram/telegram_bot_token
|
- bot_token_file: /etc/vm/secrets/alertmanager-telegram/telegram_bot_token
|
||||||
chat_id: -1003956377923
|
chat_id: -1003956377923
|
||||||
send_resolved: true
|
send_resolved: true
|
||||||
- name: telegram-dev
|
|
||||||
telegram_configs:
|
|
||||||
- bot_token_file: /etc/vm/secrets/alertmanager-telegram/telegram_bot_token
|
|
||||||
chat_id: -1003651477106
|
|
||||||
send_resolved: true
|
|
||||||
resources:
|
resources:
|
||||||
requests:
|
requests:
|
||||||
cpu: 25m
|
cpu: 25m
|
||||||
|
|||||||
Reference in New Issue
Block a user