Compare commits
7
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 panxiao81
|
f44d4349a7 | ||
|
panxiao81
|
9368ac559e | ||
 panxiao81andClaude Opus 5.5
|
fef75ad34f
|
||
|
panxiao81andClaude Opus 5.5
|
0c7728fe0b
|
||
 flux-bot
|
a4e2c60346 | ||
|
panxiao81
|
746d326292 | ||
|
panxiao81andClaude Opus 5.5
|
4a6b5c8a1d
|
@@ -1,32 +0,0 @@
|
|||||||
---
|
|
||||||
name: dind-fuse-image
|
|
||||||
|
|
||||||
on:
|
|
||||||
workflow_dispatch:
|
|
||||||
push:
|
|
||||||
branches: [main]
|
|
||||||
paths:
|
|
||||||
- platform/gitea-runner/images/dind-fuse/**
|
|
||||||
- .gitea/workflows/dind-fuse-image.yml
|
|
||||||
|
|
||||||
jobs:
|
|
||||||
publish:
|
|
||||||
runs-on: [self-hosted, pod]
|
|
||||||
steps:
|
|
||||||
- uses: actions/checkout@v4
|
|
||||||
|
|
||||||
- name: Log in to the Gitea container registry
|
|
||||||
uses: docker/login-action@v3
|
|
||||||
with:
|
|
||||||
registry: git.ddupan.top
|
|
||||||
username: ${{ gitea.actor }}
|
|
||||||
password: ${{ secrets.REGISTRY_TOKEN }}
|
|
||||||
|
|
||||||
- name: Build and publish
|
|
||||||
uses: docker/build-push-action@v6
|
|
||||||
with:
|
|
||||||
context: platform/gitea-runner/images/dind-fuse
|
|
||||||
push: true
|
|
||||||
tags: |
|
|
||||||
git.ddupan.top/panxiao81/dind-fuse:29.7.1
|
|
||||||
git.ddupan.top/panxiao81/dind-fuse:latest
|
|
||||||
@@ -1,24 +0,0 @@
|
|||||||
---
|
|
||||||
name: kata-runner-smoke
|
|
||||||
|
|
||||||
on:
|
|
||||||
workflow_dispatch:
|
|
||||||
push:
|
|
||||||
branches:
|
|
||||||
- poc/kata-runner-smoke
|
|
||||||
paths:
|
|
||||||
- .gitea/workflows/kata-runner-smoke.yml
|
|
||||||
|
|
||||||
jobs:
|
|
||||||
smoke:
|
|
||||||
runs-on: [self-hosted, pod]
|
|
||||||
steps:
|
|
||||||
- name: Verify isolated job environment
|
|
||||||
shell: sh
|
|
||||||
run: |
|
|
||||||
set -eu
|
|
||||||
uname -a
|
|
||||||
grep -q '^ID=alpine$' /etc/os-release
|
|
||||||
test -f /.dockerenv
|
|
||||||
test ! -e /dev/kvm
|
|
||||||
printf 'kata ephemeral runner job ok\n'
|
|
||||||
@@ -1,63 +0,0 @@
|
|||||||
---
|
|
||||||
name: kind-on-kata-smoke
|
|
||||||
|
|
||||||
on:
|
|
||||||
push:
|
|
||||||
branches:
|
|
||||||
- poc/kind-on-kata
|
|
||||||
paths:
|
|
||||||
- .gitea/workflows/kind-on-kata-smoke.yml
|
|
||||||
workflow_dispatch:
|
|
||||||
|
|
||||||
jobs:
|
|
||||||
smoke:
|
|
||||||
runs-on: [self-hosted, pod]
|
|
||||||
steps:
|
|
||||||
- name: Create nested kind cluster
|
|
||||||
shell: sh
|
|
||||||
env:
|
|
||||||
KIND_VERSION: v0.27.0
|
|
||||||
KIND_NODE_IMAGE: kindest/node:v1.32.2@sha256:f226345927d7e348497136874b6d207e0b32cc52154ad8323129352923a3142f
|
|
||||||
run: |
|
|
||||||
set -eu
|
|
||||||
apk add --no-cache ca-certificates curl docker-cli
|
|
||||||
curl --retry 5 --retry-all-errors --connect-timeout 15 -fsSLo /tmp/kind \
|
|
||||||
"https://kind.sigs.k8s.io/dl/${KIND_VERSION}/kind-linux-amd64"
|
|
||||||
curl --retry 5 --retry-all-errors --connect-timeout 15 -fsSLo /tmp/kind.sha256sum \
|
|
||||||
"https://kind.sigs.k8s.io/dl/${KIND_VERSION}/kind-linux-amd64.sha256sum"
|
|
||||||
expected="$(awk '{print $1}' /tmp/kind.sha256sum)"
|
|
||||||
printf '%s %s\n' "$expected" /tmp/kind | sha256sum -c -
|
|
||||||
install -m 0755 /tmp/kind /usr/local/bin/kind
|
|
||||||
docker info --format 'kernel={{.KernelVersion}} driver={{.Driver}}'
|
|
||||||
test "$(docker info --format '{{.Driver}}')" = overlay2
|
|
||||||
|
|
||||||
cleanup() {
|
|
||||||
kind delete cluster --name nested >/dev/null 2>&1 || true
|
|
||||||
}
|
|
||||||
trap cleanup EXIT
|
|
||||||
|
|
||||||
cat >/tmp/kind-config.yaml <<'EOF'
|
|
||||||
kind: Cluster
|
|
||||||
apiVersion: kind.x-k8s.io/v1alpha4
|
|
||||||
name: nested
|
|
||||||
nodes:
|
|
||||||
- role: control-plane
|
|
||||||
extraMounts:
|
|
||||||
- hostPath: /dev/kmsg
|
|
||||||
containerPath: /dev/kmsg
|
|
||||||
EOF
|
|
||||||
kind create cluster -v 9 --retain --config /tmp/kind-config.yaml --image "$KIND_NODE_IMAGE" --wait 5m
|
|
||||||
docker exec nested-control-plane kubectl \
|
|
||||||
--kubeconfig=/etc/kubernetes/admin.conf wait \
|
|
||||||
--for=condition=Ready node/nested-control-plane --timeout=2m
|
|
||||||
docker exec nested-control-plane kubectl \
|
|
||||||
--kubeconfig=/etc/kubernetes/admin.conf run smoke \
|
|
||||||
--image=docker.io/library/busybox:1.37 --restart=Never \
|
|
||||||
--command -- sh -c 'echo kind-on-kata-ok'
|
|
||||||
docker exec nested-control-plane kubectl \
|
|
||||||
--kubeconfig=/etc/kubernetes/admin.conf wait \
|
|
||||||
--for=jsonpath='{.status.phase}'=Succeeded pod/smoke --timeout=2m
|
|
||||||
test "$(docker exec nested-control-plane kubectl \
|
|
||||||
--kubeconfig=/etc/kubernetes/admin.conf logs smoke)" = kind-on-kata-ok
|
|
||||||
kind delete cluster --name nested
|
|
||||||
trap - EXIT
|
|
||||||
@@ -8,8 +8,15 @@ metadata:
|
|||||||
backstage.io/kubernetes-id: homelab-backstage
|
backstage.io/kubernetes-id: homelab-backstage
|
||||||
spec:
|
spec:
|
||||||
replicas: 1
|
replicas: 1
|
||||||
|
# Backstage is multi-instance safe (scheduler locks, event bus, and auth keys
|
||||||
|
# live in PostgreSQL), so surge one new pod and keep the old one serving
|
||||||
|
# until it is ready. Migrations run on the new pod while the old one still
|
||||||
|
# serves; revisit if an upgrade ships a backward-incompatible migration.
|
||||||
strategy:
|
strategy:
|
||||||
type: Recreate
|
type: RollingUpdate
|
||||||
|
rollingUpdate:
|
||||||
|
maxSurge: 1
|
||||||
|
maxUnavailable: 0
|
||||||
selector:
|
selector:
|
||||||
matchLabels:
|
matchLabels:
|
||||||
app.kubernetes.io/name: backstage
|
app.kubernetes.io/name: backstage
|
||||||
@@ -27,7 +34,7 @@ spec:
|
|||||||
type: RuntimeDefault
|
type: RuntimeDefault
|
||||||
containers:
|
containers:
|
||||||
- name: backstage
|
- name: backstage
|
||||||
image: zot.ad.ddupan.top/panxiao81/backstage:latest@sha256:da55be5c2f5c8b33de2d87ee0ef02edeacf52a349c8b19a5fac9be4152b2723d # {"$imagepolicy": "flux-system:backstage"}
|
image: zot.ad.ddupan.top/panxiao81/backstage:latest@sha256:476b40ea5cdf7301edb4f5dab5a012686a0009d1b0f1fce21fab34b969543ebb # {"$imagepolicy": "flux-system:backstage"}
|
||||||
imagePullPolicy: IfNotPresent
|
imagePullPolicy: IfNotPresent
|
||||||
env:
|
env:
|
||||||
- name: BACKSTAGE_BASE_URL
|
- name: BACKSTAGE_BASE_URL
|
||||||
|
|||||||
@@ -27,9 +27,10 @@ Hydra 回调为 `/oauth2/start`、`/oauth2/consent`、`/oauth2/logout`；默认
|
|||||||
|
|
||||||
### 客户端管理 `/console`
|
### 客户端管理 `/console`
|
||||||
|
|
||||||
iam-login 的客户端管理 API 只接受 Hydra 签发、带 `iam.clients.manage` scope 的 access token；
|
iam-login 的客户端管理 API 是普通 resource server：只接受 Hydra 签发、`aud` 含 `iam-login`
|
||||||
`/console` 是调用该 API 的前端，在 Hydra 中登记为普通 **public** 客户端 `iam-admin-ui`。
|
且 `groups` 含管理组的 access token。`/console` 是调用该 API 的前端，在 Hydra 中登记为普通
|
||||||
该 scope 只在 consent 时发给 `iam-admin-ui` 且直接属于管理组的用户，规则在 iam-login。
|
**public** 客户端 `iam-admin-ui`；只有它被允许申请 audience `iam-login`（Hydra 按客户端登记的
|
||||||
|
audience 白名单执行），所以其他应用拿到的管理员 token 不能调用该 API。
|
||||||
|
|
||||||
浏览器直接向 Hydra 换取 token，因此 `hydra.yaml` 为 public 端口开启 CORS，只允许开发实例
|
浏览器直接向 Hydra 换取 token，因此 `hydra.yaml` 为 public 端口开启 CORS，只允许开发实例
|
||||||
origin；不放行 cookie 凭据。Gitea 等服务端客户端不受影响。
|
origin；不放行 cookie 凭据。Gitea 等服务端客户端不受影响。
|
||||||
@@ -42,11 +43,11 @@ curl -fsS -X POST http://127.0.0.1:18445/admin/clients -H 'Content-Type: applica
|
|||||||
"client_id": "iam-admin-ui", "client_name": "IAM 管理",
|
"client_id": "iam-admin-ui", "client_name": "IAM 管理",
|
||||||
"redirect_uris": ["https://laptop.tail7e769.ts.net:18082/console/callback"],
|
"redirect_uris": ["https://laptop.tail7e769.ts.net:18082/console/callback"],
|
||||||
"grant_types": ["authorization_code"], "response_types": ["code"],
|
"grant_types": ["authorization_code"], "response_types": ["code"],
|
||||||
"scope": "openid iam.clients.manage", "token_endpoint_auth_method": "none",
|
"scope": "openid groups", "audience": ["iam-login"], "token_endpoint_auth_method": "none",
|
||||||
"subject_type": "public", "metadata": {"iam_login_enabled": true}}'
|
"subject_type": "public", "metadata": {"iam_login_enabled": true}}'
|
||||||
```
|
```
|
||||||
|
|
||||||
开发实例另需 `iam.clients.admin-ui-client-id=iam-admin-ui`；未配置时 `/console` 不提供，API
|
开发实例另需 `iam.clients.admin-ui-client-id=iam-admin-ui` 与 `iam.clients.admin-groups`（组名）；未配置时 `/console` 不提供，API
|
||||||
无法获得可用 token（fail closed）。
|
无法获得可用 token（fail closed）。
|
||||||
|
|
||||||
从 Gitea `/user/oauth2/hydra` 发起，应进入新密码/Passkey 页，确认授权后返回原账号，核对
|
从 Gitea `/user/oauth2/hydra` 发起，应进入新密码/Passkey 页，确认授权后返回原账号，核对
|
||||||
|
|||||||
+5
-2
@@ -26,8 +26,11 @@ SPIRE 认证链路已经验证，但当前没有常驻 publisher 或删除授权
|
|||||||
`/var/lib/registry` 是 `emptyDir`，仅用于运行时本地工作数据。
|
`/var/lib/registry` 是 `emptyDir`，仅用于运行时本地工作数据。
|
||||||
|
|
||||||
两个单副本实例共用同一 bucket 和前缀：`zot` 负责鉴权写入，`zot-reader` 负责匿名
|
两个单副本实例共用同一 bucket 和前缀：`zot` 负责鉴权写入，`zot-reader` 负责匿名
|
||||||
读取。关闭跨仓库 dedupe，不额外部署 Redis/DynamoDB 缓存。只有写入实例启用 GC，
|
读取。关闭跨仓库 dedupe，不额外部署 Redis/DynamoDB 缓存。只有写入实例启用 GC。
|
||||||
暂不配置自动删除已发布版本的 retention policy。增加副本、启用 dedupe 或搜索等
|
retention policy 只针对 `panxiao81/backstage`：仅保留 `latest`，历史 sha tag 与失去
|
||||||
|
tag 的 digest 在 24h 后回收（dev 镜像由 Flux 跟踪 `latest` 的 digest）。其余仓库由
|
||||||
|
`**` 兜底策略保留全部 tag，行为与未配置 retention 时一致；新增按仓库的清理规则时，
|
||||||
|
必须放在兜底策略之前，否则不会生效。增加副本、启用 dedupe 或搜索等
|
||||||
扩展前，需要重新检查共享元数据与缓存的持久化要求。
|
扩展前，需要重新检查共享元数据与缓存的持久化要求。
|
||||||
|
|
||||||
凭据链路：
|
凭据链路：
|
||||||
|
|||||||
@@ -27,6 +27,21 @@ configFiles:
|
|||||||
"gc": true,
|
"gc": true,
|
||||||
"gcDelay": "24h",
|
"gcDelay": "24h",
|
||||||
"gcInterval": "24h",
|
"gcInterval": "24h",
|
||||||
|
"retention": {
|
||||||
|
"delay": "24h",
|
||||||
|
"policies": [
|
||||||
|
{
|
||||||
|
"repositories": ["panxiao81/backstage"],
|
||||||
|
"deleteUntagged": true,
|
||||||
|
"keepTags": [{ "patterns": ["^latest$"] }]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"repositories": ["**"],
|
||||||
|
"deleteUntagged": true,
|
||||||
|
"keepTags": [{ "patterns": [".*"] }]
|
||||||
|
}
|
||||||
|
]
|
||||||
|
},
|
||||||
"storageDriver": {
|
"storageDriver": {
|
||||||
"name": "s3",
|
"name": "s3",
|
||||||
"region": "us-east-1",
|
"region": "us-east-1",
|
||||||
|
|||||||
@@ -1,72 +0,0 @@
|
|||||||
# kata-lab
|
|
||||||
|
|
||||||
`kata-lab` 是位于 Proxmox `pve2` 的可销毁验证环境，用来验证 nested KVM、k3s、
|
|
||||||
Kata Containers，以及后续一 job 一 Kata Pod 的 Gitea Runner。它不是 dev/stg
|
|
||||||
长期服务，也不承载持久数据。
|
|
||||||
|
|
||||||
> 状态：VMID 147 已于 2026-09-14 销毁，Terraform state 为空。目录保留首次验证结果
|
|
||||||
> 与可复现配置；后续验证转向跨 PVE 节点的轻量 LXC worker 方案。
|
|
||||||
|
|
||||||
## Terraform bootstrap
|
|
||||||
|
|
||||||
1. 将 Proxmox API token 写入被 Git 忽略且权限为 `0600` 的
|
|
||||||
`terraform/credentials.auto.tfvars`。
|
|
||||||
2. 取得内部 CA，并仅为当前 Terraform 进程设置 Go TLS trust：
|
|
||||||
|
|
||||||
```bash
|
|
||||||
curl -fsSL https://bao.ad.ddupan.top:8200/v1/pki/ca/pem \
|
|
||||||
-o /tmp/kata-lab-ddupan-ca.pem
|
|
||||||
export SSL_CERT_FILE=/tmp/kata-lab-ddupan-ca.pem
|
|
||||||
```
|
|
||||||
|
|
||||||
3. 初始化并审阅 plan：
|
|
||||||
|
|
||||||
```bash
|
|
||||||
cd infrastructure/kata-lab/terraform
|
|
||||||
terraform init
|
|
||||||
terraform plan
|
|
||||||
```
|
|
||||||
|
|
||||||
Terraform 使用 `laptop:import/noble-server-cloudimg-amd64.qcow2`（40 GiB
|
|
||||||
virtual size），在
|
|
||||||
`pve-rg-hdd` 创建 VM root disk，并把 cloud-init snippet 放到 `laptop`
|
|
||||||
datastore。VM root disk 设为 41 GiB，以容纳 PVE import 的块对齐增量。VM 使用
|
|
||||||
DHCP、`cpu.type = host`、4 vCPU 和 4 GiB 内存；cloud-init
|
|
||||||
安装 `qemu-guest-agent`、单节点 k3s，并记录 `/dev/kvm` 检查结果。
|
|
||||||
|
|
||||||
## Kata Containers
|
|
||||||
|
|
||||||
Ansible 使用 Kata Containers 4.1.0 官方 `kata-deploy` chart。配置明确选择 k3s，
|
|
||||||
只安装 `qemu-runtime-rs`，不安装额外 snapshotter，并采用 `job` 模式，安装结束后
|
|
||||||
不保留 privileged DaemonSet。默认 `RuntimeClass` 名为 `kata`。
|
|
||||||
|
|
||||||
Terraform 输出的 DHCP 地址变化时，先更新 `ansible/inventory/hosts.yml`，然后运行：
|
|
||||||
|
|
||||||
```bash
|
|
||||||
cd infrastructure/kata-lab/ansible
|
|
||||||
ansible-playbook kata.yml
|
|
||||||
ansible-playbook verify.yml
|
|
||||||
```
|
|
||||||
|
|
||||||
`verify.yml` 首先创建一个 `runtimeClassName: kata` 的短生命周期 Pod，并确认 Pod 内
|
|
||||||
看到的 Kata guest kernel 与 k3s node 的宿主 kernel 不同。随后它在另一个 Kata Pod
|
|
||||||
内启动 privileged `dockerd` sidecar，让普通 Docker client 容器通过共享网络命名空间
|
|
||||||
的 `127.0.0.1:2375` 运行一个嵌套容器。这对应后续 Gitea Runner 的目标形态：runner
|
|
||||||
直接执行 job，只在需要 Docker API 时连接同 Pod 的 daemon，而不把 Docker executor
|
|
||||||
作为 job 的外层。Kata 下共享 volume 中的 Unix socket 文件虽然双方可见，但不能跨
|
|
||||||
容器连接，因此不能在这里沿用普通 Pod 常见的 `/var/run/docker.sock` 方案；loopback
|
|
||||||
端口没有通过 Service 暴露到 Pod 外。
|
|
||||||
验证 Pod 会保留以供排查；再次运行时会先替换它们。
|
|
||||||
|
|
||||||
2026-09-13 的首次验证结果：Kata guest kernel 为 `6.18.35`，node host kernel 为
|
|
||||||
`6.8.0-90-generic`；sidecar daemon 成功运行 `busybox:1.37` 嵌套容器。Docker 27.5.1
|
|
||||||
在当前 Kata guest 中无法挂载 overlay2，自动回退到 `vfs`；功能验证通过，但正式用于
|
|
||||||
CI 前需要解决或接受其镜像层复制与磁盘性能开销。
|
|
||||||
|
|
||||||
不要提交 `credentials.auto.tfvars`、Terraform state 或 saved plan。销毁前确认目标
|
|
||||||
仍然是 VMID 147 / `kata-lab`：
|
|
||||||
|
|
||||||
```bash
|
|
||||||
terraform plan -destroy
|
|
||||||
terraform destroy
|
|
||||||
```
|
|
||||||
@@ -1,10 +0,0 @@
|
|||||||
[defaults]
|
|
||||||
inventory = inventory/hosts.yml
|
|
||||||
host_key_checking = True
|
|
||||||
interpreter_python = auto_silent
|
|
||||||
retry_files_enabled = False
|
|
||||||
local_tmp = /tmp/ansible-kata-lab-local
|
|
||||||
remote_tmp = /tmp/ansible-kata-lab-remote
|
|
||||||
|
|
||||||
[ssh_connection]
|
|
||||||
pipelining = True
|
|
||||||
@@ -1,8 +0,0 @@
|
|||||||
---
|
|
||||||
all:
|
|
||||||
hosts:
|
|
||||||
kata-lab:
|
|
||||||
ansible_host: 192.168.10.13
|
|
||||||
ansible_user: ansible
|
|
||||||
vars:
|
|
||||||
ansible_become: true
|
|
||||||
@@ -1,88 +0,0 @@
|
|||||||
---
|
|
||||||
- name: Install Kata Containers in the disposable k3s lab
|
|
||||||
hosts: kata-lab
|
|
||||||
gather_facts: false
|
|
||||||
vars:
|
|
||||||
kata_version: "4.1.0"
|
|
||||||
kata_chart_url: >-
|
|
||||||
https://github.com/kata-containers/kata-containers/releases/download/{{ kata_version }}/kata-deploy-{{ kata_version }}.tgz
|
|
||||||
tasks:
|
|
||||||
- name: Check nested KVM is usable
|
|
||||||
ansible.builtin.stat:
|
|
||||||
path: /dev/kvm
|
|
||||||
register: kata_kvm
|
|
||||||
|
|
||||||
- name: Require nested KVM
|
|
||||||
ansible.builtin.assert:
|
|
||||||
that:
|
|
||||||
- kata_kvm.stat.exists
|
|
||||||
- kata_kvm.stat.ischr
|
|
||||||
fail_msg: /dev/kvm is unavailable; Kata QEMU cannot start efficiently
|
|
||||||
|
|
||||||
- name: Install Kata through the k3s Helm controller
|
|
||||||
ansible.builtin.copy:
|
|
||||||
dest: /var/lib/rancher/k3s/server/manifests/kata-deploy.yaml
|
|
||||||
owner: root
|
|
||||||
group: root
|
|
||||||
mode: "0644"
|
|
||||||
content: |
|
|
||||||
apiVersion: helm.cattle.io/v1
|
|
||||||
kind: HelmChart
|
|
||||||
metadata:
|
|
||||||
name: kata-deploy
|
|
||||||
namespace: kube-system
|
|
||||||
spec:
|
|
||||||
chart: {{ kata_chart_url }}
|
|
||||||
targetNamespace: kata-system
|
|
||||||
createNamespace: true
|
|
||||||
timeout: 15m
|
|
||||||
failurePolicy: abort
|
|
||||||
valuesContent: |-
|
|
||||||
deploymentMode: job
|
|
||||||
k8sDistribution: k3s
|
|
||||||
snapshotter:
|
|
||||||
setup: []
|
|
||||||
shims:
|
|
||||||
disableAll: true
|
|
||||||
qemu-runtime-rs:
|
|
||||||
enabled: true
|
|
||||||
runtimeClasses:
|
|
||||||
enabled: true
|
|
||||||
createDefault: true
|
|
||||||
node-feature-discovery:
|
|
||||||
enabled: false
|
|
||||||
|
|
||||||
- name: Wait for the Helm installation job to appear
|
|
||||||
ansible.builtin.command:
|
|
||||||
cmd: k3s kubectl get job/helm-install-kata-deploy -n kube-system
|
|
||||||
register: kata_helm_job
|
|
||||||
retries: 60
|
|
||||||
delay: 2
|
|
||||||
until: kata_helm_job.rc == 0
|
|
||||||
changed_when: false
|
|
||||||
|
|
||||||
- name: Wait for the Helm installation job to finish
|
|
||||||
ansible.builtin.command:
|
|
||||||
cmd: >-
|
|
||||||
k3s kubectl wait --for=condition=complete
|
|
||||||
job/helm-install-kata-deploy -n kube-system --timeout=20m
|
|
||||||
changed_when: false
|
|
||||||
|
|
||||||
- name: Wait for Kata node installation jobs
|
|
||||||
ansible.builtin.shell:
|
|
||||||
cmd: |
|
|
||||||
set -euo pipefail
|
|
||||||
jobs=$(k3s kubectl get jobs -n kata-system -l app.kubernetes.io/instance=kata-deploy -o name)
|
|
||||||
test -n "${jobs}"
|
|
||||||
k3s kubectl wait --for=condition=complete -n kata-system $jobs --timeout=20m
|
|
||||||
executable: /bin/bash
|
|
||||||
changed_when: false
|
|
||||||
|
|
||||||
- name: Wait for the default Kata RuntimeClass
|
|
||||||
ansible.builtin.command:
|
|
||||||
cmd: k3s kubectl get runtimeclass kata
|
|
||||||
register: kata_runtime_class
|
|
||||||
retries: 30
|
|
||||||
delay: 2
|
|
||||||
until: kata_runtime_class.rc == 0
|
|
||||||
changed_when: false
|
|
||||||
@@ -1,125 +0,0 @@
|
|||||||
---
|
|
||||||
- name: Verify Kata isolation with a disposable Pod
|
|
||||||
hosts: kata-lab
|
|
||||||
gather_facts: false
|
|
||||||
tasks:
|
|
||||||
- name: Remove an earlier smoke Pod
|
|
||||||
ansible.builtin.command:
|
|
||||||
cmd: k3s kubectl delete pod kata-smoke --ignore-not-found --wait=true
|
|
||||||
changed_when: false
|
|
||||||
|
|
||||||
- name: Create the Kata smoke Pod
|
|
||||||
ansible.builtin.shell:
|
|
||||||
cmd: |
|
|
||||||
set -o pipefail
|
|
||||||
k3s kubectl apply -f - <<'EOF'
|
|
||||||
apiVersion: v1
|
|
||||||
kind: Pod
|
|
||||||
metadata:
|
|
||||||
name: kata-smoke
|
|
||||||
spec:
|
|
||||||
runtimeClassName: kata
|
|
||||||
restartPolicy: Never
|
|
||||||
containers:
|
|
||||||
- name: smoke
|
|
||||||
image: docker.io/library/busybox:1.37
|
|
||||||
command:
|
|
||||||
- sh
|
|
||||||
- -c
|
|
||||||
- 'printf "guest-kernel="; uname -r; test -c /dev/kvm && exit 1 || true'
|
|
||||||
EOF
|
|
||||||
executable: /bin/bash
|
|
||||||
changed_when: true
|
|
||||||
|
|
||||||
- name: Wait for the Kata Pod to finish
|
|
||||||
ansible.builtin.command:
|
|
||||||
cmd: k3s kubectl wait --for=jsonpath='{.status.phase}'=Succeeded pod/kata-smoke --timeout=10m
|
|
||||||
changed_when: false
|
|
||||||
|
|
||||||
- name: Read the guest kernel version
|
|
||||||
ansible.builtin.command:
|
|
||||||
cmd: k3s kubectl logs kata-smoke
|
|
||||||
register: kata_smoke_log
|
|
||||||
changed_when: false
|
|
||||||
|
|
||||||
- name: Read the host kernel version
|
|
||||||
ansible.builtin.command:
|
|
||||||
cmd: uname -r
|
|
||||||
register: kata_host_kernel
|
|
||||||
changed_when: false
|
|
||||||
|
|
||||||
- name: Require a distinct guest kernel
|
|
||||||
ansible.builtin.assert:
|
|
||||||
that:
|
|
||||||
- kata_smoke_log.stdout is match('^guest-kernel=.+')
|
|
||||||
- kata_smoke_log.stdout | regex_replace('^guest-kernel=', '') != kata_host_kernel.stdout
|
|
||||||
success_msg: >-
|
|
||||||
Kata guest {{ kata_smoke_log.stdout }} differs from host-kernel={{ kata_host_kernel.stdout }}
|
|
||||||
|
|
||||||
- name: Remove an earlier Docker sidecar smoke Pod
|
|
||||||
ansible.builtin.command:
|
|
||||||
cmd: k3s kubectl delete pod kata-docker-smoke --ignore-not-found --wait=true
|
|
||||||
changed_when: false
|
|
||||||
|
|
||||||
- name: Create a Kata Pod with a Docker daemon sidecar
|
|
||||||
ansible.builtin.shell:
|
|
||||||
cmd: |
|
|
||||||
set -o pipefail
|
|
||||||
k3s kubectl apply -f - <<'EOF'
|
|
||||||
apiVersion: v1
|
|
||||||
kind: Pod
|
|
||||||
metadata:
|
|
||||||
name: kata-docker-smoke
|
|
||||||
spec:
|
|
||||||
runtimeClassName: kata
|
|
||||||
restartPolicy: Never
|
|
||||||
volumes:
|
|
||||||
- name: docker-run
|
|
||||||
emptyDir: {}
|
|
||||||
containers:
|
|
||||||
- name: dockerd
|
|
||||||
image: docker.io/library/docker:27-dind
|
|
||||||
securityContext:
|
|
||||||
privileged: true
|
|
||||||
env:
|
|
||||||
- name: DOCKER_TLS_CERTDIR
|
|
||||||
value: ""
|
|
||||||
volumeMounts:
|
|
||||||
- name: docker-run
|
|
||||||
mountPath: /var/run
|
|
||||||
- name: client
|
|
||||||
image: docker.io/library/docker:27-cli
|
|
||||||
env:
|
|
||||||
- name: DOCKER_HOST
|
|
||||||
value: tcp://127.0.0.1:2375
|
|
||||||
volumeMounts:
|
|
||||||
- name: docker-run
|
|
||||||
mountPath: /var/run
|
|
||||||
command:
|
|
||||||
- sh
|
|
||||||
- -c
|
|
||||||
- |
|
|
||||||
until docker info >/dev/null 2>&1; do sleep 1; done
|
|
||||||
docker run --rm docker.io/library/busybox:1.37 echo nested-docker-ok
|
|
||||||
EOF
|
|
||||||
executable: /bin/bash
|
|
||||||
changed_when: true
|
|
||||||
|
|
||||||
- name: Wait for the Docker client to finish
|
|
||||||
ansible.builtin.command:
|
|
||||||
cmd: >-
|
|
||||||
k3s kubectl wait --for=jsonpath='{.status.containerStatuses[?(@.name=="client")].state.terminated.exitCode}'=0
|
|
||||||
pod/kata-docker-smoke --timeout=10m
|
|
||||||
changed_when: false
|
|
||||||
|
|
||||||
- name: Read the Docker client result
|
|
||||||
ansible.builtin.command:
|
|
||||||
cmd: k3s kubectl logs kata-docker-smoke -c client
|
|
||||||
register: kata_docker_smoke_log
|
|
||||||
changed_when: false
|
|
||||||
|
|
||||||
- name: Require Docker to run a nested container
|
|
||||||
ansible.builtin.assert:
|
|
||||||
that:
|
|
||||||
- "'nested-docker-ok' in kata_docker_smoke_log.stdout"
|
|
||||||
success_msg: Docker daemon sidecar completed a nested container inside the Kata VM
|
|
||||||
@@ -1,24 +0,0 @@
|
|||||||
# This file is maintained automatically by "terraform init".
|
|
||||||
# Manual edits may be lost in future updates.
|
|
||||||
|
|
||||||
provider "registry.terraform.io/bpg/proxmox" {
|
|
||||||
version = "0.111.1"
|
|
||||||
constraints = "0.111.1"
|
|
||||||
hashes = [
|
|
||||||
"h1:ML2D3UUZTM99yrll/EBXj7wBYMb8xmQgomqFNybEoxY=",
|
|
||||||
"zh:18fb7c31a08dde6bffa1a4d4a211e604d6d17eec7092fd59331b3db3c6f3742c",
|
|
||||||
"zh:1cd60761538289d4dd2a1086b3ae62a7b0bdd4b1a2f824e9a44e243413168dba",
|
|
||||||
"zh:2eb76f6fc8299b6820ff678c8252332cc3366e226b5ae2e61748fd2449c1ed92",
|
|
||||||
"zh:45e6f7ebd0bf48911d37060359a4f359b5743b3092e985295733990e406d0416",
|
|
||||||
"zh:4aa8ba912eae37975d2e983394d173e595ca34fc76b5bf220b37d0e99d76e98c",
|
|
||||||
"zh:58e0789923103a77d502a0a9fc3eb920625e8eb935ec2d4ac0d006aebd1d186c",
|
|
||||||
"zh:6df8aa85fb8865915537e946c19b02538ad188018a629759c213c6f03730f642",
|
|
||||||
"zh:6ed47bc00d0913a1d0880618fa1376115e9edab6b4a658c081061a7f0e4ca360",
|
|
||||||
"zh:c5b10ff4f33df7e4c29e8f1127d49845b561b37b57517e844fb0954d7923d65e",
|
|
||||||
"zh:d016510e14b738499f0db9d9b3aafe82fc6877fb4ab4e9f831fb68a8d70a1385",
|
|
||||||
"zh:d941f394069bbf24351b363da1c64383f487067aaee0a84f9b96476d4912e212",
|
|
||||||
"zh:ddf271dbc2632ae8ffa8de3972f243ee47d260cb2ac90aa784f2746d98e21a0f",
|
|
||||||
"zh:ed0caa3501c42f611b7e9622c9b1df69fd85dc25a3cd88d3076381829688cd62",
|
|
||||||
"zh:f26e0763dbe6a6b2195c94b44696f2110f7f55433dc142839be16b9697fa5597",
|
|
||||||
]
|
|
||||||
}
|
|
||||||
@@ -1,34 +0,0 @@
|
|||||||
#cloud-config
|
|
||||||
hostname: ${hostname}
|
|
||||||
manage_etc_hosts: true
|
|
||||||
timezone: Etc/UTC
|
|
||||||
|
|
||||||
users:
|
|
||||||
- default
|
|
||||||
- name: ansible
|
|
||||||
groups: [adm, sudo]
|
|
||||||
shell: /bin/bash
|
|
||||||
lock_passwd: true
|
|
||||||
sudo: ALL=(ALL) NOPASSWD:ALL
|
|
||||||
ssh_authorized_keys:
|
|
||||||
- ${ssh_public_key}
|
|
||||||
|
|
||||||
package_update: true
|
|
||||||
package_upgrade: false
|
|
||||||
packages:
|
|
||||||
- ca-certificates
|
|
||||||
- curl
|
|
||||||
- jq
|
|
||||||
- qemu-guest-agent
|
|
||||||
|
|
||||||
runcmd:
|
|
||||||
- [systemctl, enable, --now, qemu-guest-agent]
|
|
||||||
- [modprobe, kvm_amd]
|
|
||||||
- [sh, -c, 'test -c /dev/kvm && echo available > /var/lib/cloud/nested-kvm.status || echo unavailable > /var/lib/cloud/nested-kvm.status']
|
|
||||||
- [sh, -c, 'curl -sfL https://get.k3s.io -o /tmp/install-k3s.sh']
|
|
||||||
- [chmod, '0755', /tmp/install-k3s.sh]
|
|
||||||
- [sh, -c, 'INSTALL_K3S_VERSION="${k3s_version}" INSTALL_K3S_EXEC="server --disable=traefik --write-kubeconfig-mode=0644" /tmp/install-k3s.sh']
|
|
||||||
- [sh, -c, 'kubectl wait --for=condition=Ready node/${hostname} --timeout=180s']
|
|
||||||
- [touch, /var/lib/cloud/kata-lab-bootstrap.done]
|
|
||||||
|
|
||||||
final_message: "kata-lab bootstrap completed after $UPTIME seconds"
|
|
||||||
@@ -1,3 +0,0 @@
|
|||||||
# Copy this file to credentials.auto.tfvars and replace the placeholder.
|
|
||||||
# Format: user@realm!token-id=token-secret
|
|
||||||
proxmox_api_token = "REPLACE_ME"
|
|
||||||
@@ -1,86 +0,0 @@
|
|||||||
locals {
|
|
||||||
ssh_public_key = trimspace(file(pathexpand(var.ssh_public_key_file)))
|
|
||||||
cloud_init = templatefile("${path.module}/cloud-init.yaml.tftpl", {
|
|
||||||
hostname = var.vm_name
|
|
||||||
ssh_public_key = local.ssh_public_key
|
|
||||||
k3s_version = var.k3s_version
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
data "proxmox_file" "ubuntu_noble" {
|
|
||||||
content_type = "import"
|
|
||||||
datastore_id = var.snippet_datastore_id
|
|
||||||
node_name = var.node_name
|
|
||||||
file_name = "noble-server-cloudimg-amd64.qcow2"
|
|
||||||
}
|
|
||||||
|
|
||||||
resource "proxmox_virtual_environment_file" "cloud_init" {
|
|
||||||
content_type = "snippets"
|
|
||||||
datastore_id = var.snippet_datastore_id
|
|
||||||
node_name = var.node_name
|
|
||||||
|
|
||||||
source_raw {
|
|
||||||
data = local.cloud_init
|
|
||||||
file_name = "${var.vm_name}-cloud-init.yaml"
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
resource "proxmox_virtual_environment_vm" "kata_lab" {
|
|
||||||
name = var.vm_name
|
|
||||||
description = "Disposable single-node k3s and Kata Containers validation environment."
|
|
||||||
tags = ["terraform", "disposable", "kata"]
|
|
||||||
|
|
||||||
node_name = var.node_name
|
|
||||||
vm_id = var.vm_id
|
|
||||||
|
|
||||||
started = true
|
|
||||||
on_boot = false
|
|
||||||
stop_on_destroy = true
|
|
||||||
|
|
||||||
agent {
|
|
||||||
enabled = true
|
|
||||||
timeout = "15m"
|
|
||||||
}
|
|
||||||
|
|
||||||
cpu {
|
|
||||||
cores = var.vm_cores
|
|
||||||
type = "host"
|
|
||||||
}
|
|
||||||
|
|
||||||
memory {
|
|
||||||
dedicated = var.vm_memory_mb
|
|
||||||
}
|
|
||||||
|
|
||||||
operating_system {
|
|
||||||
type = "l26"
|
|
||||||
}
|
|
||||||
|
|
||||||
scsi_hardware = "virtio-scsi-single"
|
|
||||||
|
|
||||||
disk {
|
|
||||||
datastore_id = var.disk_datastore_id
|
|
||||||
import_from = data.proxmox_file.ubuntu_noble.id
|
|
||||||
interface = "scsi0"
|
|
||||||
iothread = true
|
|
||||||
discard = "on"
|
|
||||||
size = var.vm_disk_gb
|
|
||||||
}
|
|
||||||
|
|
||||||
initialization {
|
|
||||||
datastore_id = var.disk_datastore_id
|
|
||||||
user_data_file_id = proxmox_virtual_environment_file.cloud_init.id
|
|
||||||
|
|
||||||
ip_config {
|
|
||||||
ipv4 {
|
|
||||||
address = "dhcp"
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
network_device {
|
|
||||||
bridge = var.network_bridge
|
|
||||||
model = "virtio"
|
|
||||||
}
|
|
||||||
|
|
||||||
serial_device {}
|
|
||||||
}
|
|
||||||
@@ -1,12 +0,0 @@
|
|||||||
output "vm_id" {
|
|
||||||
value = proxmox_virtual_environment_vm.kata_lab.vm_id
|
|
||||||
}
|
|
||||||
|
|
||||||
output "vm_ipv4_addresses" {
|
|
||||||
description = "QEMU guest agent 报告的地址；忽略 loopback 和 CNI 地址后再用于 SSH。"
|
|
||||||
value = proxmox_virtual_environment_vm.kata_lab.ipv4_addresses
|
|
||||||
}
|
|
||||||
|
|
||||||
output "ssh_user" {
|
|
||||||
value = "ansible"
|
|
||||||
}
|
|
||||||
@@ -1,17 +0,0 @@
|
|||||||
provider "proxmox" {
|
|
||||||
endpoint = var.proxmox_endpoint
|
|
||||||
api_token = var.proxmox_api_token
|
|
||||||
|
|
||||||
# Snippet uploads use SSH. The provider deliberately does not read
|
|
||||||
# ~/.ssh/config, so map the PVE node explicitly and use the current agent.
|
|
||||||
ssh {
|
|
||||||
agent = false
|
|
||||||
username = "root"
|
|
||||||
private_key = file(pathexpand(var.proxmox_ssh_private_key_file))
|
|
||||||
|
|
||||||
node {
|
|
||||||
name = "pve2"
|
|
||||||
address = "192.168.10.7"
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,83 +0,0 @@
|
|||||||
variable "proxmox_endpoint" {
|
|
||||||
description = "Proxmox VE API endpoint，不包含 /api2/json。"
|
|
||||||
type = string
|
|
||||||
default = "https://pve1.ad.ddupan.top:8006/"
|
|
||||||
}
|
|
||||||
|
|
||||||
variable "proxmox_api_token" {
|
|
||||||
description = "Proxmox API token，格式为 user@realm!token-id=secret。"
|
|
||||||
type = string
|
|
||||||
sensitive = true
|
|
||||||
}
|
|
||||||
|
|
||||||
variable "node_name" {
|
|
||||||
description = "承载 disposable kata-lab VM 的 PVE 节点。"
|
|
||||||
type = string
|
|
||||||
default = "pve2"
|
|
||||||
}
|
|
||||||
|
|
||||||
variable "vm_id" {
|
|
||||||
description = "kata-lab VMID。"
|
|
||||||
type = number
|
|
||||||
default = 147
|
|
||||||
}
|
|
||||||
|
|
||||||
variable "vm_name" {
|
|
||||||
description = "kata-lab VM 名称和 guest hostname。"
|
|
||||||
type = string
|
|
||||||
default = "kata-lab"
|
|
||||||
}
|
|
||||||
|
|
||||||
variable "vm_memory_mb" {
|
|
||||||
description = "VM 固定内存；pve2 只有约 7.4 GiB 可见内存。"
|
|
||||||
type = number
|
|
||||||
default = 4096
|
|
||||||
}
|
|
||||||
|
|
||||||
variable "vm_cores" {
|
|
||||||
description = "VM vCPU 数量。"
|
|
||||||
type = number
|
|
||||||
default = 4
|
|
||||||
}
|
|
||||||
|
|
||||||
variable "vm_disk_gb" {
|
|
||||||
description = "VM root disk 大小。"
|
|
||||||
type = number
|
|
||||||
default = 41
|
|
||||||
}
|
|
||||||
|
|
||||||
variable "disk_datastore_id" {
|
|
||||||
description = "VM root disk 所在 datastore。"
|
|
||||||
type = string
|
|
||||||
default = "pve-rg-hdd"
|
|
||||||
}
|
|
||||||
|
|
||||||
variable "snippet_datastore_id" {
|
|
||||||
description = "启用 snippets 的共享 datastore。"
|
|
||||||
type = string
|
|
||||||
default = "laptop"
|
|
||||||
}
|
|
||||||
|
|
||||||
variable "network_bridge" {
|
|
||||||
description = "连接 kata-lab VM 的 PVE bridge。"
|
|
||||||
type = string
|
|
||||||
default = "vmbr0"
|
|
||||||
}
|
|
||||||
|
|
||||||
variable "ssh_public_key_file" {
|
|
||||||
description = "注入 cloud-init 用户的 SSH 公钥路径。"
|
|
||||||
type = string
|
|
||||||
default = "~/.ssh/id_ed25519.pub"
|
|
||||||
}
|
|
||||||
|
|
||||||
variable "proxmox_ssh_private_key_file" {
|
|
||||||
description = "provider 上传 snippet 时登录 PVE 节点使用的私钥路径。"
|
|
||||||
type = string
|
|
||||||
default = "~/.ssh/id_ed25519"
|
|
||||||
}
|
|
||||||
|
|
||||||
variable "k3s_version" {
|
|
||||||
description = "可选的固定 k3s 版本；空值使用 stable channel。"
|
|
||||||
type = string
|
|
||||||
default = ""
|
|
||||||
}
|
|
||||||
@@ -1,10 +0,0 @@
|
|||||||
terraform {
|
|
||||||
required_version = ">= 1.10.0"
|
|
||||||
|
|
||||||
required_providers {
|
|
||||||
proxmox = {
|
|
||||||
source = "bpg/proxmox"
|
|
||||||
version = "0.111.1"
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,180 +0,0 @@
|
|||||||
# kata-lxc-lab
|
|
||||||
|
|
||||||
在 `pve2` 上验证 privileged LXC 内运行 k3s、Kata/QEMU，并直接使用 PVE host 的
|
|
||||||
`/dev/kvm`、`/dev/vhost-net` 与 `/dev/vhost-vsock`。这是可销毁 PoC，不承载持久数据。
|
|
||||||
|
|
||||||
Terraform 复用原 VM PoC 的 ignored credential 文件，不复制 token：
|
|
||||||
|
|
||||||
```bash
|
|
||||||
cd infrastructure/kata-lxc-lab/terraform
|
|
||||||
export SSL_CERT_FILE=/tmp/kata-lab-ddupan-ca.pem
|
|
||||||
terraform init
|
|
||||||
terraform plan \
|
|
||||||
-var-file=../../kata-lab/terraform/credentials.auto.tfvars
|
|
||||||
terraform apply \
|
|
||||||
-var-file=../../kata-lab/terraform/credentials.auto.tfvars
|
|
||||||
```
|
|
||||||
|
|
||||||
当前 PoC 使用 `pve2`、VMID 147、Ubuntu 24.04 LXC template 和 16 GiB
|
|
||||||
`local-lvm` rootfs。容器为 privileged，并开启 nesting、keyctl、FUSE、mknod；
|
|
||||||
Kata 所需的 KVM、vhost 与 `/dev/net/tun` 设备显式透传；另透传 `/dev/kmsg`，因为
|
|
||||||
kubelet 启动时会打开该设备。
|
|
||||||
PVE 9 的 `force_rw_sys=1` feature 用于开放 kubelet 所需的少量 `/proc/sys` 写操作；
|
|
||||||
provider 0.111.1 尚未暴露该 feature，因此同样由 `pct` 设置。
|
|
||||||
|
|
||||||
k3s/Kata worker 还使用以下 privileged LXC 原生配置；PVE 9 使用 cgroup v2，因此设备
|
|
||||||
规则采用 `lxc.cgroup2.devices.allow`：
|
|
||||||
|
|
||||||
```text
|
|
||||||
lxc.apparmor.profile: unconfined
|
|
||||||
lxc.cgroup2.devices.allow: a
|
|
||||||
lxc.cap.drop:
|
|
||||||
lxc.mount.auto: proc:rw sys:rw
|
|
||||||
```
|
|
||||||
|
|
||||||
这些设置让容器内 kubelet、containerd 和 Kata shim 能管理 mount、cgroup、设备与
|
|
||||||
共享宿主 sysctl。因此外层 LXC 不是安全边界；不可信 CI 的安全边界是内层 Kata VM。
|
|
||||||
`/dev/kmsg` 当前直接透传，没有使用 `/dev/console` symlink fallback。
|
|
||||||
|
|
||||||
PVE 将 privileged LXC 的创建限制为 `root@pam`（API token 即使拥有 `PVEAdmin` 也
|
|
||||||
缺少 root-only 的 `Sys.Modify`）。因此实际创建入口使用 root SSH 上的 Ansible/pct：
|
|
||||||
|
|
||||||
```bash
|
|
||||||
cd infrastructure/kata-lxc-lab/ansible
|
|
||||||
ansible-playbook create.yml
|
|
||||||
ansible-playbook bootstrap.yml
|
|
||||||
ansible-playbook kata.yml
|
|
||||||
ansible-playbook verify.yml
|
|
||||||
ansible-playbook verify-fuse.yml
|
|
||||||
```
|
|
||||||
|
|
||||||
Terraform 文件保留用于记录 provider 权限边界与声明式目标；当前 token 执行 apply 会
|
|
||||||
在创建前返回 403，不会产生资源或 state。
|
|
||||||
|
|
||||||
## 验证结果
|
|
||||||
|
|
||||||
当前管理地址为 `192.168.10.128`：2026-09-14 主 LAN DHCP 池调整后，从 `.11`
|
|
||||||
续租迁移到 `.128`，并同步 Ansible inventory。它仍是动态租约，不是固定地址。
|
|
||||||
|
|
||||||
2026-09-14，pve2 VMID 147 / `192.168.10.11` 验证通过：
|
|
||||||
|
|
||||||
- LXC 内 k3s `v1.36.4+k3s1` node Ready；
|
|
||||||
- Kata Containers 4.1.0 `qemu-runtime-rs` 安装成功；
|
|
||||||
- Kata guest kernel `6.18.35`，外层共享的 PVE kernel `7.0.2-6-pve`；
|
|
||||||
- Kata Pod 内的 Docker 27.5.1 daemon 成功运行 `busybox:1.37` 嵌套容器；
|
|
||||||
- `/dev/net/tun` 是 Kata 创建 TAP/link 的必要设备，未透传时 runtime-rs 在
|
|
||||||
`create link` 返回 `ENOENT`；
|
|
||||||
- `fuse-overlayfs` smoke test 成功：Docker 报告
|
|
||||||
`storage-driver=fuse-overlayfs`，并成功运行嵌套容器。
|
|
||||||
|
|
||||||
正式方案保持 Kata 默认 `emptydir_mode = "shared-fs"`，不向 LXC 暴露 host
|
|
||||||
`/dev/loop-control` 或 `/dev/loopN`。dockerd 镜像需要包含 `fuse-overlayfs`；启动前在
|
|
||||||
Kata guest 内创建临时设备节点 `mknod /dev/fuse c 10 229`，随后强制传入
|
|
||||||
`--storage-driver=fuse-overlayfs`。该 `/dev/fuse` 属于内层 guest kernel，不是 PVE
|
|
||||||
host 设备透传，并随 Kata Pod 一起销毁。
|
|
||||||
|
|
||||||
删除三个 smoke Pod 后，LXC cgroup 统计约 505 MB anonymous memory、1.34 GB file
|
|
||||||
cache；后者主要来自刚拉取的 Kata/Docker images，可由宿主回收。也就是说常驻 worker
|
|
||||||
的不可回收 working set 约 500 MB，8 GiB 配置是 cgroup 上限而非预分配。
|
|
||||||
|
|
||||||
## Gitea ephemeral runner PoC
|
|
||||||
|
|
||||||
`runner/` 定义一次性 Gitea runner Pod。整个 Pod 使用 `runtimeClassName: kata`；
|
|
||||||
runner 通过 guest 内同 Pod 的 dockerd 创建 job container，不连接 LXC/PVE host 的
|
|
||||||
Docker socket。dockerd 使用 zot 中预先发布的 `dind-fuse:29.7.1`，其
|
|
||||||
`/var/lib/docker` 是随 Kata Pod 删除的 `emptyDir`。
|
|
||||||
|
|
||||||
runner 以 `GITEA_RUNNER_EPHEMERAL=1` 注册，只接收一个 `runs-on: kata-poc` job；接单
|
|
||||||
后 Gitea 撤销其 runner credential，job 完成后进程与 Kata VM 一起退出。当前 PoC
|
|
||||||
手动创建一个 runner 等待 `.gitea/workflows/kata-runner-smoke.yml`，尚未部署
|
|
||||||
`workflow_job` webhook controller，因此不会保留常驻空闲 runner。
|
|
||||||
|
|
||||||
注册 token 只在运行时从现有 OpenBao/ESO 消费副本投递到 PoC 集群的临时 Secret，
|
|
||||||
不写入此目录。两个 Pod image 使用 `imagePullPolicy: Never`，部署前从可信工作站将
|
|
||||||
已验证镜像导入 PoC k3s containerd；这是当前 zot 拉取仍要求 SPIFFE 身份时的
|
|
||||||
bootstrap 边界，不是正式镜像分发方案。
|
|
||||||
|
|
||||||
正式接入 zot 前需要把这个 cluster 注册到 SPIRE，并为 runner Pod 创建专用
|
|
||||||
`ClusterSPIFFEID`。届时 runner 与 dockerd sidecar 在同一路径只读挂载 CSI Workload
|
|
||||||
API socket，runner 的 `container.options` 再把该路径 bind-mount 到 job container。
|
|
||||||
zot 只对最终 SPIFFE ID 的 `panxiao81/dind-fuse` 仓库授予
|
|
||||||
`read/create/update`；禁止授予 namespace 或整个 trust domain 写权限。
|
|
||||||
|
|
||||||
2026-09-14 已完成真实 Gitea job 验收：Actions run `242` / job `445` 使用
|
|
||||||
`alpine:3.22`，10 秒成功；runner container 退出码为 `0`，原生 dockerd sidecar 随后
|
|
||||||
由 kubelet 终止，Pod 最终为 `Succeeded`，没有保留空闲 Kata VM。guest 内核为
|
|
||||||
`6.18.35`，dockerd `29.7.1` 使用 `fuse-overlayfs`。
|
|
||||||
|
|
||||||
首次尝试暴露了两个部署陷阱：`COPY` 默认保留源文件 mode，入口脚本原为 `0664`，
|
|
||||||
必须在 Dockerfile 使用 `COPY --chmod=0755`；只配置 `ephemeral-storage: 20Gi` limit
|
|
||||||
会形成同值 request，使 12 GiB PoC 节点无法调度。当前 manifest 显式 request 1 GiB、
|
|
||||||
limit 8 GiB。完整 `ubuntu-latest` job image 也不适合这个小 rootfs，基础生命周期验收
|
|
||||||
改用 Alpine；正式 worker 必须扩容本地 image/cache 空间或使用更精简的 job image。
|
|
||||||
|
|
||||||
## Cloud Hypervisor 内存记账验证
|
|
||||||
|
|
||||||
2026-09-17 在 pve2 的一次性 VMID 148 中安装 k3s v1.36.4+k3s1 与 Kata 4.1.0，
|
|
||||||
只启用 `clh-runtime-rs`，验证 Cloud Hypervisor 在 privileged LXC 内不会重现早期
|
|
||||||
microVM PoC 的 private memfd 双重记账问题。测试完成后已删除 VMID 148。
|
|
||||||
|
|
||||||
Cloud Hypervisor `/api/v1/vm.info` 报告：
|
|
||||||
|
|
||||||
```json
|
|
||||||
{"size":3254779904,"shared":true,"hugepages":false,"prefault":false,"thp":true}
|
|
||||||
```
|
|
||||||
|
|
||||||
Kata guest 在 memory-backed `emptyDir` 中写入 1 GiB 后：
|
|
||||||
|
|
||||||
- VMM `Pss_Shmem` 从约 247 MiB 增至约 1269 MiB；
|
|
||||||
- VMM `Pss_Anon` 写入前后均约 1.2 MiB；
|
|
||||||
- 外层 LXC `shmem` 增加约 1 GiB，`anon` 基本不变；
|
|
||||||
- LXC `memory.events` 的 `max`、`oom` 与 `oom_kill` 均为 0；
|
|
||||||
- 删除 Pod 后 VMM 退出，LXC `shmem` 回落到约 1.6 MiB。
|
|
||||||
|
|
||||||
因此 Kata 的 Cloud Hypervisor handler 已使用 shared memfd，等价于独立 launcher 的
|
|
||||||
`--memory shared=on` 修复；后续 OpenSandbox/Kata 方案无需为 guest RAM 预留近似双倍
|
|
||||||
的 LXC cgroup 内存。部署验收仍应检查 `vm.info.config.memory.shared=true`，防止上游
|
|
||||||
配置或 runtime handler 变化造成回归。
|
|
||||||
|
|
||||||
测试期间还发现 pve2 `local-lvm` thin pool 已达到 100%；临时 rootfs 必须放到
|
|
||||||
`pve-rg-hdd`，不能根据容器内部 `df` 的剩余空间判断 thin pool 是否可写。
|
|
||||||
|
|
||||||
## Block-backed emptyDir 与 CI 构建验证
|
|
||||||
|
|
||||||
2026-09-17 又在 pve2 的一次性 VMID 148 中验证 Kata 4.1.0
|
|
||||||
`clh-runtime-rs` 的以下 drop-in：
|
|
||||||
|
|
||||||
```toml
|
|
||||||
[runtime]
|
|
||||||
emptydir_mode = "block-plain"
|
|
||||||
```
|
|
||||||
|
|
||||||
结果如下：
|
|
||||||
|
|
||||||
- 普通 Kubernetes `emptyDir` 被创建为稀疏 `disk.img`，由 Cloud Hypervisor 直接作为
|
|
||||||
block device 热插拔；guest 内 `/var/lib/docker` 是 `/dev/vdb` 上的 ext4。
|
|
||||||
- Docker 29.1.5 成功强制使用原生 `overlay2`，不再需要 `fuse-overlayfs`；整个过程
|
|
||||||
没有使用 PVE host 或 LXC 的 loop device。
|
|
||||||
- `emptyDir.sizeLimit: 8Gi` 没有决定虚拟磁盘容量。由于 kubelet 所在文件系统约
|
|
||||||
12 GiB，guest 看到的 ext4 约 11.7 GiB；这与 Kata 已知的 block-backed emptyDir
|
|
||||||
限制一致。
|
|
||||||
- 包含 256 MiB payload、2000 个小文件和一个额外复制层的冷 BuildKit 构建耗时
|
|
||||||
46 秒；同一 LXC、同一 DRBD HDD 存储上的 runc 对照为 34 秒，Kata 约慢 35%。
|
|
||||||
- 在 overlay2 容器层内写入并 fsync 512 MiB：Kata 为 12 秒（40.5 MB/s），runc 为
|
|
||||||
10 秒（50.3 MB/s）；创建一万个小文件两者均约 2 秒。
|
|
||||||
- kind v0.27.0 / Kubernetes v1.32.2 首次启动失败是因为 Kata guest 内的 dockerd
|
|
||||||
容器没有 `/dev/kmsg`，不是 block storage 或 cgroup 故障。在 privileged dockerd
|
|
||||||
启动前执行 `mknod /dev/kmsg c 1 11` 后，kind 创建成功：总计 122 秒，进入等待阶段
|
|
||||||
后 23 秒 control-plane Ready，随后可正常删除。
|
|
||||||
- 测试时稀疏 backing file 逻辑大小约 12 GiB，构建和 kind 后实际占用约 1.8 GiB。
|
|
||||||
删除 Pod 后 backing file、Cloud Hypervisor 进程和 kind 容器全部消失；host/LXC
|
|
||||||
均无 loop device 残留，LXC 没有 OOM 事件。
|
|
||||||
|
|
||||||
因此 `block-plain` 能解决 virtio-fs 不能作为 overlayfs upperdir 的问题，并满足
|
|
||||||
BuildKit 与 kind 的功能要求。正式 CI RuntimeClass 必须使用独立的 block-backed
|
|
||||||
配置，并在 dockerd bootstrap 中创建 guest `/dev/kmsg`。它仍比同机 runc 构建慢约
|
|
||||||
20–35%，且 `emptyDir.sizeLimit` 不能可靠限制虚拟磁盘容量。PoC 曾据此建议为 kubelet
|
|
||||||
目录提供独立文件系统；正式 sandbox 集群最终没有采用该设计：独立 volume 增加了
|
|
||||||
DRBD 与 LXC 挂载生命周期复杂度，却没有隔离 containerd 等其他节点数据。正式节点
|
|
||||||
改用单一、容量明确的 rootfs，并以磁盘占用监控、Pod 删除后的 backing-file 回收检查
|
|
||||||
和实际构建基准作为上线门槛。
|
|
||||||
@@ -1,10 +0,0 @@
|
|||||||
[defaults]
|
|
||||||
inventory = inventory/hosts.yml
|
|
||||||
host_key_checking = True
|
|
||||||
interpreter_python = auto_silent
|
|
||||||
retry_files_enabled = False
|
|
||||||
local_tmp = /tmp/ansible-kata-lxc-local
|
|
||||||
remote_tmp = /tmp/ansible-kata-lxc-remote
|
|
||||||
|
|
||||||
[ssh_connection]
|
|
||||||
pipelining = True
|
|
||||||
@@ -1,58 +0,0 @@
|
|||||||
---
|
|
||||||
- name: Bootstrap k3s in the Kata LXC worker
|
|
||||||
hosts: pve2
|
|
||||||
gather_facts: false
|
|
||||||
vars:
|
|
||||||
kata_lxc_id: 147
|
|
||||||
tasks:
|
|
||||||
- name: Check base packages in LXC
|
|
||||||
ansible.builtin.command:
|
|
||||||
cmd: >-
|
|
||||||
pct exec {{ kata_lxc_id }} -- dpkg-query -W
|
|
||||||
ca-certificates curl jq openssh-server
|
|
||||||
register: kata_lxc_packages
|
|
||||||
changed_when: false
|
|
||||||
failed_when: false
|
|
||||||
|
|
||||||
- name: Install base packages and SSH server in LXC
|
|
||||||
ansible.builtin.command:
|
|
||||||
cmd: >-
|
|
||||||
pct exec {{ kata_lxc_id }} -- bash -lc
|
|
||||||
'apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y
|
|
||||||
ca-certificates curl jq openssh-server'
|
|
||||||
when: kata_lxc_packages.rc != 0
|
|
||||||
|
|
||||||
- name: Install k3s in LXC
|
|
||||||
ansible.builtin.shell:
|
|
||||||
cmd: |
|
|
||||||
set -euo pipefail
|
|
||||||
pct exec {{ kata_lxc_id }} -- bash -lc '
|
|
||||||
if ! command -v k3s >/dev/null; then
|
|
||||||
curl -sfL https://get.k3s.io -o /tmp/install-k3s.sh
|
|
||||||
chmod 0755 /tmp/install-k3s.sh
|
|
||||||
INSTALL_K3S_EXEC="server --disable=traefik --write-kubeconfig-mode=0644" /tmp/install-k3s.sh
|
|
||||||
fi
|
|
||||||
'
|
|
||||||
executable: /bin/bash
|
|
||||||
register: kata_lxc_k3s_install
|
|
||||||
changed_when: "'No change detected' not in kata_lxc_k3s_install.stdout"
|
|
||||||
|
|
||||||
- name: Wait for k3s node readiness
|
|
||||||
ansible.builtin.command:
|
|
||||||
cmd: >-
|
|
||||||
pct exec {{ kata_lxc_id }} -- /usr/local/bin/k3s kubectl wait
|
|
||||||
--for=condition=Ready node/kata-lxc-lab --timeout=180s
|
|
||||||
changed_when: false
|
|
||||||
|
|
||||||
- name: Check LXC virtualization and devices
|
|
||||||
ansible.builtin.command:
|
|
||||||
cmd: >-
|
|
||||||
pct exec {{ kata_lxc_id }} -- bash -lc
|
|
||||||
'systemd-detect-virt; ls -l /dev/kvm /dev/vhost-net /dev/vhost-vsock;
|
|
||||||
/usr/local/bin/k3s kubectl get node -o wide'
|
|
||||||
register: kata_lxc_ready
|
|
||||||
changed_when: false
|
|
||||||
|
|
||||||
- name: Report k3s readiness
|
|
||||||
ansible.builtin.debug:
|
|
||||||
var: kata_lxc_ready.stdout_lines
|
|
||||||
@@ -1,137 +0,0 @@
|
|||||||
---
|
|
||||||
- name: Create the disposable Kata LXC worker
|
|
||||||
hosts: pve2
|
|
||||||
gather_facts: false
|
|
||||||
vars:
|
|
||||||
kata_lxc_id: 147
|
|
||||||
kata_lxc_template: laptop:vztmpl/ubuntu-24.04-standard_24.04-2_amd64.tar.zst
|
|
||||||
tasks:
|
|
||||||
- name: Check whether the LXC already exists
|
|
||||||
ansible.builtin.stat:
|
|
||||||
path: /etc/pve/lxc/{{ kata_lxc_id }}.conf
|
|
||||||
register: kata_lxc_config
|
|
||||||
|
|
||||||
- name: Create privileged LXC with Kata host devices
|
|
||||||
ansible.builtin.command:
|
|
||||||
argv:
|
|
||||||
- pct
|
|
||||||
- create
|
|
||||||
- "{{ kata_lxc_id }}"
|
|
||||||
- "{{ kata_lxc_template }}"
|
|
||||||
- --hostname
|
|
||||||
- kata-lxc-lab
|
|
||||||
- --ostype
|
|
||||||
- ubuntu
|
|
||||||
- --rootfs
|
|
||||||
- local-lvm:12
|
|
||||||
- --cores
|
|
||||||
- "4"
|
|
||||||
- --memory
|
|
||||||
- "8192"
|
|
||||||
- --swap
|
|
||||||
- "0"
|
|
||||||
- --net0
|
|
||||||
- name=eth0,bridge=vmbr0,ip=dhcp
|
|
||||||
- --features
|
|
||||||
- nesting=1,keyctl=1,fuse=1,mknod=1,force_rw_sys=1
|
|
||||||
- --unprivileged
|
|
||||||
- "0"
|
|
||||||
- --onboot
|
|
||||||
- "0"
|
|
||||||
- --ssh-public-keys
|
|
||||||
- /root/.ssh/authorized_keys
|
|
||||||
- --dev0
|
|
||||||
- path=/dev/kvm,mode=0660
|
|
||||||
- --dev1
|
|
||||||
- path=/dev/vhost-net,mode=0660
|
|
||||||
- --dev2
|
|
||||||
- path=/dev/vhost-vsock,mode=0660
|
|
||||||
- --dev3
|
|
||||||
- path=/dev/kmsg,mode=0660
|
|
||||||
- --dev4
|
|
||||||
- path=/dev/net/tun,mode=0666
|
|
||||||
- --tags
|
|
||||||
- disposable;kata;lxc;ansible
|
|
||||||
when: not kata_lxc_config.stat.exists
|
|
||||||
|
|
||||||
- name: Read LXC status
|
|
||||||
ansible.builtin.command:
|
|
||||||
cmd: pct status {{ kata_lxc_id }}
|
|
||||||
register: kata_lxc_status
|
|
||||||
changed_when: false
|
|
||||||
|
|
||||||
- name: Read current LXC configuration
|
|
||||||
ansible.builtin.command:
|
|
||||||
cmd: pct config {{ kata_lxc_id }}
|
|
||||||
register: kata_lxc_current_config
|
|
||||||
changed_when: false
|
|
||||||
|
|
||||||
- name: Ensure kubelet kernel log device is present
|
|
||||||
ansible.builtin.command:
|
|
||||||
cmd: pct set {{ kata_lxc_id }} --dev3 path=/dev/kmsg,mode=0660
|
|
||||||
register: kata_lxc_kmsg
|
|
||||||
when: "'/dev/kmsg' not in kata_lxc_current_config.stdout"
|
|
||||||
|
|
||||||
- name: Enable writable sysctls required by kubelet
|
|
||||||
ansible.builtin.command:
|
|
||||||
cmd: >-
|
|
||||||
pct set {{ kata_lxc_id }} --features
|
|
||||||
nesting=1,keyctl=1,fuse=1,mknod=1,force_rw_sys=1
|
|
||||||
register: kata_lxc_rw_sys
|
|
||||||
when: "'force_rw_sys=1' not in kata_lxc_current_config.stdout"
|
|
||||||
|
|
||||||
- name: Ensure TUN device required by Kata networking is present
|
|
||||||
ansible.builtin.command:
|
|
||||||
cmd: pct set {{ kata_lxc_id }} --dev4 path=/dev/net/tun,mode=0666
|
|
||||||
register: kata_lxc_tun
|
|
||||||
when: "'/dev/net/tun' not in kata_lxc_current_config.stdout"
|
|
||||||
|
|
||||||
- name: Configure privileged nested-runtime LXC directives
|
|
||||||
ansible.builtin.lineinfile:
|
|
||||||
path: /etc/pve/lxc/{{ kata_lxc_id }}.conf
|
|
||||||
regexp: "^{{ item.key | regex_escape }}:"
|
|
||||||
line: "{{ item.key }}: {{ item.value }}"
|
|
||||||
loop:
|
|
||||||
- key: lxc.apparmor.profile
|
|
||||||
value: unconfined
|
|
||||||
- key: lxc.cgroup2.devices.allow
|
|
||||||
value: a
|
|
||||||
- key: lxc.cap.drop
|
|
||||||
value: ""
|
|
||||||
- key: lxc.mount.auto
|
|
||||||
value: proc:rw sys:rw
|
|
||||||
register: kata_lxc_raw_config
|
|
||||||
|
|
||||||
- name: Restart LXC after device configuration change
|
|
||||||
ansible.builtin.command:
|
|
||||||
cmd: pct reboot {{ kata_lxc_id }}
|
|
||||||
when: >-
|
|
||||||
kata_lxc_kmsg.changed or kata_lxc_rw_sys.changed or kata_lxc_tun.changed or
|
|
||||||
kata_lxc_raw_config.changed
|
|
||||||
|
|
||||||
- name: Start LXC
|
|
||||||
ansible.builtin.command:
|
|
||||||
cmd: pct start {{ kata_lxc_id }}
|
|
||||||
when: "'status: stopped' in kata_lxc_status.stdout"
|
|
||||||
|
|
||||||
- name: Wait for systemd in LXC
|
|
||||||
ansible.builtin.command:
|
|
||||||
cmd: pct exec {{ kata_lxc_id }} -- systemctl is-system-running --wait
|
|
||||||
register: kata_lxc_systemd
|
|
||||||
retries: 30
|
|
||||||
delay: 2
|
|
||||||
until: kata_lxc_systemd.rc in [0, 1]
|
|
||||||
changed_when: false
|
|
||||||
failed_when: kata_lxc_systemd.rc not in [0, 1]
|
|
||||||
|
|
||||||
- name: Show LXC address and Kata devices
|
|
||||||
ansible.builtin.command:
|
|
||||||
cmd: >-
|
|
||||||
pct exec {{ kata_lxc_id }} -- sh -c
|
|
||||||
'hostname -I; ls -l /dev/kvm /dev/vhost-net /dev/vhost-vsock /dev/net/tun'
|
|
||||||
register: kata_lxc_facts
|
|
||||||
changed_when: false
|
|
||||||
|
|
||||||
- name: Report LXC address and devices
|
|
||||||
ansible.builtin.debug:
|
|
||||||
var: kata_lxc_facts.stdout_lines
|
|
||||||
@@ -1,9 +0,0 @@
|
|||||||
---
|
|
||||||
all:
|
|
||||||
hosts:
|
|
||||||
pve2:
|
|
||||||
ansible_host: 192.168.10.7
|
|
||||||
ansible_user: root
|
|
||||||
kata-lab:
|
|
||||||
ansible_host: 192.168.10.128
|
|
||||||
ansible_user: root
|
|
||||||
@@ -1,3 +0,0 @@
|
|||||||
---
|
|
||||||
# Reuse the proven Kata 4.1.0 installation playbook against the LXC inventory.
|
|
||||||
- import_playbook: ../../kata-lab/ansible/kata.yml
|
|
||||||
@@ -1,73 +0,0 @@
|
|||||||
---
|
|
||||||
- name: Verify fuse-overlayfs for Docker inside Kata
|
|
||||||
hosts: kata-lab
|
|
||||||
gather_facts: false
|
|
||||||
tasks:
|
|
||||||
- name: Remove an earlier fuse-overlayfs smoke Pod
|
|
||||||
ansible.builtin.command:
|
|
||||||
cmd: k3s kubectl delete pod kata-fuse-smoke --ignore-not-found --wait=true
|
|
||||||
changed_when: false
|
|
||||||
|
|
||||||
- name: Create Kata Docker Pod using fuse-overlayfs
|
|
||||||
ansible.builtin.shell:
|
|
||||||
cmd: |
|
|
||||||
set -o pipefail
|
|
||||||
k3s kubectl apply -f - <<'EOF'
|
|
||||||
apiVersion: v1
|
|
||||||
kind: Pod
|
|
||||||
metadata:
|
|
||||||
name: kata-fuse-smoke
|
|
||||||
spec:
|
|
||||||
runtimeClassName: kata
|
|
||||||
restartPolicy: Never
|
|
||||||
containers:
|
|
||||||
- name: dockerd
|
|
||||||
image: docker.io/library/docker:27-dind
|
|
||||||
securityContext:
|
|
||||||
privileged: true
|
|
||||||
env:
|
|
||||||
- name: DOCKER_TLS_CERTDIR
|
|
||||||
value: ""
|
|
||||||
command:
|
|
||||||
- sh
|
|
||||||
- -c
|
|
||||||
- |
|
|
||||||
apk add --no-cache fuse-overlayfs
|
|
||||||
test -e /dev/fuse || mknod /dev/fuse c 10 229
|
|
||||||
chmod 0666 /dev/fuse
|
|
||||||
exec dockerd-entrypoint.sh --storage-driver=fuse-overlayfs
|
|
||||||
- name: client
|
|
||||||
image: docker.io/library/docker:27-cli
|
|
||||||
env:
|
|
||||||
- name: DOCKER_HOST
|
|
||||||
value: tcp://127.0.0.1:2375
|
|
||||||
command:
|
|
||||||
- sh
|
|
||||||
- -c
|
|
||||||
- |
|
|
||||||
until docker info >/dev/null 2>&1; do sleep 1; done
|
|
||||||
docker info --format 'storage-driver={{ "{{" }}.Driver{{ "}}" }}'
|
|
||||||
docker run --rm docker.io/library/busybox:1.37 echo fuse-nested-docker-ok
|
|
||||||
EOF
|
|
||||||
executable: /bin/bash
|
|
||||||
changed_when: true
|
|
||||||
|
|
||||||
- name: Wait for the Docker client to finish
|
|
||||||
ansible.builtin.command:
|
|
||||||
cmd: >-
|
|
||||||
k3s kubectl wait --for=jsonpath='{.status.containerStatuses[?(@.name=="client")].state.terminated.exitCode}'=0
|
|
||||||
pod/kata-fuse-smoke --timeout=10m
|
|
||||||
changed_when: false
|
|
||||||
|
|
||||||
- name: Read Docker client result
|
|
||||||
ansible.builtin.command:
|
|
||||||
cmd: k3s kubectl logs kata-fuse-smoke -c client
|
|
||||||
register: kata_fuse_smoke_log
|
|
||||||
changed_when: false
|
|
||||||
|
|
||||||
- name: Require fuse-overlayfs and nested Docker
|
|
||||||
ansible.builtin.assert:
|
|
||||||
that:
|
|
||||||
- "'storage-driver=fuse-overlayfs' in kata_fuse_smoke_log.stdout"
|
|
||||||
- "'fuse-nested-docker-ok' in kata_fuse_smoke_log.stdout"
|
|
||||||
success_msg: Docker used fuse-overlayfs and completed a nested container inside Kata
|
|
||||||
@@ -1,3 +0,0 @@
|
|||||||
---
|
|
||||||
# Reuse the guest-kernel and Docker sidecar smoke tests against the LXC worker.
|
|
||||||
- import_playbook: ../../kata-lab/ansible/verify.yml
|
|
||||||
@@ -1,35 +0,0 @@
|
|||||||
apiVersion: v1
|
|
||||||
kind: Namespace
|
|
||||||
metadata:
|
|
||||||
name: gitea-actions
|
|
||||||
labels:
|
|
||||||
pod-security.kubernetes.io/enforce: privileged
|
|
||||||
---
|
|
||||||
apiVersion: v1
|
|
||||||
kind: ConfigMap
|
|
||||||
metadata:
|
|
||||||
name: kata-ephemeral-runner-config
|
|
||||||
namespace: gitea-actions
|
|
||||||
data:
|
|
||||||
config.yaml: |
|
|
||||||
log:
|
|
||||||
level: info
|
|
||||||
runner:
|
|
||||||
file: /runner-state/.runner
|
|
||||||
capacity: 1
|
|
||||||
envs:
|
|
||||||
DOCKER_HOST: tcp://host.docker.internal:2375
|
|
||||||
timeout: 3h
|
|
||||||
shutdown_timeout: 5m
|
|
||||||
labels:
|
|
||||||
- kata-poc:docker://docker.io/library/alpine:3.22
|
|
||||||
cache:
|
|
||||||
enabled: false
|
|
||||||
container:
|
|
||||||
network: ""
|
|
||||||
options: --add-host=host.docker.internal:host-gateway
|
|
||||||
docker_host: tcp://127.0.0.1:2375
|
|
||||||
require_docker: true
|
|
||||||
docker_timeout: 5m
|
|
||||||
force_pull: true
|
|
||||||
bind_workdir: false
|
|
||||||
@@ -1,111 +0,0 @@
|
|||||||
apiVersion: v1
|
|
||||||
kind: Pod
|
|
||||||
metadata:
|
|
||||||
generateName: kata-ephemeral-runner-
|
|
||||||
namespace: gitea-actions
|
|
||||||
labels:
|
|
||||||
app.kubernetes.io/name: kata-ephemeral-runner
|
|
||||||
spec:
|
|
||||||
runtimeClassName: kata
|
|
||||||
restartPolicy: Never
|
|
||||||
terminationGracePeriodSeconds: 330
|
|
||||||
automountServiceAccountToken: false
|
|
||||||
initContainers:
|
|
||||||
# Native sidecar: startupProbe gates the runner, and kubelet terminates this
|
|
||||||
# container after the ephemeral runner exits so the Kata sandbox can end.
|
|
||||||
- name: docker
|
|
||||||
image: zot.ad.ddupan.top/panxiao81/dind-fuse:29.7.1
|
|
||||||
imagePullPolicy: Never
|
|
||||||
restartPolicy: Always
|
|
||||||
env:
|
|
||||||
- name: DOCKER_TLS_CERTDIR
|
|
||||||
value: ""
|
|
||||||
- name: DIND_LOOPBACK_SIZE
|
|
||||||
value: 6G
|
|
||||||
- name: DIND_LOOPBACK_FILE
|
|
||||||
value: /loopback/docker.img
|
|
||||||
args:
|
|
||||||
- --host=tcp://0.0.0.0:2375
|
|
||||||
- --tls=false
|
|
||||||
- --mtu=1450
|
|
||||||
- --default-network-opt=bridge=com.docker.network.driver.mtu=1450
|
|
||||||
securityContext:
|
|
||||||
privileged: true
|
|
||||||
resources:
|
|
||||||
requests:
|
|
||||||
cpu: 250m
|
|
||||||
memory: 512Mi
|
|
||||||
ephemeral-storage: 1Gi
|
|
||||||
limits:
|
|
||||||
cpu: "4"
|
|
||||||
memory: 6Gi
|
|
||||||
ephemeral-storage: 8Gi
|
|
||||||
volumeMounts:
|
|
||||||
- name: docker-data
|
|
||||||
mountPath: /var/lib/docker
|
|
||||||
- name: loopback-data
|
|
||||||
mountPath: /loopback
|
|
||||||
startupProbe:
|
|
||||||
tcpSocket:
|
|
||||||
port: 2375
|
|
||||||
periodSeconds: 2
|
|
||||||
failureThreshold: 150
|
|
||||||
readinessProbe:
|
|
||||||
tcpSocket:
|
|
||||||
port: 2375
|
|
||||||
periodSeconds: 5
|
|
||||||
containers:
|
|
||||||
- name: runner
|
|
||||||
image: docker.io/gitea/runner:2.3.0
|
|
||||||
imagePullPolicy: Never
|
|
||||||
env:
|
|
||||||
- name: CONFIG_FILE
|
|
||||||
value: /config/config.yaml
|
|
||||||
- name: DOCKER_HOST
|
|
||||||
value: tcp://127.0.0.1:2375
|
|
||||||
- name: GITEA_INSTANCE_URL
|
|
||||||
value: https://git.ddupan.top
|
|
||||||
- name: GITEA_RUNNER_EPHEMERAL
|
|
||||||
value: "1"
|
|
||||||
- name: GITEA_RUNNER_NAME
|
|
||||||
valueFrom:
|
|
||||||
fieldRef:
|
|
||||||
fieldPath: metadata.name
|
|
||||||
- name: GITEA_RUNNER_LABELS
|
|
||||||
value: kata-poc:docker://docker.io/library/alpine:3.22
|
|
||||||
- name: GITEA_RUNNER_REGISTRATION_TOKEN_FILE
|
|
||||||
value: /registration/token
|
|
||||||
resources:
|
|
||||||
requests:
|
|
||||||
cpu: 100m
|
|
||||||
memory: 128Mi
|
|
||||||
limits:
|
|
||||||
cpu: "1"
|
|
||||||
memory: 1Gi
|
|
||||||
volumeMounts:
|
|
||||||
- name: config
|
|
||||||
mountPath: /config
|
|
||||||
readOnly: true
|
|
||||||
- name: registration
|
|
||||||
mountPath: /registration
|
|
||||||
readOnly: true
|
|
||||||
- name: runner-state
|
|
||||||
mountPath: /runner-state
|
|
||||||
volumes:
|
|
||||||
- name: config
|
|
||||||
configMap:
|
|
||||||
name: kata-ephemeral-runner-config
|
|
||||||
- name: registration
|
|
||||||
secret:
|
|
||||||
secretName: gitea-runner-registration
|
|
||||||
- name: runner-state
|
|
||||||
emptyDir:
|
|
||||||
medium: Memory
|
|
||||||
sizeLimit: 16Mi
|
|
||||||
- name: docker-data
|
|
||||||
emptyDir:
|
|
||||||
sizeLimit: 8Gi
|
|
||||||
- name: loopback-data
|
|
||||||
emptyDir:
|
|
||||||
medium: Memory
|
|
||||||
sizeLimit: 6Gi
|
|
||||||
@@ -1,24 +0,0 @@
|
|||||||
# This file is maintained automatically by "terraform init".
|
|
||||||
# Manual edits may be lost in future updates.
|
|
||||||
|
|
||||||
provider "registry.terraform.io/bpg/proxmox" {
|
|
||||||
version = "0.111.1"
|
|
||||||
constraints = "0.111.1"
|
|
||||||
hashes = [
|
|
||||||
"h1:ML2D3UUZTM99yrll/EBXj7wBYMb8xmQgomqFNybEoxY=",
|
|
||||||
"zh:18fb7c31a08dde6bffa1a4d4a211e604d6d17eec7092fd59331b3db3c6f3742c",
|
|
||||||
"zh:1cd60761538289d4dd2a1086b3ae62a7b0bdd4b1a2f824e9a44e243413168dba",
|
|
||||||
"zh:2eb76f6fc8299b6820ff678c8252332cc3366e226b5ae2e61748fd2449c1ed92",
|
|
||||||
"zh:45e6f7ebd0bf48911d37060359a4f359b5743b3092e985295733990e406d0416",
|
|
||||||
"zh:4aa8ba912eae37975d2e983394d173e595ca34fc76b5bf220b37d0e99d76e98c",
|
|
||||||
"zh:58e0789923103a77d502a0a9fc3eb920625e8eb935ec2d4ac0d006aebd1d186c",
|
|
||||||
"zh:6df8aa85fb8865915537e946c19b02538ad188018a629759c213c6f03730f642",
|
|
||||||
"zh:6ed47bc00d0913a1d0880618fa1376115e9edab6b4a658c081061a7f0e4ca360",
|
|
||||||
"zh:c5b10ff4f33df7e4c29e8f1127d49845b561b37b57517e844fb0954d7923d65e",
|
|
||||||
"zh:d016510e14b738499f0db9d9b3aafe82fc6877fb4ab4e9f831fb68a8d70a1385",
|
|
||||||
"zh:d941f394069bbf24351b363da1c64383f487067aaee0a84f9b96476d4912e212",
|
|
||||||
"zh:ddf271dbc2632ae8ffa8de3972f243ee47d260cb2ac90aa784f2746d98e21a0f",
|
|
||||||
"zh:ed0caa3501c42f611b7e9622c9b1df69fd85dc25a3cd88d3076381829688cd62",
|
|
||||||
"zh:f26e0763dbe6a6b2195c94b44696f2110f7f55433dc142839be16b9697fa5597",
|
|
||||||
]
|
|
||||||
}
|
|
||||||
@@ -1,95 +0,0 @@
|
|||||||
locals {
|
|
||||||
ssh_public_key = trimspace(file(pathexpand(var.ssh_public_key_file)))
|
|
||||||
}
|
|
||||||
|
|
||||||
data "proxmox_file" "ubuntu_noble" {
|
|
||||||
content_type = "vztmpl"
|
|
||||||
datastore_id = var.template_datastore
|
|
||||||
node_name = var.proxmox_node
|
|
||||||
file_name = var.template_file_name
|
|
||||||
}
|
|
||||||
|
|
||||||
resource "proxmox_virtual_environment_container" "kata_lxc_lab" {
|
|
||||||
node_name = var.proxmox_node
|
|
||||||
vm_id = var.container_id
|
|
||||||
|
|
||||||
description = "Disposable privileged LXC for validating k3s and Kata with direct host KVM access."
|
|
||||||
unprivileged = false
|
|
||||||
started = true
|
|
||||||
start_on_boot = false
|
|
||||||
tags = ["disposable", "kata", "lxc", "terraform"]
|
|
||||||
|
|
||||||
cpu {
|
|
||||||
cores = 4
|
|
||||||
}
|
|
||||||
|
|
||||||
memory {
|
|
||||||
dedicated = 8192
|
|
||||||
swap = 0
|
|
||||||
}
|
|
||||||
|
|
||||||
disk {
|
|
||||||
datastore_id = var.root_datastore
|
|
||||||
size = 16
|
|
||||||
}
|
|
||||||
|
|
||||||
features {
|
|
||||||
fuse = true
|
|
||||||
keyctl = true
|
|
||||||
mknod = true
|
|
||||||
nesting = true
|
|
||||||
}
|
|
||||||
|
|
||||||
device_passthrough {
|
|
||||||
path = "/dev/kvm"
|
|
||||||
mode = "0660"
|
|
||||||
}
|
|
||||||
|
|
||||||
device_passthrough {
|
|
||||||
path = "/dev/vhost-net"
|
|
||||||
mode = "0660"
|
|
||||||
}
|
|
||||||
|
|
||||||
device_passthrough {
|
|
||||||
path = "/dev/vhost-vsock"
|
|
||||||
mode = "0660"
|
|
||||||
}
|
|
||||||
|
|
||||||
device_passthrough {
|
|
||||||
path = "/dev/kmsg"
|
|
||||||
mode = "0660"
|
|
||||||
}
|
|
||||||
|
|
||||||
device_passthrough {
|
|
||||||
path = "/dev/net/tun"
|
|
||||||
mode = "0666"
|
|
||||||
}
|
|
||||||
|
|
||||||
initialization {
|
|
||||||
hostname = var.container_name
|
|
||||||
|
|
||||||
ip_config {
|
|
||||||
ipv4 {
|
|
||||||
address = "dhcp"
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
user_account {
|
|
||||||
keys = [local.ssh_public_key]
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
network_interface {
|
|
||||||
name = "eth0"
|
|
||||||
bridge = "vmbr0"
|
|
||||||
}
|
|
||||||
|
|
||||||
operating_system {
|
|
||||||
template_file_id = data.proxmox_file.ubuntu_noble.id
|
|
||||||
type = "ubuntu"
|
|
||||||
}
|
|
||||||
|
|
||||||
wait_for_ip {
|
|
||||||
ipv4 = true
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,7 +0,0 @@
|
|||||||
output "container_id" {
|
|
||||||
value = proxmox_virtual_environment_container.kata_lxc_lab.vm_id
|
|
||||||
}
|
|
||||||
|
|
||||||
output "ipv4" {
|
|
||||||
value = proxmox_virtual_environment_container.kata_lxc_lab.ipv4
|
|
||||||
}
|
|
||||||
@@ -1,15 +0,0 @@
|
|||||||
provider "proxmox" {
|
|
||||||
endpoint = var.proxmox_endpoint
|
|
||||||
api_token = var.proxmox_api_token
|
|
||||||
|
|
||||||
ssh {
|
|
||||||
agent = false
|
|
||||||
username = "root"
|
|
||||||
private_key = file(pathexpand(var.proxmox_ssh_private_key_file))
|
|
||||||
|
|
||||||
node {
|
|
||||||
name = var.proxmox_node
|
|
||||||
address = var.proxmox_node_address
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,54 +0,0 @@
|
|||||||
variable "proxmox_endpoint" {
|
|
||||||
type = string
|
|
||||||
default = "https://pve1.ad.ddupan.top:8006"
|
|
||||||
}
|
|
||||||
|
|
||||||
variable "proxmox_api_token" {
|
|
||||||
type = string
|
|
||||||
sensitive = true
|
|
||||||
}
|
|
||||||
|
|
||||||
variable "proxmox_node" {
|
|
||||||
type = string
|
|
||||||
default = "pve2"
|
|
||||||
}
|
|
||||||
|
|
||||||
variable "proxmox_node_address" {
|
|
||||||
type = string
|
|
||||||
default = "192.168.10.7"
|
|
||||||
}
|
|
||||||
|
|
||||||
variable "proxmox_ssh_private_key_file" {
|
|
||||||
type = string
|
|
||||||
default = "~/.ssh/id_ed25519"
|
|
||||||
}
|
|
||||||
|
|
||||||
variable "ssh_public_key_file" {
|
|
||||||
type = string
|
|
||||||
default = "~/.ssh/id_ed25519.pub"
|
|
||||||
}
|
|
||||||
|
|
||||||
variable "container_id" {
|
|
||||||
type = number
|
|
||||||
default = 147
|
|
||||||
}
|
|
||||||
|
|
||||||
variable "container_name" {
|
|
||||||
type = string
|
|
||||||
default = "kata-lxc-lab"
|
|
||||||
}
|
|
||||||
|
|
||||||
variable "template_datastore" {
|
|
||||||
type = string
|
|
||||||
default = "laptop"
|
|
||||||
}
|
|
||||||
|
|
||||||
variable "template_file_name" {
|
|
||||||
type = string
|
|
||||||
default = "ubuntu-24.04-standard_24.04-2_amd64.tar.zst"
|
|
||||||
}
|
|
||||||
|
|
||||||
variable "root_datastore" {
|
|
||||||
type = string
|
|
||||||
default = "local-lvm"
|
|
||||||
}
|
|
||||||
@@ -1,10 +0,0 @@
|
|||||||
terraform {
|
|
||||||
required_version = ">= 1.10"
|
|
||||||
|
|
||||||
required_providers {
|
|
||||||
proxmox = {
|
|
||||||
source = "bpg/proxmox"
|
|
||||||
version = "0.111.1"
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,53 +0,0 @@
|
|||||||
# Gitea kind microVM runner
|
|
||||||
|
|
||||||
`kind-microvm` 是专门运行 kind / nested Kubernetes CI 的 runner label。每个 job
|
|
||||||
独占一台 Cloud Hypervisor VM；guest 内直接运行 Docker 与 ephemeral Gitea Runner，
|
|
||||||
不经过 Kata。VM 完成一个 job 后关机，临时 COW 磁盘随即删除。
|
|
||||||
|
|
||||||
集群内的 `controller.py` 消费 Gitea `workflow_job` webhook。仅当
|
|
||||||
`action=queued` 且 `workflow_job.labels` 包含 `kind-microvm` 时，向 NATS JetStream
|
|
||||||
的 `ci.runner.kind-microvm` subject 发布消息。`workflow_job.id` 写入
|
|
||||||
`Nats-Msg-Id`，重复 webhook 不会生成第二个任务。
|
|
||||||
|
|
||||||
pve2 LXC 内的 `worker.py` 使用 durable pull consumer 领取消息。领取后启动一台
|
|
||||||
microVM，运行期间每分钟发送 `InProgress`；VM/ephemeral runner 正常退出才 ACK，
|
|
||||||
启动失败则 NAK 并延迟重试。当前 `RUNNER_CAPACITY=1` 只是 pve2 的资源配置；以后可
|
|
||||||
提高单 worker capacity，或在其他宿主机增加共享同一 durable consumer 的 worker，
|
|
||||||
而无需修改 webhook/controller。相同 runner 类型必须共享 durable；WorkQueue stream
|
|
||||||
不允许多个 consumer 使用重叠的 subject filter。
|
|
||||||
|
|
||||||
worker 的凭据位于：
|
|
||||||
|
|
||||||
```text
|
|
||||||
/etc/microvm-runner/registration-token
|
|
||||||
/etc/microvm-runner/nats-password
|
|
||||||
```
|
|
||||||
|
|
||||||
二者必须为 root-only 文件，不能写入镜像、cloud-init seed 或仓库。worker 为每次
|
|
||||||
启动生成一次性 nonce；guest 只能从 TAP 网段请求一次 registration token，请求后
|
|
||||||
立即从 worker 内存删除。runner 注册成功后删除 guest 内的 token 文件，再启动
|
|
||||||
daemon。这样 job 无法从 seed disk 或 Docker inspect 取回可复用 registration token。
|
|
||||||
|
|
||||||
NATS stream 使用 `WorkQueuePolicy`，ACK 后立即删除消息；同时限制为 24 小时、
|
|
||||||
10000 条和 256 MiB，异常积压也不会无限增长。NATS 平台配置见
|
|
||||||
`../../platform/nats/`。
|
|
||||||
|
|
||||||
2026-09-16 PoC 已验证：Cloud Hypervisor v52.0、Ubuntu 24.04、4 vCPU、3 GiB RAM、
|
|
||||||
10 GiB COW disk 中，Docker 29.1.3 成功创建 kind v0.27.0 / Kubernetes v1.32.2
|
|
||||||
集群，并运行 `busybox:1.37` smoke Pod。冷启动约 20 秒。pve2 资源只允许一台：
|
|
||||||
VMM cgroup（guest RAM 与 page cache）峰值约 6.2 GiB。
|
|
||||||
|
|
||||||
正式启用前还需要安装 launcher、制作不含凭据的基础镜像、部署 controller，并在
|
|
||||||
Gitea 配置只发送 `workflow_job` 的 instance/organization webhook。workflow 使用：
|
|
||||||
|
|
||||||
```yaml
|
|
||||||
runs-on: kind-microvm
|
|
||||||
```
|
|
||||||
|
|
||||||
当前 pve2 配置：
|
|
||||||
|
|
||||||
```ini
|
|
||||||
RUNNER_CAPACITY=1
|
|
||||||
NATS_DURABLE=kind-microvm
|
|
||||||
RUNNER_MAX_INFLIGHT=64
|
|
||||||
```
|
|
||||||
@@ -1,113 +0,0 @@
|
|||||||
#!/usr/bin/env python3
|
|
||||||
"""Gitea workflow_job webhook to NATS JetStream producer."""
|
|
||||||
|
|
||||||
import hashlib
|
|
||||||
import hmac
|
|
||||||
import json
|
|
||||||
import os
|
|
||||||
import ssl
|
|
||||||
from pathlib import Path
|
|
||||||
|
|
||||||
import nats
|
|
||||||
from aiohttp import web
|
|
||||||
from nats.js.api import DiscardPolicy, RetentionPolicy, StorageType, StreamConfig
|
|
||||||
from nats.js.errors import NotFoundError
|
|
||||||
|
|
||||||
LABEL = os.environ.get("RUNNER_LABEL", "kind-microvm")
|
|
||||||
SUBJECT = os.environ.get("NATS_SUBJECT", f"ci.runner.{LABEL}")
|
|
||||||
STREAM = os.environ.get("NATS_STREAM", "CI_RUNNER")
|
|
||||||
NATS_URL = os.environ.get("NATS_URL", "tls://nats.nats.svc.cluster.local:4222")
|
|
||||||
NATS_USER = os.environ.get("NATS_USER", "ci-producer")
|
|
||||||
NATS_PASSWORD_FILE = Path(os.environ.get("NATS_PASSWORD_FILE", "/run/secrets/nats/password"))
|
|
||||||
NATS_CA_FILE = os.environ.get("NATS_CA_FILE", "/etc/ssl/certs/ca-certificates.crt")
|
|
||||||
WEBHOOK_SECRET_FILE = Path(os.environ.get("WEBHOOK_SECRET_FILE", "/run/secrets/gitea/webhook-secret"))
|
|
||||||
|
|
||||||
|
|
||||||
def accepts(payload: object) -> tuple[bool, str | None]:
|
|
||||||
if not isinstance(payload, dict) or payload.get("action") != "queued":
|
|
||||||
return False, None
|
|
||||||
job = payload.get("workflow_job")
|
|
||||||
if not isinstance(job, dict) or LABEL not in job.get("labels", []):
|
|
||||||
return False, None
|
|
||||||
job_id = job.get("id")
|
|
||||||
if not isinstance(job_id, int):
|
|
||||||
return False, None
|
|
||||||
return True, str(job_id)
|
|
||||||
|
|
||||||
|
|
||||||
def valid_signature(body: bytes, signature: str) -> bool:
|
|
||||||
expected = hmac.new(WEBHOOK_SECRET_FILE.read_bytes().strip(), body, hashlib.sha256).hexdigest()
|
|
||||||
return hmac.compare_digest(signature, expected)
|
|
||||||
|
|
||||||
|
|
||||||
async def ensure_stream(js: object) -> None:
|
|
||||||
config = StreamConfig(
|
|
||||||
name=STREAM,
|
|
||||||
subjects=["ci.runner.*"],
|
|
||||||
retention=RetentionPolicy.WORK_QUEUE,
|
|
||||||
storage=StorageType.FILE,
|
|
||||||
discard=DiscardPolicy.OLD,
|
|
||||||
max_age=24 * 60 * 60,
|
|
||||||
max_msgs=10_000,
|
|
||||||
max_bytes=256 * 1024 * 1024,
|
|
||||||
duplicate_window=24 * 60 * 60,
|
|
||||||
)
|
|
||||||
try:
|
|
||||||
await js.stream_info(STREAM)
|
|
||||||
except NotFoundError:
|
|
||||||
await js.add_stream(config=config)
|
|
||||||
else:
|
|
||||||
await js.update_stream(config=config)
|
|
||||||
|
|
||||||
|
|
||||||
async def webhook(request: web.Request) -> web.Response:
|
|
||||||
body = await request.read()
|
|
||||||
if not valid_signature(body, request.headers.get("X-Gitea-Signature", "")):
|
|
||||||
raise web.HTTPUnauthorized()
|
|
||||||
try:
|
|
||||||
payload = json.loads(body)
|
|
||||||
except json.JSONDecodeError as error:
|
|
||||||
raise web.HTTPBadRequest(text="invalid JSON\n") from error
|
|
||||||
accepted, job_id = accepts(payload)
|
|
||||||
if not accepted:
|
|
||||||
return web.Response(status=204)
|
|
||||||
await request.app["js"].publish(
|
|
||||||
SUBJECT,
|
|
||||||
body,
|
|
||||||
headers={"Nats-Msg-Id": f"gitea-workflow-job-{job_id}"},
|
|
||||||
)
|
|
||||||
return web.Response(status=202, text="queued\n")
|
|
||||||
|
|
||||||
|
|
||||||
async def health(request: web.Request) -> web.Response:
|
|
||||||
return web.Response(text="ok\n" if request.app["nc"].is_connected else "disconnected\n",
|
|
||||||
status=200 if request.app["nc"].is_connected else 503)
|
|
||||||
|
|
||||||
|
|
||||||
async def nats_context(app: web.Application):
|
|
||||||
tls = ssl.create_default_context(cafile=NATS_CA_FILE)
|
|
||||||
nc = await nats.connect(
|
|
||||||
NATS_URL,
|
|
||||||
user=NATS_USER,
|
|
||||||
password=NATS_PASSWORD_FILE.read_text().strip(),
|
|
||||||
tls=tls,
|
|
||||||
name="microvm-runner-controller",
|
|
||||||
)
|
|
||||||
app["nc"] = nc
|
|
||||||
app["js"] = nc.jetstream()
|
|
||||||
await ensure_stream(app["js"])
|
|
||||||
yield
|
|
||||||
await nc.drain()
|
|
||||||
|
|
||||||
|
|
||||||
def create_app() -> web.Application:
|
|
||||||
app = web.Application(client_max_size=1024 * 1024)
|
|
||||||
app.cleanup_ctx.append(nats_context)
|
|
||||||
app.router.add_post("/webhook", webhook)
|
|
||||||
app.router.add_get("/healthz", health)
|
|
||||||
return app
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
|
||||||
web.run_app(create_app(), host=os.environ.get("LISTEN", "0.0.0.0"),
|
|
||||||
port=int(os.environ.get("PORT", "8787")))
|
|
||||||
@@ -1,19 +0,0 @@
|
|||||||
[Unit]
|
|
||||||
Description=Gitea kind microVM runner worker
|
|
||||||
After=network-online.target
|
|
||||||
Wants=network-online.target
|
|
||||||
|
|
||||||
[Service]
|
|
||||||
Type=simple
|
|
||||||
EnvironmentFile=-/etc/microvm-runner/worker.env
|
|
||||||
ExecStart=/opt/microvm-runner/venv/bin/python /opt/microvm-runner/worker.py
|
|
||||||
Restart=on-failure
|
|
||||||
RestartSec=5s
|
|
||||||
SupplementaryGroups=kvm
|
|
||||||
PrivateTmp=yes
|
|
||||||
ProtectHome=yes
|
|
||||||
ProtectSystem=strict
|
|
||||||
ReadWritePaths=/var/lib/microvm-runner /run/microvm-runner
|
|
||||||
|
|
||||||
[Install]
|
|
||||||
WantedBy=multi-user.target
|
|
||||||
@@ -1,2 +0,0 @@
|
|||||||
aiohttp==3.12.15
|
|
||||||
nats-py==2.11.0
|
|
||||||
@@ -1,138 +0,0 @@
|
|||||||
#!/usr/bin/env python3
|
|
||||||
"""Capacity-bounded JetStream consumer that launches one ephemeral VM per job."""
|
|
||||||
|
|
||||||
import asyncio
|
|
||||||
import os
|
|
||||||
import secrets
|
|
||||||
import ssl
|
|
||||||
import uuid
|
|
||||||
from pathlib import Path
|
|
||||||
|
|
||||||
import nats
|
|
||||||
from aiohttp import web
|
|
||||||
from nats.errors import TimeoutError
|
|
||||||
from nats.js.api import AckPolicy, ConsumerConfig
|
|
||||||
|
|
||||||
CAPACITY = int(os.environ.get("RUNNER_CAPACITY", "1"))
|
|
||||||
SUBJECT = os.environ.get("NATS_SUBJECT", "ci.runner.kind-microvm")
|
|
||||||
STREAM = os.environ.get("NATS_STREAM", "CI_RUNNER")
|
|
||||||
DURABLE = os.environ.get("NATS_DURABLE", "kind-microvm")
|
|
||||||
MAX_INFLIGHT = int(os.environ.get("RUNNER_MAX_INFLIGHT", "64"))
|
|
||||||
NATS_URL = os.environ.get("NATS_URL", "tls://nats.ad.ddupan.top:4222")
|
|
||||||
NATS_USER = os.environ.get("NATS_USER", "ci-worker")
|
|
||||||
NATS_PASSWORD_FILE = Path(os.environ.get("NATS_PASSWORD_FILE", "/etc/microvm-runner/nats-password"))
|
|
||||||
NATS_CA_FILE = os.environ.get("NATS_CA_FILE", "/etc/ssl/certs/ca-certificates.crt")
|
|
||||||
REGISTRATION_TOKEN_FILE = Path(os.environ.get("REGISTRATION_TOKEN_FILE", "/etc/microvm-runner/registration-token"))
|
|
||||||
LAUNCHER = os.environ.get("LAUNCHER", "/usr/local/libexec/microvm-runner-launch")
|
|
||||||
TOKEN_LISTEN = os.environ.get("TOKEN_LISTEN", "172.30.0.1")
|
|
||||||
TOKEN_PORT = int(os.environ.get("TOKEN_PORT", "8787"))
|
|
||||||
|
|
||||||
tokens: dict[str, bytes] = {}
|
|
||||||
token_lock = asyncio.Lock()
|
|
||||||
|
|
||||||
|
|
||||||
async def token(request: web.Request) -> web.Response:
|
|
||||||
nonce = request.match_info["nonce"]
|
|
||||||
async with token_lock:
|
|
||||||
value = tokens.pop(nonce, None)
|
|
||||||
if value is None:
|
|
||||||
raise web.HTTPNotFound()
|
|
||||||
return web.Response(body=value, headers={"Cache-Control": "no-store"})
|
|
||||||
|
|
||||||
|
|
||||||
async def heartbeat(message: object, stop: asyncio.Event) -> None:
|
|
||||||
while True:
|
|
||||||
try:
|
|
||||||
await asyncio.wait_for(stop.wait(), timeout=60)
|
|
||||||
return
|
|
||||||
except asyncio.TimeoutError:
|
|
||||||
await message.in_progress()
|
|
||||||
|
|
||||||
|
|
||||||
async def run_one(message: object) -> None:
|
|
||||||
instance_id = str(uuid.uuid4())
|
|
||||||
nonce = secrets.token_urlsafe(32)
|
|
||||||
async with token_lock:
|
|
||||||
tokens[nonce] = REGISTRATION_TOKEN_FILE.read_bytes().strip()
|
|
||||||
stop = asyncio.Event()
|
|
||||||
pulse = asyncio.create_task(heartbeat(message, stop))
|
|
||||||
try:
|
|
||||||
process = await asyncio.create_subprocess_exec(LAUNCHER, instance_id, nonce)
|
|
||||||
return_code = await process.wait()
|
|
||||||
if return_code == 0:
|
|
||||||
await message.ack()
|
|
||||||
else:
|
|
||||||
await message.nak(delay=30)
|
|
||||||
except Exception:
|
|
||||||
await message.nak(delay=30)
|
|
||||||
raise
|
|
||||||
finally:
|
|
||||||
stop.set()
|
|
||||||
await pulse
|
|
||||||
async with token_lock:
|
|
||||||
tokens.pop(nonce, None)
|
|
||||||
|
|
||||||
|
|
||||||
async def consume() -> None:
|
|
||||||
if CAPACITY < 1:
|
|
||||||
raise ValueError("RUNNER_CAPACITY must be at least 1")
|
|
||||||
tls = ssl.create_default_context(cafile=NATS_CA_FILE)
|
|
||||||
nc = await nats.connect(
|
|
||||||
NATS_URL,
|
|
||||||
user=NATS_USER,
|
|
||||||
password=NATS_PASSWORD_FILE.read_text().strip(),
|
|
||||||
tls=tls,
|
|
||||||
name=DURABLE,
|
|
||||||
)
|
|
||||||
js = nc.jetstream()
|
|
||||||
subscription = await js.pull_subscribe(
|
|
||||||
SUBJECT,
|
|
||||||
durable=DURABLE,
|
|
||||||
stream=STREAM,
|
|
||||||
config=ConsumerConfig(
|
|
||||||
durable_name=DURABLE,
|
|
||||||
filter_subject=SUBJECT,
|
|
||||||
ack_policy=AckPolicy.EXPLICIT,
|
|
||||||
ack_wait=5 * 60,
|
|
||||||
# This durable consumer is shared by every host of this runner type.
|
|
||||||
# Local CAPACITY controls each host; this is only a global safety cap.
|
|
||||||
max_ack_pending=MAX_INFLIGHT,
|
|
||||||
max_deliver=5,
|
|
||||||
),
|
|
||||||
)
|
|
||||||
active: set[asyncio.Task[None]] = set()
|
|
||||||
try:
|
|
||||||
while True:
|
|
||||||
active = {task for task in active if not task.done()}
|
|
||||||
free = CAPACITY - len(active)
|
|
||||||
if free == 0:
|
|
||||||
await asyncio.wait(active, return_when=asyncio.FIRST_COMPLETED)
|
|
||||||
continue
|
|
||||||
try:
|
|
||||||
messages = await subscription.fetch(batch=free, timeout=5)
|
|
||||||
except TimeoutError:
|
|
||||||
continue
|
|
||||||
for message in messages:
|
|
||||||
task = asyncio.create_task(run_one(message))
|
|
||||||
task.add_done_callback(lambda done: done.exception())
|
|
||||||
active.add(task)
|
|
||||||
finally:
|
|
||||||
if active:
|
|
||||||
await asyncio.gather(*active, return_exceptions=True)
|
|
||||||
await nc.drain()
|
|
||||||
|
|
||||||
|
|
||||||
async def main() -> None:
|
|
||||||
app = web.Application()
|
|
||||||
app.router.add_get("/token/{nonce}", token)
|
|
||||||
runner = web.AppRunner(app)
|
|
||||||
await runner.setup()
|
|
||||||
await web.TCPSite(runner, TOKEN_LISTEN, TOKEN_PORT).start()
|
|
||||||
try:
|
|
||||||
await consume()
|
|
||||||
finally:
|
|
||||||
await runner.cleanup()
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
|
||||||
asyncio.run(main())
|
|
||||||
@@ -1,9 +0,0 @@
|
|||||||
ARG DOCKER_VERSION=29.7.1
|
|
||||||
FROM docker:${DOCKER_VERSION}-dind
|
|
||||||
|
|
||||||
RUN apk add --no-cache fuse-overlayfs
|
|
||||||
|
|
||||||
COPY --chmod=0755 entrypoint.sh /usr/local/bin/dind-fuse-entrypoint
|
|
||||||
|
|
||||||
ENTRYPOINT ["/usr/local/bin/dind-fuse-entrypoint"]
|
|
||||||
CMD []
|
|
||||||
@@ -1,42 +0,0 @@
|
|||||||
#!/bin/sh
|
|
||||||
set -eu
|
|
||||||
|
|
||||||
# /dev belongs to the Kata guest. A privileged dockerd container can create
|
|
||||||
# this standard FUSE node without exposing the PVE host's /dev/fuse to the LXC.
|
|
||||||
if [ ! -e /dev/fuse ]; then
|
|
||||||
mknod /dev/fuse c 10 229
|
|
||||||
fi
|
|
||||||
chmod 0666 /dev/fuse
|
|
||||||
|
|
||||||
# kind's nested kubelet opens /dev/kmsg. This node belongs to the Kata guest;
|
|
||||||
# exposing it to a kind node does not expose the LXC or PVE host kernel log.
|
|
||||||
if [ ! -e /dev/kmsg ]; then
|
|
||||||
mknod /dev/kmsg c 1 11
|
|
||||||
fi
|
|
||||||
chmod 0600 /dev/kmsg
|
|
||||||
|
|
||||||
storage_driver=fuse-overlayfs
|
|
||||||
|
|
||||||
# kind's kubelet/cAdvisor cannot map a virtiofs-backed fuse-overlayfs root to a
|
|
||||||
# block device. When requested, create an ext4 filesystem on a guest-local loop
|
|
||||||
# device. The backing file, loop device and filesystem all die with the Kata VM.
|
|
||||||
if [ -n "${DIND_LOOPBACK_SIZE:-}" ]; then
|
|
||||||
[ -e /dev/loop-control ] || mknod /dev/loop-control c 10 237
|
|
||||||
i=0
|
|
||||||
while [ "$i" -lt 8 ]; do
|
|
||||||
[ -e "/dev/loop$i" ] || mknod "/dev/loop$i" b 7 "$i"
|
|
||||||
i=$((i + 1))
|
|
||||||
done
|
|
||||||
|
|
||||||
backing_file="${DIND_LOOPBACK_FILE:-/var/lib/docker-loopback.img}"
|
|
||||||
truncate -s "$DIND_LOOPBACK_SIZE" "$backing_file"
|
|
||||||
# Alpine's BusyBox losetup supports -f, but not util-linux's
|
|
||||||
# --find/--show long options.
|
|
||||||
loop_device="$(losetup -f)"
|
|
||||||
losetup "$loop_device" "$backing_file"
|
|
||||||
mkfs.ext4 -q -F -m 0 "$loop_device"
|
|
||||||
mount "$loop_device" /var/lib/docker
|
|
||||||
storage_driver=overlay2
|
|
||||||
fi
|
|
||||||
|
|
||||||
exec dockerd-entrypoint.sh --storage-driver="$storage_driver" "$@"
|
|
||||||
Reference in New Issue
Block a user