diff --git a/.gitea/workflows/dind-fuse-image.yml b/.gitea/workflows/dind-fuse-image.yml new file mode 100644 index 0000000..817962a --- /dev/null +++ b/.gitea/workflows/dind-fuse-image.yml @@ -0,0 +1,32 @@ +--- +name: dind-fuse-image + +on: + workflow_dispatch: + push: + branches: [main] + paths: + - platform/gitea-runner/images/dind-fuse/** + - .gitea/workflows/dind-fuse-image.yml + +jobs: + publish: + runs-on: [self-hosted, pod] + steps: + - uses: actions/checkout@v4 + + - name: Log in to the Gitea container registry + uses: docker/login-action@v3 + with: + registry: git.ddupan.top + username: ${{ gitea.actor }} + password: ${{ secrets.REGISTRY_TOKEN }} + + - name: Build and publish + uses: docker/build-push-action@v6 + with: + context: platform/gitea-runner/images/dind-fuse + push: true + tags: | + git.ddupan.top/panxiao81/dind-fuse:29.7.1 + git.ddupan.top/panxiao81/dind-fuse:latest diff --git a/.gitea/workflows/kata-runner-smoke.yml b/.gitea/workflows/kata-runner-smoke.yml new file mode 100644 index 0000000..4bb1707 --- /dev/null +++ b/.gitea/workflows/kata-runner-smoke.yml @@ -0,0 +1,24 @@ +--- +name: kata-runner-smoke + +on: + workflow_dispatch: + push: + branches: + - poc/kata-runner-smoke + paths: + - .gitea/workflows/kata-runner-smoke.yml + +jobs: + smoke: + runs-on: [self-hosted, pod] + steps: + - name: Verify isolated job environment + shell: sh + run: | + set -eu + uname -a + grep -q '^ID=alpine$' /etc/os-release + test -f /.dockerenv + test ! -e /dev/kvm + printf 'kata ephemeral runner job ok\n' diff --git a/.gitea/workflows/kind-on-kata-smoke.yml b/.gitea/workflows/kind-on-kata-smoke.yml new file mode 100644 index 0000000..b49d070 --- /dev/null +++ b/.gitea/workflows/kind-on-kata-smoke.yml @@ -0,0 +1,63 @@ +--- +name: kind-on-kata-smoke + +on: + push: + branches: + - poc/kind-on-kata + paths: + - .gitea/workflows/kind-on-kata-smoke.yml + workflow_dispatch: + +jobs: + smoke: + runs-on: [self-hosted, pod] + steps: + - name: Create nested kind cluster + shell: sh + env: + KIND_VERSION: v0.27.0 + KIND_NODE_IMAGE: kindest/node:v1.32.2@sha256:f226345927d7e348497136874b6d207e0b32cc52154ad8323129352923a3142f + run: | + set -eu + apk add --no-cache ca-certificates curl docker-cli + curl --retry 5 --retry-all-errors --connect-timeout 15 -fsSLo /tmp/kind \ + "https://kind.sigs.k8s.io/dl/${KIND_VERSION}/kind-linux-amd64" + curl --retry 5 --retry-all-errors --connect-timeout 15 -fsSLo /tmp/kind.sha256sum \ + "https://kind.sigs.k8s.io/dl/${KIND_VERSION}/kind-linux-amd64.sha256sum" + expected="$(awk '{print $1}' /tmp/kind.sha256sum)" + printf '%s %s\n' "$expected" /tmp/kind | sha256sum -c - + install -m 0755 /tmp/kind /usr/local/bin/kind + docker info --format 'kernel={{.KernelVersion}} driver={{.Driver}}' + test "$(docker info --format '{{.Driver}}')" = overlay2 + + cleanup() { + kind delete cluster --name nested >/dev/null 2>&1 || true + } + trap cleanup EXIT + + cat >/tmp/kind-config.yaml <<'EOF' + kind: Cluster + apiVersion: kind.x-k8s.io/v1alpha4 + name: nested + nodes: + - role: control-plane + extraMounts: + - hostPath: /dev/kmsg + containerPath: /dev/kmsg + EOF + kind create cluster -v 9 --retain --config /tmp/kind-config.yaml --image "$KIND_NODE_IMAGE" --wait 5m + docker exec nested-control-plane kubectl \ + --kubeconfig=/etc/kubernetes/admin.conf wait \ + --for=condition=Ready node/nested-control-plane --timeout=2m + docker exec nested-control-plane kubectl \ + --kubeconfig=/etc/kubernetes/admin.conf run smoke \ + --image=docker.io/library/busybox:1.37 --restart=Never \ + --command -- sh -c 'echo kind-on-kata-ok' + docker exec nested-control-plane kubectl \ + --kubeconfig=/etc/kubernetes/admin.conf wait \ + --for=jsonpath='{.status.phase}'=Succeeded pod/smoke --timeout=2m + test "$(docker exec nested-control-plane kubectl \ + --kubeconfig=/etc/kubernetes/admin.conf logs smoke)" = kind-on-kata-ok + kind delete cluster --name nested + trap - EXIT diff --git a/infrastructure/kata-lab/README.md b/infrastructure/kata-lab/README.md new file mode 100644 index 0000000..0765ce8 --- /dev/null +++ b/infrastructure/kata-lab/README.md @@ -0,0 +1,72 @@ +# kata-lab + +`kata-lab` 是位于 Proxmox `pve2` 的可销毁验证环境,用来验证 nested KVM、k3s、 +Kata Containers,以及后续一 job 一 Kata Pod 的 Gitea Runner。它不是 dev/stg +长期服务,也不承载持久数据。 + +> 状态:VMID 147 已于 2026-09-14 销毁,Terraform state 为空。目录保留首次验证结果 +> 与可复现配置;后续验证转向跨 PVE 节点的轻量 LXC worker 方案。 + +## Terraform bootstrap + +1. 将 Proxmox API token 写入被 Git 忽略且权限为 `0600` 的 + `terraform/credentials.auto.tfvars`。 +2. 取得内部 CA,并仅为当前 Terraform 进程设置 Go TLS trust: + + ```bash + curl -fsSL https://bao.ad.ddupan.top:8200/v1/pki/ca/pem \ + -o /tmp/kata-lab-ddupan-ca.pem + export SSL_CERT_FILE=/tmp/kata-lab-ddupan-ca.pem + ``` + +3. 初始化并审阅 plan: + + ```bash + cd infrastructure/kata-lab/terraform + terraform init + terraform plan + ``` + +Terraform 使用 `laptop:import/noble-server-cloudimg-amd64.qcow2`(40 GiB +virtual size),在 +`pve-rg-hdd` 创建 VM root disk,并把 cloud-init snippet 放到 `laptop` +datastore。VM root disk 设为 41 GiB,以容纳 PVE import 的块对齐增量。VM 使用 +DHCP、`cpu.type = host`、4 vCPU 和 4 GiB 内存;cloud-init +安装 `qemu-guest-agent`、单节点 k3s,并记录 `/dev/kvm` 检查结果。 + +## Kata Containers + +Ansible 使用 Kata Containers 4.1.0 官方 `kata-deploy` chart。配置明确选择 k3s, +只安装 `qemu-runtime-rs`,不安装额外 snapshotter,并采用 `job` 模式,安装结束后 +不保留 privileged DaemonSet。默认 `RuntimeClass` 名为 `kata`。 + +Terraform 输出的 DHCP 地址变化时,先更新 `ansible/inventory/hosts.yml`,然后运行: + +```bash +cd infrastructure/kata-lab/ansible +ansible-playbook kata.yml +ansible-playbook verify.yml +``` + +`verify.yml` 首先创建一个 `runtimeClassName: kata` 的短生命周期 Pod,并确认 Pod 内 +看到的 Kata guest kernel 与 k3s node 的宿主 kernel 不同。随后它在另一个 Kata Pod +内启动 privileged `dockerd` sidecar,让普通 Docker client 容器通过共享网络命名空间 +的 `127.0.0.1:2375` 运行一个嵌套容器。这对应后续 Gitea Runner 的目标形态:runner +直接执行 job,只在需要 Docker API 时连接同 Pod 的 daemon,而不把 Docker executor +作为 job 的外层。Kata 下共享 volume 中的 Unix socket 文件虽然双方可见,但不能跨 +容器连接,因此不能在这里沿用普通 Pod 常见的 `/var/run/docker.sock` 方案;loopback +端口没有通过 Service 暴露到 Pod 外。 +验证 Pod 会保留以供排查;再次运行时会先替换它们。 + +2026-09-13 的首次验证结果:Kata guest kernel 为 `6.18.35`,node host kernel 为 +`6.8.0-90-generic`;sidecar daemon 成功运行 `busybox:1.37` 嵌套容器。Docker 27.5.1 +在当前 Kata guest 中无法挂载 overlay2,自动回退到 `vfs`;功能验证通过,但正式用于 +CI 前需要解决或接受其镜像层复制与磁盘性能开销。 + +不要提交 `credentials.auto.tfvars`、Terraform state 或 saved plan。销毁前确认目标 +仍然是 VMID 147 / `kata-lab`: + +```bash +terraform plan -destroy +terraform destroy +``` diff --git a/infrastructure/kata-lab/ansible/ansible.cfg b/infrastructure/kata-lab/ansible/ansible.cfg new file mode 100644 index 0000000..7403f6b --- /dev/null +++ b/infrastructure/kata-lab/ansible/ansible.cfg @@ -0,0 +1,10 @@ +[defaults] +inventory = inventory/hosts.yml +host_key_checking = True +interpreter_python = auto_silent +retry_files_enabled = False +local_tmp = /tmp/ansible-kata-lab-local +remote_tmp = /tmp/ansible-kata-lab-remote + +[ssh_connection] +pipelining = True diff --git a/infrastructure/kata-lab/ansible/inventory/hosts.yml b/infrastructure/kata-lab/ansible/inventory/hosts.yml new file mode 100644 index 0000000..8135023 --- /dev/null +++ b/infrastructure/kata-lab/ansible/inventory/hosts.yml @@ -0,0 +1,8 @@ +--- +all: + hosts: + kata-lab: + ansible_host: 192.168.10.13 + ansible_user: ansible + vars: + ansible_become: true diff --git a/infrastructure/kata-lab/ansible/kata.yml b/infrastructure/kata-lab/ansible/kata.yml new file mode 100644 index 0000000..39df4b4 --- /dev/null +++ b/infrastructure/kata-lab/ansible/kata.yml @@ -0,0 +1,88 @@ +--- +- name: Install Kata Containers in the disposable k3s lab + hosts: kata-lab + gather_facts: false + vars: + kata_version: "4.1.0" + kata_chart_url: >- + https://github.com/kata-containers/kata-containers/releases/download/{{ kata_version }}/kata-deploy-{{ kata_version }}.tgz + tasks: + - name: Check nested KVM is usable + ansible.builtin.stat: + path: /dev/kvm + register: kata_kvm + + - name: Require nested KVM + ansible.builtin.assert: + that: + - kata_kvm.stat.exists + - kata_kvm.stat.ischr + fail_msg: /dev/kvm is unavailable; Kata QEMU cannot start efficiently + + - name: Install Kata through the k3s Helm controller + ansible.builtin.copy: + dest: /var/lib/rancher/k3s/server/manifests/kata-deploy.yaml + owner: root + group: root + mode: "0644" + content: | + apiVersion: helm.cattle.io/v1 + kind: HelmChart + metadata: + name: kata-deploy + namespace: kube-system + spec: + chart: {{ kata_chart_url }} + targetNamespace: kata-system + createNamespace: true + timeout: 15m + failurePolicy: abort + valuesContent: |- + deploymentMode: job + k8sDistribution: k3s + snapshotter: + setup: [] + shims: + disableAll: true + qemu-runtime-rs: + enabled: true + runtimeClasses: + enabled: true + createDefault: true + node-feature-discovery: + enabled: false + + - name: Wait for the Helm installation job to appear + ansible.builtin.command: + cmd: k3s kubectl get job/helm-install-kata-deploy -n kube-system + register: kata_helm_job + retries: 60 + delay: 2 + until: kata_helm_job.rc == 0 + changed_when: false + + - name: Wait for the Helm installation job to finish + ansible.builtin.command: + cmd: >- + k3s kubectl wait --for=condition=complete + job/helm-install-kata-deploy -n kube-system --timeout=20m + changed_when: false + + - name: Wait for Kata node installation jobs + ansible.builtin.shell: + cmd: | + set -euo pipefail + jobs=$(k3s kubectl get jobs -n kata-system -l app.kubernetes.io/instance=kata-deploy -o name) + test -n "${jobs}" + k3s kubectl wait --for=condition=complete -n kata-system $jobs --timeout=20m + executable: /bin/bash + changed_when: false + + - name: Wait for the default Kata RuntimeClass + ansible.builtin.command: + cmd: k3s kubectl get runtimeclass kata + register: kata_runtime_class + retries: 30 + delay: 2 + until: kata_runtime_class.rc == 0 + changed_when: false diff --git a/infrastructure/kata-lab/ansible/verify.yml b/infrastructure/kata-lab/ansible/verify.yml new file mode 100644 index 0000000..664a5c5 --- /dev/null +++ b/infrastructure/kata-lab/ansible/verify.yml @@ -0,0 +1,125 @@ +--- +- name: Verify Kata isolation with a disposable Pod + hosts: kata-lab + gather_facts: false + tasks: + - name: Remove an earlier smoke Pod + ansible.builtin.command: + cmd: k3s kubectl delete pod kata-smoke --ignore-not-found --wait=true + changed_when: false + + - name: Create the Kata smoke Pod + ansible.builtin.shell: + cmd: | + set -o pipefail + k3s kubectl apply -f - <<'EOF' + apiVersion: v1 + kind: Pod + metadata: + name: kata-smoke + spec: + runtimeClassName: kata + restartPolicy: Never + containers: + - name: smoke + image: docker.io/library/busybox:1.37 + command: + - sh + - -c + - 'printf "guest-kernel="; uname -r; test -c /dev/kvm && exit 1 || true' + EOF + executable: /bin/bash + changed_when: true + + - name: Wait for the Kata Pod to finish + ansible.builtin.command: + cmd: k3s kubectl wait --for=jsonpath='{.status.phase}'=Succeeded pod/kata-smoke --timeout=10m + changed_when: false + + - name: Read the guest kernel version + ansible.builtin.command: + cmd: k3s kubectl logs kata-smoke + register: kata_smoke_log + changed_when: false + + - name: Read the host kernel version + ansible.builtin.command: + cmd: uname -r + register: kata_host_kernel + changed_when: false + + - name: Require a distinct guest kernel + ansible.builtin.assert: + that: + - kata_smoke_log.stdout is match('^guest-kernel=.+') + - kata_smoke_log.stdout | regex_replace('^guest-kernel=', '') != kata_host_kernel.stdout + success_msg: >- + Kata guest {{ kata_smoke_log.stdout }} differs from host-kernel={{ kata_host_kernel.stdout }} + + - name: Remove an earlier Docker sidecar smoke Pod + ansible.builtin.command: + cmd: k3s kubectl delete pod kata-docker-smoke --ignore-not-found --wait=true + changed_when: false + + - name: Create a Kata Pod with a Docker daemon sidecar + ansible.builtin.shell: + cmd: | + set -o pipefail + k3s kubectl apply -f - <<'EOF' + apiVersion: v1 + kind: Pod + metadata: + name: kata-docker-smoke + spec: + runtimeClassName: kata + restartPolicy: Never + volumes: + - name: docker-run + emptyDir: {} + containers: + - name: dockerd + image: docker.io/library/docker:27-dind + securityContext: + privileged: true + env: + - name: DOCKER_TLS_CERTDIR + value: "" + volumeMounts: + - name: docker-run + mountPath: /var/run + - name: client + image: docker.io/library/docker:27-cli + env: + - name: DOCKER_HOST + value: tcp://127.0.0.1:2375 + volumeMounts: + - name: docker-run + mountPath: /var/run + command: + - sh + - -c + - | + until docker info >/dev/null 2>&1; do sleep 1; done + docker run --rm docker.io/library/busybox:1.37 echo nested-docker-ok + EOF + executable: /bin/bash + changed_when: true + + - name: Wait for the Docker client to finish + ansible.builtin.command: + cmd: >- + k3s kubectl wait --for=jsonpath='{.status.containerStatuses[?(@.name=="client")].state.terminated.exitCode}'=0 + pod/kata-docker-smoke --timeout=10m + changed_when: false + + - name: Read the Docker client result + ansible.builtin.command: + cmd: k3s kubectl logs kata-docker-smoke -c client + register: kata_docker_smoke_log + changed_when: false + + - name: Require Docker to run a nested container + ansible.builtin.assert: + that: + - "'nested-docker-ok' in kata_docker_smoke_log.stdout" + success_msg: Docker daemon sidecar completed a nested container inside the Kata VM diff --git a/infrastructure/kata-lab/terraform/.terraform.lock.hcl b/infrastructure/kata-lab/terraform/.terraform.lock.hcl new file mode 100644 index 0000000..4794735 --- /dev/null +++ b/infrastructure/kata-lab/terraform/.terraform.lock.hcl @@ -0,0 +1,24 @@ +# This file is maintained automatically by "terraform init". +# Manual edits may be lost in future updates. + +provider "registry.terraform.io/bpg/proxmox" { + version = "0.111.1" + constraints = "0.111.1" + hashes = [ + "h1:ML2D3UUZTM99yrll/EBXj7wBYMb8xmQgomqFNybEoxY=", + "zh:18fb7c31a08dde6bffa1a4d4a211e604d6d17eec7092fd59331b3db3c6f3742c", + "zh:1cd60761538289d4dd2a1086b3ae62a7b0bdd4b1a2f824e9a44e243413168dba", + "zh:2eb76f6fc8299b6820ff678c8252332cc3366e226b5ae2e61748fd2449c1ed92", + "zh:45e6f7ebd0bf48911d37060359a4f359b5743b3092e985295733990e406d0416", + "zh:4aa8ba912eae37975d2e983394d173e595ca34fc76b5bf220b37d0e99d76e98c", + "zh:58e0789923103a77d502a0a9fc3eb920625e8eb935ec2d4ac0d006aebd1d186c", + "zh:6df8aa85fb8865915537e946c19b02538ad188018a629759c213c6f03730f642", + "zh:6ed47bc00d0913a1d0880618fa1376115e9edab6b4a658c081061a7f0e4ca360", + "zh:c5b10ff4f33df7e4c29e8f1127d49845b561b37b57517e844fb0954d7923d65e", + "zh:d016510e14b738499f0db9d9b3aafe82fc6877fb4ab4e9f831fb68a8d70a1385", + "zh:d941f394069bbf24351b363da1c64383f487067aaee0a84f9b96476d4912e212", + "zh:ddf271dbc2632ae8ffa8de3972f243ee47d260cb2ac90aa784f2746d98e21a0f", + "zh:ed0caa3501c42f611b7e9622c9b1df69fd85dc25a3cd88d3076381829688cd62", + "zh:f26e0763dbe6a6b2195c94b44696f2110f7f55433dc142839be16b9697fa5597", + ] +} diff --git a/infrastructure/kata-lab/terraform/cloud-init.yaml.tftpl b/infrastructure/kata-lab/terraform/cloud-init.yaml.tftpl new file mode 100644 index 0000000..2adfb00 --- /dev/null +++ b/infrastructure/kata-lab/terraform/cloud-init.yaml.tftpl @@ -0,0 +1,34 @@ +#cloud-config +hostname: ${hostname} +manage_etc_hosts: true +timezone: Etc/UTC + +users: + - default + - name: ansible + groups: [adm, sudo] + shell: /bin/bash + lock_passwd: true + sudo: ALL=(ALL) NOPASSWD:ALL + ssh_authorized_keys: + - ${ssh_public_key} + +package_update: true +package_upgrade: false +packages: + - ca-certificates + - curl + - jq + - qemu-guest-agent + +runcmd: + - [systemctl, enable, --now, qemu-guest-agent] + - [modprobe, kvm_amd] + - [sh, -c, 'test -c /dev/kvm && echo available > /var/lib/cloud/nested-kvm.status || echo unavailable > /var/lib/cloud/nested-kvm.status'] + - [sh, -c, 'curl -sfL https://get.k3s.io -o /tmp/install-k3s.sh'] + - [chmod, '0755', /tmp/install-k3s.sh] + - [sh, -c, 'INSTALL_K3S_VERSION="${k3s_version}" INSTALL_K3S_EXEC="server --disable=traefik --write-kubeconfig-mode=0644" /tmp/install-k3s.sh'] + - [sh, -c, 'kubectl wait --for=condition=Ready node/${hostname} --timeout=180s'] + - [touch, /var/lib/cloud/kata-lab-bootstrap.done] + +final_message: "kata-lab bootstrap completed after $UPTIME seconds" diff --git a/infrastructure/kata-lab/terraform/credentials.auto.tfvars.example b/infrastructure/kata-lab/terraform/credentials.auto.tfvars.example new file mode 100644 index 0000000..ff27f53 --- /dev/null +++ b/infrastructure/kata-lab/terraform/credentials.auto.tfvars.example @@ -0,0 +1,3 @@ +# Copy this file to credentials.auto.tfvars and replace the placeholder. +# Format: user@realm!token-id=token-secret +proxmox_api_token = "REPLACE_ME" diff --git a/infrastructure/kata-lab/terraform/main.tf b/infrastructure/kata-lab/terraform/main.tf new file mode 100644 index 0000000..b94cbe3 --- /dev/null +++ b/infrastructure/kata-lab/terraform/main.tf @@ -0,0 +1,86 @@ +locals { + ssh_public_key = trimspace(file(pathexpand(var.ssh_public_key_file))) + cloud_init = templatefile("${path.module}/cloud-init.yaml.tftpl", { + hostname = var.vm_name + ssh_public_key = local.ssh_public_key + k3s_version = var.k3s_version + }) +} + +data "proxmox_file" "ubuntu_noble" { + content_type = "import" + datastore_id = var.snippet_datastore_id + node_name = var.node_name + file_name = "noble-server-cloudimg-amd64.qcow2" +} + +resource "proxmox_virtual_environment_file" "cloud_init" { + content_type = "snippets" + datastore_id = var.snippet_datastore_id + node_name = var.node_name + + source_raw { + data = local.cloud_init + file_name = "${var.vm_name}-cloud-init.yaml" + } +} + +resource "proxmox_virtual_environment_vm" "kata_lab" { + name = var.vm_name + description = "Disposable single-node k3s and Kata Containers validation environment." + tags = ["terraform", "disposable", "kata"] + + node_name = var.node_name + vm_id = var.vm_id + + started = true + on_boot = false + stop_on_destroy = true + + agent { + enabled = true + timeout = "15m" + } + + cpu { + cores = var.vm_cores + type = "host" + } + + memory { + dedicated = var.vm_memory_mb + } + + operating_system { + type = "l26" + } + + scsi_hardware = "virtio-scsi-single" + + disk { + datastore_id = var.disk_datastore_id + import_from = data.proxmox_file.ubuntu_noble.id + interface = "scsi0" + iothread = true + discard = "on" + size = var.vm_disk_gb + } + + initialization { + datastore_id = var.disk_datastore_id + user_data_file_id = proxmox_virtual_environment_file.cloud_init.id + + ip_config { + ipv4 { + address = "dhcp" + } + } + } + + network_device { + bridge = var.network_bridge + model = "virtio" + } + + serial_device {} +} diff --git a/infrastructure/kata-lab/terraform/outputs.tf b/infrastructure/kata-lab/terraform/outputs.tf new file mode 100644 index 0000000..9d4dd31 --- /dev/null +++ b/infrastructure/kata-lab/terraform/outputs.tf @@ -0,0 +1,12 @@ +output "vm_id" { + value = proxmox_virtual_environment_vm.kata_lab.vm_id +} + +output "vm_ipv4_addresses" { + description = "QEMU guest agent 报告的地址;忽略 loopback 和 CNI 地址后再用于 SSH。" + value = proxmox_virtual_environment_vm.kata_lab.ipv4_addresses +} + +output "ssh_user" { + value = "ansible" +} diff --git a/infrastructure/kata-lab/terraform/provider.tf b/infrastructure/kata-lab/terraform/provider.tf new file mode 100644 index 0000000..cb13ad4 --- /dev/null +++ b/infrastructure/kata-lab/terraform/provider.tf @@ -0,0 +1,17 @@ +provider "proxmox" { + endpoint = var.proxmox_endpoint + api_token = var.proxmox_api_token + + # Snippet uploads use SSH. The provider deliberately does not read + # ~/.ssh/config, so map the PVE node explicitly and use the current agent. + ssh { + agent = false + username = "root" + private_key = file(pathexpand(var.proxmox_ssh_private_key_file)) + + node { + name = "pve2" + address = "192.168.10.7" + } + } +} diff --git a/infrastructure/kata-lab/terraform/variables.tf b/infrastructure/kata-lab/terraform/variables.tf new file mode 100644 index 0000000..8e1461e --- /dev/null +++ b/infrastructure/kata-lab/terraform/variables.tf @@ -0,0 +1,83 @@ +variable "proxmox_endpoint" { + description = "Proxmox VE API endpoint,不包含 /api2/json。" + type = string + default = "https://pve1.ad.ddupan.top:8006/" +} + +variable "proxmox_api_token" { + description = "Proxmox API token,格式为 user@realm!token-id=secret。" + type = string + sensitive = true +} + +variable "node_name" { + description = "承载 disposable kata-lab VM 的 PVE 节点。" + type = string + default = "pve2" +} + +variable "vm_id" { + description = "kata-lab VMID。" + type = number + default = 147 +} + +variable "vm_name" { + description = "kata-lab VM 名称和 guest hostname。" + type = string + default = "kata-lab" +} + +variable "vm_memory_mb" { + description = "VM 固定内存;pve2 只有约 7.4 GiB 可见内存。" + type = number + default = 4096 +} + +variable "vm_cores" { + description = "VM vCPU 数量。" + type = number + default = 4 +} + +variable "vm_disk_gb" { + description = "VM root disk 大小。" + type = number + default = 41 +} + +variable "disk_datastore_id" { + description = "VM root disk 所在 datastore。" + type = string + default = "pve-rg-hdd" +} + +variable "snippet_datastore_id" { + description = "启用 snippets 的共享 datastore。" + type = string + default = "laptop" +} + +variable "network_bridge" { + description = "连接 kata-lab VM 的 PVE bridge。" + type = string + default = "vmbr0" +} + +variable "ssh_public_key_file" { + description = "注入 cloud-init 用户的 SSH 公钥路径。" + type = string + default = "~/.ssh/id_ed25519.pub" +} + +variable "proxmox_ssh_private_key_file" { + description = "provider 上传 snippet 时登录 PVE 节点使用的私钥路径。" + type = string + default = "~/.ssh/id_ed25519" +} + +variable "k3s_version" { + description = "可选的固定 k3s 版本;空值使用 stable channel。" + type = string + default = "" +} diff --git a/infrastructure/kata-lab/terraform/versions.tf b/infrastructure/kata-lab/terraform/versions.tf new file mode 100644 index 0000000..dd891e7 --- /dev/null +++ b/infrastructure/kata-lab/terraform/versions.tf @@ -0,0 +1,10 @@ +terraform { + required_version = ">= 1.10.0" + + required_providers { + proxmox = { + source = "bpg/proxmox" + version = "0.111.1" + } + } +} diff --git a/infrastructure/kata-lxc-lab/README.md b/infrastructure/kata-lxc-lab/README.md new file mode 100644 index 0000000..83b451d --- /dev/null +++ b/infrastructure/kata-lxc-lab/README.md @@ -0,0 +1,180 @@ +# kata-lxc-lab + +在 `pve2` 上验证 privileged LXC 内运行 k3s、Kata/QEMU,并直接使用 PVE host 的 +`/dev/kvm`、`/dev/vhost-net` 与 `/dev/vhost-vsock`。这是可销毁 PoC,不承载持久数据。 + +Terraform 复用原 VM PoC 的 ignored credential 文件,不复制 token: + +```bash +cd infrastructure/kata-lxc-lab/terraform +export SSL_CERT_FILE=/tmp/kata-lab-ddupan-ca.pem +terraform init +terraform plan \ + -var-file=../../kata-lab/terraform/credentials.auto.tfvars +terraform apply \ + -var-file=../../kata-lab/terraform/credentials.auto.tfvars +``` + +当前 PoC 使用 `pve2`、VMID 147、Ubuntu 24.04 LXC template 和 16 GiB +`local-lvm` rootfs。容器为 privileged,并开启 nesting、keyctl、FUSE、mknod; +Kata 所需的 KVM、vhost 与 `/dev/net/tun` 设备显式透传;另透传 `/dev/kmsg`,因为 +kubelet 启动时会打开该设备。 +PVE 9 的 `force_rw_sys=1` feature 用于开放 kubelet 所需的少量 `/proc/sys` 写操作; +provider 0.111.1 尚未暴露该 feature,因此同样由 `pct` 设置。 + +k3s/Kata worker 还使用以下 privileged LXC 原生配置;PVE 9 使用 cgroup v2,因此设备 +规则采用 `lxc.cgroup2.devices.allow`: + +```text +lxc.apparmor.profile: unconfined +lxc.cgroup2.devices.allow: a +lxc.cap.drop: +lxc.mount.auto: proc:rw sys:rw +``` + +这些设置让容器内 kubelet、containerd 和 Kata shim 能管理 mount、cgroup、设备与 +共享宿主 sysctl。因此外层 LXC 不是安全边界;不可信 CI 的安全边界是内层 Kata VM。 +`/dev/kmsg` 当前直接透传,没有使用 `/dev/console` symlink fallback。 + +PVE 将 privileged LXC 的创建限制为 `root@pam`(API token 即使拥有 `PVEAdmin` 也 +缺少 root-only 的 `Sys.Modify`)。因此实际创建入口使用 root SSH 上的 Ansible/pct: + +```bash +cd infrastructure/kata-lxc-lab/ansible +ansible-playbook create.yml +ansible-playbook bootstrap.yml +ansible-playbook kata.yml +ansible-playbook verify.yml +ansible-playbook verify-fuse.yml +``` + +Terraform 文件保留用于记录 provider 权限边界与声明式目标;当前 token 执行 apply 会 +在创建前返回 403,不会产生资源或 state。 + +## 验证结果 + +当前管理地址为 `192.168.10.128`:2026-09-14 主 LAN DHCP 池调整后,从 `.11` +续租迁移到 `.128`,并同步 Ansible inventory。它仍是动态租约,不是固定地址。 + +2026-09-14,pve2 VMID 147 / `192.168.10.11` 验证通过: + +- LXC 内 k3s `v1.36.4+k3s1` node Ready; +- Kata Containers 4.1.0 `qemu-runtime-rs` 安装成功; +- Kata guest kernel `6.18.35`,外层共享的 PVE kernel `7.0.2-6-pve`; +- Kata Pod 内的 Docker 27.5.1 daemon 成功运行 `busybox:1.37` 嵌套容器; +- `/dev/net/tun` 是 Kata 创建 TAP/link 的必要设备,未透传时 runtime-rs 在 + `create link` 返回 `ENOENT`; +- `fuse-overlayfs` smoke test 成功:Docker 报告 + `storage-driver=fuse-overlayfs`,并成功运行嵌套容器。 + +正式方案保持 Kata 默认 `emptydir_mode = "shared-fs"`,不向 LXC 暴露 host +`/dev/loop-control` 或 `/dev/loopN`。dockerd 镜像需要包含 `fuse-overlayfs`;启动前在 +Kata guest 内创建临时设备节点 `mknod /dev/fuse c 10 229`,随后强制传入 +`--storage-driver=fuse-overlayfs`。该 `/dev/fuse` 属于内层 guest kernel,不是 PVE +host 设备透传,并随 Kata Pod 一起销毁。 + +删除三个 smoke Pod 后,LXC cgroup 统计约 505 MB anonymous memory、1.34 GB file +cache;后者主要来自刚拉取的 Kata/Docker images,可由宿主回收。也就是说常驻 worker +的不可回收 working set 约 500 MB,8 GiB 配置是 cgroup 上限而非预分配。 + +## Gitea ephemeral runner PoC + +`runner/` 定义一次性 Gitea runner Pod。整个 Pod 使用 `runtimeClassName: kata`; +runner 通过 guest 内同 Pod 的 dockerd 创建 job container,不连接 LXC/PVE host 的 +Docker socket。dockerd 使用 zot 中预先发布的 `dind-fuse:29.7.1`,其 +`/var/lib/docker` 是随 Kata Pod 删除的 `emptyDir`。 + +runner 以 `GITEA_RUNNER_EPHEMERAL=1` 注册,只接收一个 `runs-on: kata-poc` job;接单 +后 Gitea 撤销其 runner credential,job 完成后进程与 Kata VM 一起退出。当前 PoC +手动创建一个 runner 等待 `.gitea/workflows/kata-runner-smoke.yml`,尚未部署 +`workflow_job` webhook controller,因此不会保留常驻空闲 runner。 + +注册 token 只在运行时从现有 OpenBao/ESO 消费副本投递到 PoC 集群的临时 Secret, +不写入此目录。两个 Pod image 使用 `imagePullPolicy: Never`,部署前从可信工作站将 +已验证镜像导入 PoC k3s containerd;这是当前 zot 拉取仍要求 SPIFFE 身份时的 +bootstrap 边界,不是正式镜像分发方案。 + +正式接入 zot 前需要把这个 cluster 注册到 SPIRE,并为 runner Pod 创建专用 +`ClusterSPIFFEID`。届时 runner 与 dockerd sidecar 在同一路径只读挂载 CSI Workload +API socket,runner 的 `container.options` 再把该路径 bind-mount 到 job container。 +zot 只对最终 SPIFFE ID 的 `panxiao81/dind-fuse` 仓库授予 +`read/create/update`;禁止授予 namespace 或整个 trust domain 写权限。 + +2026-09-14 已完成真实 Gitea job 验收:Actions run `242` / job `445` 使用 +`alpine:3.22`,10 秒成功;runner container 退出码为 `0`,原生 dockerd sidecar 随后 +由 kubelet 终止,Pod 最终为 `Succeeded`,没有保留空闲 Kata VM。guest 内核为 +`6.18.35`,dockerd `29.7.1` 使用 `fuse-overlayfs`。 + +首次尝试暴露了两个部署陷阱:`COPY` 默认保留源文件 mode,入口脚本原为 `0664`, +必须在 Dockerfile 使用 `COPY --chmod=0755`;只配置 `ephemeral-storage: 20Gi` limit +会形成同值 request,使 12 GiB PoC 节点无法调度。当前 manifest 显式 request 1 GiB、 +limit 8 GiB。完整 `ubuntu-latest` job image 也不适合这个小 rootfs,基础生命周期验收 +改用 Alpine;正式 worker 必须扩容本地 image/cache 空间或使用更精简的 job image。 + +## Cloud Hypervisor 内存记账验证 + +2026-09-17 在 pve2 的一次性 VMID 148 中安装 k3s v1.36.4+k3s1 与 Kata 4.1.0, +只启用 `clh-runtime-rs`,验证 Cloud Hypervisor 在 privileged LXC 内不会重现早期 +microVM PoC 的 private memfd 双重记账问题。测试完成后已删除 VMID 148。 + +Cloud Hypervisor `/api/v1/vm.info` 报告: + +```json +{"size":3254779904,"shared":true,"hugepages":false,"prefault":false,"thp":true} +``` + +Kata guest 在 memory-backed `emptyDir` 中写入 1 GiB 后: + +- VMM `Pss_Shmem` 从约 247 MiB 增至约 1269 MiB; +- VMM `Pss_Anon` 写入前后均约 1.2 MiB; +- 外层 LXC `shmem` 增加约 1 GiB,`anon` 基本不变; +- LXC `memory.events` 的 `max`、`oom` 与 `oom_kill` 均为 0; +- 删除 Pod 后 VMM 退出,LXC `shmem` 回落到约 1.6 MiB。 + +因此 Kata 的 Cloud Hypervisor handler 已使用 shared memfd,等价于独立 launcher 的 +`--memory shared=on` 修复;后续 OpenSandbox/Kata 方案无需为 guest RAM 预留近似双倍 +的 LXC cgroup 内存。部署验收仍应检查 `vm.info.config.memory.shared=true`,防止上游 +配置或 runtime handler 变化造成回归。 + +测试期间还发现 pve2 `local-lvm` thin pool 已达到 100%;临时 rootfs 必须放到 +`pve-rg-hdd`,不能根据容器内部 `df` 的剩余空间判断 thin pool 是否可写。 + +## Block-backed emptyDir 与 CI 构建验证 + +2026-09-17 又在 pve2 的一次性 VMID 148 中验证 Kata 4.1.0 +`clh-runtime-rs` 的以下 drop-in: + +```toml +[runtime] +emptydir_mode = "block-plain" +``` + +结果如下: + +- 普通 Kubernetes `emptyDir` 被创建为稀疏 `disk.img`,由 Cloud Hypervisor 直接作为 + block device 热插拔;guest 内 `/var/lib/docker` 是 `/dev/vdb` 上的 ext4。 +- Docker 29.1.5 成功强制使用原生 `overlay2`,不再需要 `fuse-overlayfs`;整个过程 + 没有使用 PVE host 或 LXC 的 loop device。 +- `emptyDir.sizeLimit: 8Gi` 没有决定虚拟磁盘容量。由于 kubelet 所在文件系统约 + 12 GiB,guest 看到的 ext4 约 11.7 GiB;这与 Kata 已知的 block-backed emptyDir + 限制一致。 +- 包含 256 MiB payload、2000 个小文件和一个额外复制层的冷 BuildKit 构建耗时 + 46 秒;同一 LXC、同一 DRBD HDD 存储上的 runc 对照为 34 秒,Kata 约慢 35%。 +- 在 overlay2 容器层内写入并 fsync 512 MiB:Kata 为 12 秒(40.5 MB/s),runc 为 + 10 秒(50.3 MB/s);创建一万个小文件两者均约 2 秒。 +- kind v0.27.0 / Kubernetes v1.32.2 首次启动失败是因为 Kata guest 内的 dockerd + 容器没有 `/dev/kmsg`,不是 block storage 或 cgroup 故障。在 privileged dockerd + 启动前执行 `mknod /dev/kmsg c 1 11` 后,kind 创建成功:总计 122 秒,进入等待阶段 + 后 23 秒 control-plane Ready,随后可正常删除。 +- 测试时稀疏 backing file 逻辑大小约 12 GiB,构建和 kind 后实际占用约 1.8 GiB。 + 删除 Pod 后 backing file、Cloud Hypervisor 进程和 kind 容器全部消失;host/LXC + 均无 loop device 残留,LXC 没有 OOM 事件。 + +因此 `block-plain` 能解决 virtio-fs 不能作为 overlayfs upperdir 的问题,并满足 +BuildKit 与 kind 的功能要求。正式 CI RuntimeClass 必须使用独立的 block-backed +配置,并在 dockerd bootstrap 中创建 guest `/dev/kmsg`。它仍比同机 runc 构建慢约 +20–35%,且 `emptyDir.sizeLimit` 不能可靠限制虚拟磁盘容量。PoC 曾据此建议为 kubelet +目录提供独立文件系统;正式 sandbox 集群最终没有采用该设计:独立 volume 增加了 +DRBD 与 LXC 挂载生命周期复杂度,却没有隔离 containerd 等其他节点数据。正式节点 +改用单一、容量明确的 rootfs,并以磁盘占用监控、Pod 删除后的 backing-file 回收检查 +和实际构建基准作为上线门槛。 diff --git a/infrastructure/kata-lxc-lab/ansible/ansible.cfg b/infrastructure/kata-lxc-lab/ansible/ansible.cfg new file mode 100644 index 0000000..7763a8c --- /dev/null +++ b/infrastructure/kata-lxc-lab/ansible/ansible.cfg @@ -0,0 +1,10 @@ +[defaults] +inventory = inventory/hosts.yml +host_key_checking = True +interpreter_python = auto_silent +retry_files_enabled = False +local_tmp = /tmp/ansible-kata-lxc-local +remote_tmp = /tmp/ansible-kata-lxc-remote + +[ssh_connection] +pipelining = True diff --git a/infrastructure/kata-lxc-lab/ansible/bootstrap.yml b/infrastructure/kata-lxc-lab/ansible/bootstrap.yml new file mode 100644 index 0000000..09fbbb2 --- /dev/null +++ b/infrastructure/kata-lxc-lab/ansible/bootstrap.yml @@ -0,0 +1,58 @@ +--- +- name: Bootstrap k3s in the Kata LXC worker + hosts: pve2 + gather_facts: false + vars: + kata_lxc_id: 147 + tasks: + - name: Check base packages in LXC + ansible.builtin.command: + cmd: >- + pct exec {{ kata_lxc_id }} -- dpkg-query -W + ca-certificates curl jq openssh-server + register: kata_lxc_packages + changed_when: false + failed_when: false + + - name: Install base packages and SSH server in LXC + ansible.builtin.command: + cmd: >- + pct exec {{ kata_lxc_id }} -- bash -lc + 'apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y + ca-certificates curl jq openssh-server' + when: kata_lxc_packages.rc != 0 + + - name: Install k3s in LXC + ansible.builtin.shell: + cmd: | + set -euo pipefail + pct exec {{ kata_lxc_id }} -- bash -lc ' + if ! command -v k3s >/dev/null; then + curl -sfL https://get.k3s.io -o /tmp/install-k3s.sh + chmod 0755 /tmp/install-k3s.sh + INSTALL_K3S_EXEC="server --disable=traefik --write-kubeconfig-mode=0644" /tmp/install-k3s.sh + fi + ' + executable: /bin/bash + register: kata_lxc_k3s_install + changed_when: "'No change detected' not in kata_lxc_k3s_install.stdout" + + - name: Wait for k3s node readiness + ansible.builtin.command: + cmd: >- + pct exec {{ kata_lxc_id }} -- /usr/local/bin/k3s kubectl wait + --for=condition=Ready node/kata-lxc-lab --timeout=180s + changed_when: false + + - name: Check LXC virtualization and devices + ansible.builtin.command: + cmd: >- + pct exec {{ kata_lxc_id }} -- bash -lc + 'systemd-detect-virt; ls -l /dev/kvm /dev/vhost-net /dev/vhost-vsock; + /usr/local/bin/k3s kubectl get node -o wide' + register: kata_lxc_ready + changed_when: false + + - name: Report k3s readiness + ansible.builtin.debug: + var: kata_lxc_ready.stdout_lines diff --git a/infrastructure/kata-lxc-lab/ansible/create.yml b/infrastructure/kata-lxc-lab/ansible/create.yml new file mode 100644 index 0000000..7cd6ad8 --- /dev/null +++ b/infrastructure/kata-lxc-lab/ansible/create.yml @@ -0,0 +1,137 @@ +--- +- name: Create the disposable Kata LXC worker + hosts: pve2 + gather_facts: false + vars: + kata_lxc_id: 147 + kata_lxc_template: laptop:vztmpl/ubuntu-24.04-standard_24.04-2_amd64.tar.zst + tasks: + - name: Check whether the LXC already exists + ansible.builtin.stat: + path: /etc/pve/lxc/{{ kata_lxc_id }}.conf + register: kata_lxc_config + + - name: Create privileged LXC with Kata host devices + ansible.builtin.command: + argv: + - pct + - create + - "{{ kata_lxc_id }}" + - "{{ kata_lxc_template }}" + - --hostname + - kata-lxc-lab + - --ostype + - ubuntu + - --rootfs + - local-lvm:12 + - --cores + - "4" + - --memory + - "8192" + - --swap + - "0" + - --net0 + - name=eth0,bridge=vmbr0,ip=dhcp + - --features + - nesting=1,keyctl=1,fuse=1,mknod=1,force_rw_sys=1 + - --unprivileged + - "0" + - --onboot + - "0" + - --ssh-public-keys + - /root/.ssh/authorized_keys + - --dev0 + - path=/dev/kvm,mode=0660 + - --dev1 + - path=/dev/vhost-net,mode=0660 + - --dev2 + - path=/dev/vhost-vsock,mode=0660 + - --dev3 + - path=/dev/kmsg,mode=0660 + - --dev4 + - path=/dev/net/tun,mode=0666 + - --tags + - disposable;kata;lxc;ansible + when: not kata_lxc_config.stat.exists + + - name: Read LXC status + ansible.builtin.command: + cmd: pct status {{ kata_lxc_id }} + register: kata_lxc_status + changed_when: false + + - name: Read current LXC configuration + ansible.builtin.command: + cmd: pct config {{ kata_lxc_id }} + register: kata_lxc_current_config + changed_when: false + + - name: Ensure kubelet kernel log device is present + ansible.builtin.command: + cmd: pct set {{ kata_lxc_id }} --dev3 path=/dev/kmsg,mode=0660 + register: kata_lxc_kmsg + when: "'/dev/kmsg' not in kata_lxc_current_config.stdout" + + - name: Enable writable sysctls required by kubelet + ansible.builtin.command: + cmd: >- + pct set {{ kata_lxc_id }} --features + nesting=1,keyctl=1,fuse=1,mknod=1,force_rw_sys=1 + register: kata_lxc_rw_sys + when: "'force_rw_sys=1' not in kata_lxc_current_config.stdout" + + - name: Ensure TUN device required by Kata networking is present + ansible.builtin.command: + cmd: pct set {{ kata_lxc_id }} --dev4 path=/dev/net/tun,mode=0666 + register: kata_lxc_tun + when: "'/dev/net/tun' not in kata_lxc_current_config.stdout" + + - name: Configure privileged nested-runtime LXC directives + ansible.builtin.lineinfile: + path: /etc/pve/lxc/{{ kata_lxc_id }}.conf + regexp: "^{{ item.key | regex_escape }}:" + line: "{{ item.key }}: {{ item.value }}" + loop: + - key: lxc.apparmor.profile + value: unconfined + - key: lxc.cgroup2.devices.allow + value: a + - key: lxc.cap.drop + value: "" + - key: lxc.mount.auto + value: proc:rw sys:rw + register: kata_lxc_raw_config + + - name: Restart LXC after device configuration change + ansible.builtin.command: + cmd: pct reboot {{ kata_lxc_id }} + when: >- + kata_lxc_kmsg.changed or kata_lxc_rw_sys.changed or kata_lxc_tun.changed or + kata_lxc_raw_config.changed + + - name: Start LXC + ansible.builtin.command: + cmd: pct start {{ kata_lxc_id }} + when: "'status: stopped' in kata_lxc_status.stdout" + + - name: Wait for systemd in LXC + ansible.builtin.command: + cmd: pct exec {{ kata_lxc_id }} -- systemctl is-system-running --wait + register: kata_lxc_systemd + retries: 30 + delay: 2 + until: kata_lxc_systemd.rc in [0, 1] + changed_when: false + failed_when: kata_lxc_systemd.rc not in [0, 1] + + - name: Show LXC address and Kata devices + ansible.builtin.command: + cmd: >- + pct exec {{ kata_lxc_id }} -- sh -c + 'hostname -I; ls -l /dev/kvm /dev/vhost-net /dev/vhost-vsock /dev/net/tun' + register: kata_lxc_facts + changed_when: false + + - name: Report LXC address and devices + ansible.builtin.debug: + var: kata_lxc_facts.stdout_lines diff --git a/infrastructure/kata-lxc-lab/ansible/inventory/hosts.yml b/infrastructure/kata-lxc-lab/ansible/inventory/hosts.yml new file mode 100644 index 0000000..be2b2aa --- /dev/null +++ b/infrastructure/kata-lxc-lab/ansible/inventory/hosts.yml @@ -0,0 +1,9 @@ +--- +all: + hosts: + pve2: + ansible_host: 192.168.10.7 + ansible_user: root + kata-lab: + ansible_host: 192.168.10.128 + ansible_user: root diff --git a/infrastructure/kata-lxc-lab/ansible/kata.yml b/infrastructure/kata-lxc-lab/ansible/kata.yml new file mode 100644 index 0000000..80f1769 --- /dev/null +++ b/infrastructure/kata-lxc-lab/ansible/kata.yml @@ -0,0 +1,3 @@ +--- +# Reuse the proven Kata 4.1.0 installation playbook against the LXC inventory. +- import_playbook: ../../kata-lab/ansible/kata.yml diff --git a/infrastructure/kata-lxc-lab/ansible/verify-fuse.yml b/infrastructure/kata-lxc-lab/ansible/verify-fuse.yml new file mode 100644 index 0000000..63e9602 --- /dev/null +++ b/infrastructure/kata-lxc-lab/ansible/verify-fuse.yml @@ -0,0 +1,73 @@ +--- +- name: Verify fuse-overlayfs for Docker inside Kata + hosts: kata-lab + gather_facts: false + tasks: + - name: Remove an earlier fuse-overlayfs smoke Pod + ansible.builtin.command: + cmd: k3s kubectl delete pod kata-fuse-smoke --ignore-not-found --wait=true + changed_when: false + + - name: Create Kata Docker Pod using fuse-overlayfs + ansible.builtin.shell: + cmd: | + set -o pipefail + k3s kubectl apply -f - <<'EOF' + apiVersion: v1 + kind: Pod + metadata: + name: kata-fuse-smoke + spec: + runtimeClassName: kata + restartPolicy: Never + containers: + - name: dockerd + image: docker.io/library/docker:27-dind + securityContext: + privileged: true + env: + - name: DOCKER_TLS_CERTDIR + value: "" + command: + - sh + - -c + - | + apk add --no-cache fuse-overlayfs + test -e /dev/fuse || mknod /dev/fuse c 10 229 + chmod 0666 /dev/fuse + exec dockerd-entrypoint.sh --storage-driver=fuse-overlayfs + - name: client + image: docker.io/library/docker:27-cli + env: + - name: DOCKER_HOST + value: tcp://127.0.0.1:2375 + command: + - sh + - -c + - | + until docker info >/dev/null 2>&1; do sleep 1; done + docker info --format 'storage-driver={{ "{{" }}.Driver{{ "}}" }}' + docker run --rm docker.io/library/busybox:1.37 echo fuse-nested-docker-ok + EOF + executable: /bin/bash + changed_when: true + + - name: Wait for the Docker client to finish + ansible.builtin.command: + cmd: >- + k3s kubectl wait --for=jsonpath='{.status.containerStatuses[?(@.name=="client")].state.terminated.exitCode}'=0 + pod/kata-fuse-smoke --timeout=10m + changed_when: false + + - name: Read Docker client result + ansible.builtin.command: + cmd: k3s kubectl logs kata-fuse-smoke -c client + register: kata_fuse_smoke_log + changed_when: false + + - name: Require fuse-overlayfs and nested Docker + ansible.builtin.assert: + that: + - "'storage-driver=fuse-overlayfs' in kata_fuse_smoke_log.stdout" + - "'fuse-nested-docker-ok' in kata_fuse_smoke_log.stdout" + success_msg: Docker used fuse-overlayfs and completed a nested container inside Kata diff --git a/infrastructure/kata-lxc-lab/ansible/verify.yml b/infrastructure/kata-lxc-lab/ansible/verify.yml new file mode 100644 index 0000000..72680f0 --- /dev/null +++ b/infrastructure/kata-lxc-lab/ansible/verify.yml @@ -0,0 +1,3 @@ +--- +# Reuse the guest-kernel and Docker sidecar smoke tests against the LXC worker. +- import_playbook: ../../kata-lab/ansible/verify.yml diff --git a/infrastructure/kata-lxc-lab/runner/namespace.yaml b/infrastructure/kata-lxc-lab/runner/namespace.yaml new file mode 100644 index 0000000..7baf6c8 --- /dev/null +++ b/infrastructure/kata-lxc-lab/runner/namespace.yaml @@ -0,0 +1,35 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: gitea-actions + labels: + pod-security.kubernetes.io/enforce: privileged +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: kata-ephemeral-runner-config + namespace: gitea-actions +data: + config.yaml: | + log: + level: info + runner: + file: /runner-state/.runner + capacity: 1 + envs: + DOCKER_HOST: tcp://host.docker.internal:2375 + timeout: 3h + shutdown_timeout: 5m + labels: + - kata-poc:docker://docker.io/library/alpine:3.22 + cache: + enabled: false + container: + network: "" + options: --add-host=host.docker.internal:host-gateway + docker_host: tcp://127.0.0.1:2375 + require_docker: true + docker_timeout: 5m + force_pull: true + bind_workdir: false diff --git a/infrastructure/kata-lxc-lab/runner/runner.yaml b/infrastructure/kata-lxc-lab/runner/runner.yaml new file mode 100644 index 0000000..0bd924b --- /dev/null +++ b/infrastructure/kata-lxc-lab/runner/runner.yaml @@ -0,0 +1,111 @@ +apiVersion: v1 +kind: Pod +metadata: + generateName: kata-ephemeral-runner- + namespace: gitea-actions + labels: + app.kubernetes.io/name: kata-ephemeral-runner +spec: + runtimeClassName: kata + restartPolicy: Never + terminationGracePeriodSeconds: 330 + automountServiceAccountToken: false + initContainers: + # Native sidecar: startupProbe gates the runner, and kubelet terminates this + # container after the ephemeral runner exits so the Kata sandbox can end. + - name: docker + image: zot.ad.ddupan.top/panxiao81/dind-fuse:29.7.1 + imagePullPolicy: Never + restartPolicy: Always + env: + - name: DOCKER_TLS_CERTDIR + value: "" + - name: DIND_LOOPBACK_SIZE + value: 6G + - name: DIND_LOOPBACK_FILE + value: /loopback/docker.img + args: + - --host=tcp://0.0.0.0:2375 + - --tls=false + - --mtu=1450 + - --default-network-opt=bridge=com.docker.network.driver.mtu=1450 + securityContext: + privileged: true + resources: + requests: + cpu: 250m + memory: 512Mi + ephemeral-storage: 1Gi + limits: + cpu: "4" + memory: 6Gi + ephemeral-storage: 8Gi + volumeMounts: + - name: docker-data + mountPath: /var/lib/docker + - name: loopback-data + mountPath: /loopback + startupProbe: + tcpSocket: + port: 2375 + periodSeconds: 2 + failureThreshold: 150 + readinessProbe: + tcpSocket: + port: 2375 + periodSeconds: 5 + containers: + - name: runner + image: docker.io/gitea/runner:2.3.0 + imagePullPolicy: Never + env: + - name: CONFIG_FILE + value: /config/config.yaml + - name: DOCKER_HOST + value: tcp://127.0.0.1:2375 + - name: GITEA_INSTANCE_URL + value: https://git.ddupan.top + - name: GITEA_RUNNER_EPHEMERAL + value: "1" + - name: GITEA_RUNNER_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: GITEA_RUNNER_LABELS + value: kata-poc:docker://docker.io/library/alpine:3.22 + - name: GITEA_RUNNER_REGISTRATION_TOKEN_FILE + value: /registration/token + resources: + requests: + cpu: 100m + memory: 128Mi + limits: + cpu: "1" + memory: 1Gi + volumeMounts: + - name: config + mountPath: /config + readOnly: true + - name: registration + mountPath: /registration + readOnly: true + - name: runner-state + mountPath: /runner-state + volumes: + - name: config + configMap: + name: kata-ephemeral-runner-config + - name: registration + secret: + secretName: gitea-runner-registration + - name: runner-state + emptyDir: + medium: Memory + sizeLimit: 16Mi + - name: docker-data + emptyDir: + sizeLimit: 8Gi + - name: loopback-data + emptyDir: + medium: Memory + sizeLimit: 6Gi diff --git a/infrastructure/kata-lxc-lab/terraform/.terraform.lock.hcl b/infrastructure/kata-lxc-lab/terraform/.terraform.lock.hcl new file mode 100644 index 0000000..4794735 --- /dev/null +++ b/infrastructure/kata-lxc-lab/terraform/.terraform.lock.hcl @@ -0,0 +1,24 @@ +# This file is maintained automatically by "terraform init". +# Manual edits may be lost in future updates. + +provider "registry.terraform.io/bpg/proxmox" { + version = "0.111.1" + constraints = "0.111.1" + hashes = [ + "h1:ML2D3UUZTM99yrll/EBXj7wBYMb8xmQgomqFNybEoxY=", + "zh:18fb7c31a08dde6bffa1a4d4a211e604d6d17eec7092fd59331b3db3c6f3742c", + "zh:1cd60761538289d4dd2a1086b3ae62a7b0bdd4b1a2f824e9a44e243413168dba", + "zh:2eb76f6fc8299b6820ff678c8252332cc3366e226b5ae2e61748fd2449c1ed92", + "zh:45e6f7ebd0bf48911d37060359a4f359b5743b3092e985295733990e406d0416", + "zh:4aa8ba912eae37975d2e983394d173e595ca34fc76b5bf220b37d0e99d76e98c", + "zh:58e0789923103a77d502a0a9fc3eb920625e8eb935ec2d4ac0d006aebd1d186c", + "zh:6df8aa85fb8865915537e946c19b02538ad188018a629759c213c6f03730f642", + "zh:6ed47bc00d0913a1d0880618fa1376115e9edab6b4a658c081061a7f0e4ca360", + "zh:c5b10ff4f33df7e4c29e8f1127d49845b561b37b57517e844fb0954d7923d65e", + "zh:d016510e14b738499f0db9d9b3aafe82fc6877fb4ab4e9f831fb68a8d70a1385", + "zh:d941f394069bbf24351b363da1c64383f487067aaee0a84f9b96476d4912e212", + "zh:ddf271dbc2632ae8ffa8de3972f243ee47d260cb2ac90aa784f2746d98e21a0f", + "zh:ed0caa3501c42f611b7e9622c9b1df69fd85dc25a3cd88d3076381829688cd62", + "zh:f26e0763dbe6a6b2195c94b44696f2110f7f55433dc142839be16b9697fa5597", + ] +} diff --git a/infrastructure/kata-lxc-lab/terraform/main.tf b/infrastructure/kata-lxc-lab/terraform/main.tf new file mode 100644 index 0000000..85177d3 --- /dev/null +++ b/infrastructure/kata-lxc-lab/terraform/main.tf @@ -0,0 +1,95 @@ +locals { + ssh_public_key = trimspace(file(pathexpand(var.ssh_public_key_file))) +} + +data "proxmox_file" "ubuntu_noble" { + content_type = "vztmpl" + datastore_id = var.template_datastore + node_name = var.proxmox_node + file_name = var.template_file_name +} + +resource "proxmox_virtual_environment_container" "kata_lxc_lab" { + node_name = var.proxmox_node + vm_id = var.container_id + + description = "Disposable privileged LXC for validating k3s and Kata with direct host KVM access." + unprivileged = false + started = true + start_on_boot = false + tags = ["disposable", "kata", "lxc", "terraform"] + + cpu { + cores = 4 + } + + memory { + dedicated = 8192 + swap = 0 + } + + disk { + datastore_id = var.root_datastore + size = 16 + } + + features { + fuse = true + keyctl = true + mknod = true + nesting = true + } + + device_passthrough { + path = "/dev/kvm" + mode = "0660" + } + + device_passthrough { + path = "/dev/vhost-net" + mode = "0660" + } + + device_passthrough { + path = "/dev/vhost-vsock" + mode = "0660" + } + + device_passthrough { + path = "/dev/kmsg" + mode = "0660" + } + + device_passthrough { + path = "/dev/net/tun" + mode = "0666" + } + + initialization { + hostname = var.container_name + + ip_config { + ipv4 { + address = "dhcp" + } + } + + user_account { + keys = [local.ssh_public_key] + } + } + + network_interface { + name = "eth0" + bridge = "vmbr0" + } + + operating_system { + template_file_id = data.proxmox_file.ubuntu_noble.id + type = "ubuntu" + } + + wait_for_ip { + ipv4 = true + } +} diff --git a/infrastructure/kata-lxc-lab/terraform/outputs.tf b/infrastructure/kata-lxc-lab/terraform/outputs.tf new file mode 100644 index 0000000..f7f464a --- /dev/null +++ b/infrastructure/kata-lxc-lab/terraform/outputs.tf @@ -0,0 +1,7 @@ +output "container_id" { + value = proxmox_virtual_environment_container.kata_lxc_lab.vm_id +} + +output "ipv4" { + value = proxmox_virtual_environment_container.kata_lxc_lab.ipv4 +} diff --git a/infrastructure/kata-lxc-lab/terraform/provider.tf b/infrastructure/kata-lxc-lab/terraform/provider.tf new file mode 100644 index 0000000..0150857 --- /dev/null +++ b/infrastructure/kata-lxc-lab/terraform/provider.tf @@ -0,0 +1,15 @@ +provider "proxmox" { + endpoint = var.proxmox_endpoint + api_token = var.proxmox_api_token + + ssh { + agent = false + username = "root" + private_key = file(pathexpand(var.proxmox_ssh_private_key_file)) + + node { + name = var.proxmox_node + address = var.proxmox_node_address + } + } +} diff --git a/infrastructure/kata-lxc-lab/terraform/variables.tf b/infrastructure/kata-lxc-lab/terraform/variables.tf new file mode 100644 index 0000000..717af1d --- /dev/null +++ b/infrastructure/kata-lxc-lab/terraform/variables.tf @@ -0,0 +1,54 @@ +variable "proxmox_endpoint" { + type = string + default = "https://pve1.ad.ddupan.top:8006" +} + +variable "proxmox_api_token" { + type = string + sensitive = true +} + +variable "proxmox_node" { + type = string + default = "pve2" +} + +variable "proxmox_node_address" { + type = string + default = "192.168.10.7" +} + +variable "proxmox_ssh_private_key_file" { + type = string + default = "~/.ssh/id_ed25519" +} + +variable "ssh_public_key_file" { + type = string + default = "~/.ssh/id_ed25519.pub" +} + +variable "container_id" { + type = number + default = 147 +} + +variable "container_name" { + type = string + default = "kata-lxc-lab" +} + +variable "template_datastore" { + type = string + default = "laptop" +} + +variable "template_file_name" { + type = string + default = "ubuntu-24.04-standard_24.04-2_amd64.tar.zst" +} + +variable "root_datastore" { + type = string + default = "local-lvm" +} diff --git a/infrastructure/kata-lxc-lab/terraform/versions.tf b/infrastructure/kata-lxc-lab/terraform/versions.tf new file mode 100644 index 0000000..7a6a8b9 --- /dev/null +++ b/infrastructure/kata-lxc-lab/terraform/versions.tf @@ -0,0 +1,10 @@ +terraform { + required_version = ">= 1.10" + + required_providers { + proxmox = { + source = "bpg/proxmox" + version = "0.111.1" + } + } +} diff --git a/infrastructure/microvm-runner/README.md b/infrastructure/microvm-runner/README.md new file mode 100644 index 0000000..deb56cd --- /dev/null +++ b/infrastructure/microvm-runner/README.md @@ -0,0 +1,53 @@ +# Gitea kind microVM runner + +`kind-microvm` 是专门运行 kind / nested Kubernetes CI 的 runner label。每个 job +独占一台 Cloud Hypervisor VM;guest 内直接运行 Docker 与 ephemeral Gitea Runner, +不经过 Kata。VM 完成一个 job 后关机,临时 COW 磁盘随即删除。 + +集群内的 `controller.py` 消费 Gitea `workflow_job` webhook。仅当 +`action=queued` 且 `workflow_job.labels` 包含 `kind-microvm` 时,向 NATS JetStream +的 `ci.runner.kind-microvm` subject 发布消息。`workflow_job.id` 写入 +`Nats-Msg-Id`,重复 webhook 不会生成第二个任务。 + +pve2 LXC 内的 `worker.py` 使用 durable pull consumer 领取消息。领取后启动一台 +microVM,运行期间每分钟发送 `InProgress`;VM/ephemeral runner 正常退出才 ACK, +启动失败则 NAK 并延迟重试。当前 `RUNNER_CAPACITY=1` 只是 pve2 的资源配置;以后可 +提高单 worker capacity,或在其他宿主机增加共享同一 durable consumer 的 worker, +而无需修改 webhook/controller。相同 runner 类型必须共享 durable;WorkQueue stream +不允许多个 consumer 使用重叠的 subject filter。 + +worker 的凭据位于: + +```text +/etc/microvm-runner/registration-token +/etc/microvm-runner/nats-password +``` + +二者必须为 root-only 文件,不能写入镜像、cloud-init seed 或仓库。worker 为每次 +启动生成一次性 nonce;guest 只能从 TAP 网段请求一次 registration token,请求后 +立即从 worker 内存删除。runner 注册成功后删除 guest 内的 token 文件,再启动 +daemon。这样 job 无法从 seed disk 或 Docker inspect 取回可复用 registration token。 + +NATS stream 使用 `WorkQueuePolicy`,ACK 后立即删除消息;同时限制为 24 小时、 +10000 条和 256 MiB,异常积压也不会无限增长。NATS 平台配置见 +`../../platform/nats/`。 + +2026-09-16 PoC 已验证:Cloud Hypervisor v52.0、Ubuntu 24.04、4 vCPU、3 GiB RAM、 +10 GiB COW disk 中,Docker 29.1.3 成功创建 kind v0.27.0 / Kubernetes v1.32.2 +集群,并运行 `busybox:1.37` smoke Pod。冷启动约 20 秒。pve2 资源只允许一台: +VMM cgroup(guest RAM 与 page cache)峰值约 6.2 GiB。 + +正式启用前还需要安装 launcher、制作不含凭据的基础镜像、部署 controller,并在 +Gitea 配置只发送 `workflow_job` 的 instance/organization webhook。workflow 使用: + +```yaml +runs-on: kind-microvm +``` + +当前 pve2 配置: + +```ini +RUNNER_CAPACITY=1 +NATS_DURABLE=kind-microvm +RUNNER_MAX_INFLIGHT=64 +``` diff --git a/infrastructure/microvm-runner/controller.py b/infrastructure/microvm-runner/controller.py new file mode 100644 index 0000000..59d6079 --- /dev/null +++ b/infrastructure/microvm-runner/controller.py @@ -0,0 +1,113 @@ +#!/usr/bin/env python3 +"""Gitea workflow_job webhook to NATS JetStream producer.""" + +import hashlib +import hmac +import json +import os +import ssl +from pathlib import Path + +import nats +from aiohttp import web +from nats.js.api import DiscardPolicy, RetentionPolicy, StorageType, StreamConfig +from nats.js.errors import NotFoundError + +LABEL = os.environ.get("RUNNER_LABEL", "kind-microvm") +SUBJECT = os.environ.get("NATS_SUBJECT", f"ci.runner.{LABEL}") +STREAM = os.environ.get("NATS_STREAM", "CI_RUNNER") +NATS_URL = os.environ.get("NATS_URL", "tls://nats.nats.svc.cluster.local:4222") +NATS_USER = os.environ.get("NATS_USER", "ci-producer") +NATS_PASSWORD_FILE = Path(os.environ.get("NATS_PASSWORD_FILE", "/run/secrets/nats/password")) +NATS_CA_FILE = os.environ.get("NATS_CA_FILE", "/etc/ssl/certs/ca-certificates.crt") +WEBHOOK_SECRET_FILE = Path(os.environ.get("WEBHOOK_SECRET_FILE", "/run/secrets/gitea/webhook-secret")) + + +def accepts(payload: object) -> tuple[bool, str | None]: + if not isinstance(payload, dict) or payload.get("action") != "queued": + return False, None + job = payload.get("workflow_job") + if not isinstance(job, dict) or LABEL not in job.get("labels", []): + return False, None + job_id = job.get("id") + if not isinstance(job_id, int): + return False, None + return True, str(job_id) + + +def valid_signature(body: bytes, signature: str) -> bool: + expected = hmac.new(WEBHOOK_SECRET_FILE.read_bytes().strip(), body, hashlib.sha256).hexdigest() + return hmac.compare_digest(signature, expected) + + +async def ensure_stream(js: object) -> None: + config = StreamConfig( + name=STREAM, + subjects=["ci.runner.*"], + retention=RetentionPolicy.WORK_QUEUE, + storage=StorageType.FILE, + discard=DiscardPolicy.OLD, + max_age=24 * 60 * 60, + max_msgs=10_000, + max_bytes=256 * 1024 * 1024, + duplicate_window=24 * 60 * 60, + ) + try: + await js.stream_info(STREAM) + except NotFoundError: + await js.add_stream(config=config) + else: + await js.update_stream(config=config) + + +async def webhook(request: web.Request) -> web.Response: + body = await request.read() + if not valid_signature(body, request.headers.get("X-Gitea-Signature", "")): + raise web.HTTPUnauthorized() + try: + payload = json.loads(body) + except json.JSONDecodeError as error: + raise web.HTTPBadRequest(text="invalid JSON\n") from error + accepted, job_id = accepts(payload) + if not accepted: + return web.Response(status=204) + await request.app["js"].publish( + SUBJECT, + body, + headers={"Nats-Msg-Id": f"gitea-workflow-job-{job_id}"}, + ) + return web.Response(status=202, text="queued\n") + + +async def health(request: web.Request) -> web.Response: + return web.Response(text="ok\n" if request.app["nc"].is_connected else "disconnected\n", + status=200 if request.app["nc"].is_connected else 503) + + +async def nats_context(app: web.Application): + tls = ssl.create_default_context(cafile=NATS_CA_FILE) + nc = await nats.connect( + NATS_URL, + user=NATS_USER, + password=NATS_PASSWORD_FILE.read_text().strip(), + tls=tls, + name="microvm-runner-controller", + ) + app["nc"] = nc + app["js"] = nc.jetstream() + await ensure_stream(app["js"]) + yield + await nc.drain() + + +def create_app() -> web.Application: + app = web.Application(client_max_size=1024 * 1024) + app.cleanup_ctx.append(nats_context) + app.router.add_post("/webhook", webhook) + app.router.add_get("/healthz", health) + return app + + +if __name__ == "__main__": + web.run_app(create_app(), host=os.environ.get("LISTEN", "0.0.0.0"), + port=int(os.environ.get("PORT", "8787"))) diff --git a/infrastructure/microvm-runner/microvm-runner-worker.service b/infrastructure/microvm-runner/microvm-runner-worker.service new file mode 100644 index 0000000..81ce8c2 --- /dev/null +++ b/infrastructure/microvm-runner/microvm-runner-worker.service @@ -0,0 +1,19 @@ +[Unit] +Description=Gitea kind microVM runner worker +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +EnvironmentFile=-/etc/microvm-runner/worker.env +ExecStart=/opt/microvm-runner/venv/bin/python /opt/microvm-runner/worker.py +Restart=on-failure +RestartSec=5s +SupplementaryGroups=kvm +PrivateTmp=yes +ProtectHome=yes +ProtectSystem=strict +ReadWritePaths=/var/lib/microvm-runner /run/microvm-runner + +[Install] +WantedBy=multi-user.target diff --git a/infrastructure/microvm-runner/requirements.txt b/infrastructure/microvm-runner/requirements.txt new file mode 100644 index 0000000..747670a --- /dev/null +++ b/infrastructure/microvm-runner/requirements.txt @@ -0,0 +1,2 @@ +aiohttp==3.12.15 +nats-py==2.11.0 diff --git a/infrastructure/microvm-runner/worker.py b/infrastructure/microvm-runner/worker.py new file mode 100644 index 0000000..8a10e48 --- /dev/null +++ b/infrastructure/microvm-runner/worker.py @@ -0,0 +1,138 @@ +#!/usr/bin/env python3 +"""Capacity-bounded JetStream consumer that launches one ephemeral VM per job.""" + +import asyncio +import os +import secrets +import ssl +import uuid +from pathlib import Path + +import nats +from aiohttp import web +from nats.errors import TimeoutError +from nats.js.api import AckPolicy, ConsumerConfig + +CAPACITY = int(os.environ.get("RUNNER_CAPACITY", "1")) +SUBJECT = os.environ.get("NATS_SUBJECT", "ci.runner.kind-microvm") +STREAM = os.environ.get("NATS_STREAM", "CI_RUNNER") +DURABLE = os.environ.get("NATS_DURABLE", "kind-microvm") +MAX_INFLIGHT = int(os.environ.get("RUNNER_MAX_INFLIGHT", "64")) +NATS_URL = os.environ.get("NATS_URL", "tls://nats.ad.ddupan.top:4222") +NATS_USER = os.environ.get("NATS_USER", "ci-worker") +NATS_PASSWORD_FILE = Path(os.environ.get("NATS_PASSWORD_FILE", "/etc/microvm-runner/nats-password")) +NATS_CA_FILE = os.environ.get("NATS_CA_FILE", "/etc/ssl/certs/ca-certificates.crt") +REGISTRATION_TOKEN_FILE = Path(os.environ.get("REGISTRATION_TOKEN_FILE", "/etc/microvm-runner/registration-token")) +LAUNCHER = os.environ.get("LAUNCHER", "/usr/local/libexec/microvm-runner-launch") +TOKEN_LISTEN = os.environ.get("TOKEN_LISTEN", "172.30.0.1") +TOKEN_PORT = int(os.environ.get("TOKEN_PORT", "8787")) + +tokens: dict[str, bytes] = {} +token_lock = asyncio.Lock() + + +async def token(request: web.Request) -> web.Response: + nonce = request.match_info["nonce"] + async with token_lock: + value = tokens.pop(nonce, None) + if value is None: + raise web.HTTPNotFound() + return web.Response(body=value, headers={"Cache-Control": "no-store"}) + + +async def heartbeat(message: object, stop: asyncio.Event) -> None: + while True: + try: + await asyncio.wait_for(stop.wait(), timeout=60) + return + except asyncio.TimeoutError: + await message.in_progress() + + +async def run_one(message: object) -> None: + instance_id = str(uuid.uuid4()) + nonce = secrets.token_urlsafe(32) + async with token_lock: + tokens[nonce] = REGISTRATION_TOKEN_FILE.read_bytes().strip() + stop = asyncio.Event() + pulse = asyncio.create_task(heartbeat(message, stop)) + try: + process = await asyncio.create_subprocess_exec(LAUNCHER, instance_id, nonce) + return_code = await process.wait() + if return_code == 0: + await message.ack() + else: + await message.nak(delay=30) + except Exception: + await message.nak(delay=30) + raise + finally: + stop.set() + await pulse + async with token_lock: + tokens.pop(nonce, None) + + +async def consume() -> None: + if CAPACITY < 1: + raise ValueError("RUNNER_CAPACITY must be at least 1") + tls = ssl.create_default_context(cafile=NATS_CA_FILE) + nc = await nats.connect( + NATS_URL, + user=NATS_USER, + password=NATS_PASSWORD_FILE.read_text().strip(), + tls=tls, + name=DURABLE, + ) + js = nc.jetstream() + subscription = await js.pull_subscribe( + SUBJECT, + durable=DURABLE, + stream=STREAM, + config=ConsumerConfig( + durable_name=DURABLE, + filter_subject=SUBJECT, + ack_policy=AckPolicy.EXPLICIT, + ack_wait=5 * 60, + # This durable consumer is shared by every host of this runner type. + # Local CAPACITY controls each host; this is only a global safety cap. + max_ack_pending=MAX_INFLIGHT, + max_deliver=5, + ), + ) + active: set[asyncio.Task[None]] = set() + try: + while True: + active = {task for task in active if not task.done()} + free = CAPACITY - len(active) + if free == 0: + await asyncio.wait(active, return_when=asyncio.FIRST_COMPLETED) + continue + try: + messages = await subscription.fetch(batch=free, timeout=5) + except TimeoutError: + continue + for message in messages: + task = asyncio.create_task(run_one(message)) + task.add_done_callback(lambda done: done.exception()) + active.add(task) + finally: + if active: + await asyncio.gather(*active, return_exceptions=True) + await nc.drain() + + +async def main() -> None: + app = web.Application() + app.router.add_get("/token/{nonce}", token) + runner = web.AppRunner(app) + await runner.setup() + await web.TCPSite(runner, TOKEN_LISTEN, TOKEN_PORT).start() + try: + await consume() + finally: + await runner.cleanup() + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/platform/gitea-runner/images/dind-fuse/Dockerfile b/platform/gitea-runner/images/dind-fuse/Dockerfile new file mode 100644 index 0000000..5a00a38 --- /dev/null +++ b/platform/gitea-runner/images/dind-fuse/Dockerfile @@ -0,0 +1,9 @@ +ARG DOCKER_VERSION=29.7.1 +FROM docker:${DOCKER_VERSION}-dind + +RUN apk add --no-cache fuse-overlayfs + +COPY --chmod=0755 entrypoint.sh /usr/local/bin/dind-fuse-entrypoint + +ENTRYPOINT ["/usr/local/bin/dind-fuse-entrypoint"] +CMD [] diff --git a/platform/gitea-runner/images/dind-fuse/entrypoint.sh b/platform/gitea-runner/images/dind-fuse/entrypoint.sh new file mode 100644 index 0000000..beacb49 --- /dev/null +++ b/platform/gitea-runner/images/dind-fuse/entrypoint.sh @@ -0,0 +1,42 @@ +#!/bin/sh +set -eu + +# /dev belongs to the Kata guest. A privileged dockerd container can create +# this standard FUSE node without exposing the PVE host's /dev/fuse to the LXC. +if [ ! -e /dev/fuse ]; then + mknod /dev/fuse c 10 229 +fi +chmod 0666 /dev/fuse + +# kind's nested kubelet opens /dev/kmsg. This node belongs to the Kata guest; +# exposing it to a kind node does not expose the LXC or PVE host kernel log. +if [ ! -e /dev/kmsg ]; then + mknod /dev/kmsg c 1 11 +fi +chmod 0600 /dev/kmsg + +storage_driver=fuse-overlayfs + +# kind's kubelet/cAdvisor cannot map a virtiofs-backed fuse-overlayfs root to a +# block device. When requested, create an ext4 filesystem on a guest-local loop +# device. The backing file, loop device and filesystem all die with the Kata VM. +if [ -n "${DIND_LOOPBACK_SIZE:-}" ]; then + [ -e /dev/loop-control ] || mknod /dev/loop-control c 10 237 + i=0 + while [ "$i" -lt 8 ]; do + [ -e "/dev/loop$i" ] || mknod "/dev/loop$i" b 7 "$i" + i=$((i + 1)) + done + + backing_file="${DIND_LOOPBACK_FILE:-/var/lib/docker-loopback.img}" + truncate -s "$DIND_LOOPBACK_SIZE" "$backing_file" + # Alpine's BusyBox losetup supports -f, but not util-linux's + # --find/--show long options. + loop_device="$(losetup -f)" + losetup "$loop_device" "$backing_file" + mkfs.ext4 -q -F -m 0 "$loop_device" + mount "$loop_device" /var/lib/docker + storage_driver=overlay2 +fi + +exec dockerd-entrypoint.sh --storage-driver="$storage_driver" "$@"