归档：Kata / microVM runner 实验（2026-09-17 工作区快照）

从旧工作区 chore/recover-old-workspace 清理时保存，内容与 2026-09-17 stash@{0} 快照中的版本一致；未合并、未在 main 上使用，仅作参考，不开 PR。被 gitignore 的 tfstate 与凭据文件不在此分支，仍留在本地工作区。 Co-Authored-By: Claude Opus 5.5 <[email protected]>
2026-10-01 17:30:48 +00:00
parent 69d3476a6b
commit 55bb5be8b6
39 changed files with 1891 additions and 0 deletions
@@ -0,0 +1,10 @@
+[defaults]
+inventory = inventory/hosts.yml
+host_key_checking = True
+interpreter_python = auto_silent
+retry_files_enabled = False
+local_tmp = /tmp/ansible-kata-lxc-local
+remote_tmp = /tmp/ansible-kata-lxc-remote
+
+[ssh_connection]
+pipelining = True
@@ -0,0 +1,58 @@
+---
+- name: Bootstrap k3s in the Kata LXC worker
+  hosts: pve2
+  gather_facts: false
+  vars:
+    kata_lxc_id: 147
+  tasks:
+    - name: Check base packages in LXC
+      ansible.builtin.command:
+        cmd: >-
+          pct exec {{ kata_lxc_id }} -- dpkg-query -W
+          ca-certificates curl jq openssh-server
+      register: kata_lxc_packages
+      changed_when: false
+      failed_when: false
+
+    - name: Install base packages and SSH server in LXC
+      ansible.builtin.command:
+        cmd: >-
+          pct exec {{ kata_lxc_id }} -- bash -lc
+          'apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y
+          ca-certificates curl jq openssh-server'
+      when: kata_lxc_packages.rc != 0
+
+    - name: Install k3s in LXC
+      ansible.builtin.shell:
+        cmd: |
+          set -euo pipefail
+          pct exec {{ kata_lxc_id }} -- bash -lc '
+            if ! command -v k3s >/dev/null; then
+              curl -sfL https://get.k3s.io -o /tmp/install-k3s.sh
+              chmod 0755 /tmp/install-k3s.sh
+              INSTALL_K3S_EXEC="server --disable=traefik --write-kubeconfig-mode=0644" /tmp/install-k3s.sh
+            fi
+          '
+        executable: /bin/bash
+      register: kata_lxc_k3s_install
+      changed_when: "'No change detected' not in kata_lxc_k3s_install.stdout"
+
+    - name: Wait for k3s node readiness
+      ansible.builtin.command:
+        cmd: >-
+          pct exec {{ kata_lxc_id }} -- /usr/local/bin/k3s kubectl wait
+          --for=condition=Ready node/kata-lxc-lab --timeout=180s
+      changed_when: false
+
+    - name: Check LXC virtualization and devices
+      ansible.builtin.command:
+        cmd: >-
+          pct exec {{ kata_lxc_id }} -- bash -lc
+          'systemd-detect-virt; ls -l /dev/kvm /dev/vhost-net /dev/vhost-vsock;
+          /usr/local/bin/k3s kubectl get node -o wide'
+      register: kata_lxc_ready
+      changed_when: false
+
+    - name: Report k3s readiness
+      ansible.builtin.debug:
+        var: kata_lxc_ready.stdout_lines
@@ -0,0 +1,137 @@
+---
+- name: Create the disposable Kata LXC worker
+  hosts: pve2
+  gather_facts: false
+  vars:
+    kata_lxc_id: 147
+    kata_lxc_template: laptop:vztmpl/ubuntu-24.04-standard_24.04-2_amd64.tar.zst
+  tasks:
+    - name: Check whether the LXC already exists
+      ansible.builtin.stat:
+        path: /etc/pve/lxc/{{ kata_lxc_id }}.conf
+      register: kata_lxc_config
+
+    - name: Create privileged LXC with Kata host devices
+      ansible.builtin.command:
+        argv:
+          - pct
+          - create
+          - "{{ kata_lxc_id }}"
+          - "{{ kata_lxc_template }}"
+          - --hostname
+          - kata-lxc-lab
+          - --ostype
+          - ubuntu
+          - --rootfs
+          - local-lvm:12
+          - --cores
+          - "4"
+          - --memory
+          - "8192"
+          - --swap
+          - "0"
+          - --net0
+          - name=eth0,bridge=vmbr0,ip=dhcp
+          - --features
+          - nesting=1,keyctl=1,fuse=1,mknod=1,force_rw_sys=1
+          - --unprivileged
+          - "0"
+          - --onboot
+          - "0"
+          - --ssh-public-keys
+          - /root/.ssh/authorized_keys
+          - --dev0
+          - path=/dev/kvm,mode=0660
+          - --dev1
+          - path=/dev/vhost-net,mode=0660
+          - --dev2
+          - path=/dev/vhost-vsock,mode=0660
+          - --dev3
+          - path=/dev/kmsg,mode=0660
+          - --dev4
+          - path=/dev/net/tun,mode=0666
+          - --tags
+          - disposable;kata;lxc;ansible
+      when: not kata_lxc_config.stat.exists
+
+    - name: Read LXC status
+      ansible.builtin.command:
+        cmd: pct status {{ kata_lxc_id }}
+      register: kata_lxc_status
+      changed_when: false
+
+    - name: Read current LXC configuration
+      ansible.builtin.command:
+        cmd: pct config {{ kata_lxc_id }}
+      register: kata_lxc_current_config
+      changed_when: false
+
+    - name: Ensure kubelet kernel log device is present
+      ansible.builtin.command:
+        cmd: pct set {{ kata_lxc_id }} --dev3 path=/dev/kmsg,mode=0660
+      register: kata_lxc_kmsg
+      when: "'/dev/kmsg' not in kata_lxc_current_config.stdout"
+
+    - name: Enable writable sysctls required by kubelet
+      ansible.builtin.command:
+        cmd: >-
+          pct set {{ kata_lxc_id }} --features
+          nesting=1,keyctl=1,fuse=1,mknod=1,force_rw_sys=1
+      register: kata_lxc_rw_sys
+      when: "'force_rw_sys=1' not in kata_lxc_current_config.stdout"
+
+    - name: Ensure TUN device required by Kata networking is present
+      ansible.builtin.command:
+        cmd: pct set {{ kata_lxc_id }} --dev4 path=/dev/net/tun,mode=0666
+      register: kata_lxc_tun
+      when: "'/dev/net/tun' not in kata_lxc_current_config.stdout"
+
+    - name: Configure privileged nested-runtime LXC directives
+      ansible.builtin.lineinfile:
+        path: /etc/pve/lxc/{{ kata_lxc_id }}.conf
+        regexp: "^{{ item.key | regex_escape }}:"
+        line: "{{ item.key }}: {{ item.value }}"
+      loop:
+        - key: lxc.apparmor.profile
+          value: unconfined
+        - key: lxc.cgroup2.devices.allow
+          value: a
+        - key: lxc.cap.drop
+          value: ""
+        - key: lxc.mount.auto
+          value: proc:rw sys:rw
+      register: kata_lxc_raw_config
+
+    - name: Restart LXC after device configuration change
+      ansible.builtin.command:
+        cmd: pct reboot {{ kata_lxc_id }}
+      when: >-
+        kata_lxc_kmsg.changed or kata_lxc_rw_sys.changed or kata_lxc_tun.changed or
+        kata_lxc_raw_config.changed
+
+    - name: Start LXC
+      ansible.builtin.command:
+        cmd: pct start {{ kata_lxc_id }}
+      when: "'status: stopped' in kata_lxc_status.stdout"
+
+    - name: Wait for systemd in LXC
+      ansible.builtin.command:
+        cmd: pct exec {{ kata_lxc_id }} -- systemctl is-system-running --wait
+      register: kata_lxc_systemd
+      retries: 30
+      delay: 2
+      until: kata_lxc_systemd.rc in [0, 1]
+      changed_when: false
+      failed_when: kata_lxc_systemd.rc not in [0, 1]
+
+    - name: Show LXC address and Kata devices
+      ansible.builtin.command:
+        cmd: >-
+          pct exec {{ kata_lxc_id }} -- sh -c
+          'hostname -I; ls -l /dev/kvm /dev/vhost-net /dev/vhost-vsock /dev/net/tun'
+      register: kata_lxc_facts
+      changed_when: false
+
+    - name: Report LXC address and devices
+      ansible.builtin.debug:
+        var: kata_lxc_facts.stdout_lines
@@ -0,0 +1,9 @@
+---
+all:
+  hosts:
+    pve2:
+      ansible_host: 192.168.10.7
+      ansible_user: root
+    kata-lab:
+      ansible_host: 192.168.10.128
+      ansible_user: root
@@ -0,0 +1,3 @@
+---
+# Reuse the proven Kata 4.1.0 installation playbook against the LXC inventory.
+- import_playbook: ../../kata-lab/ansible/kata.yml
@@ -0,0 +1,73 @@
+---
+- name: Verify fuse-overlayfs for Docker inside Kata
+  hosts: kata-lab
+  gather_facts: false
+  tasks:
+    - name: Remove an earlier fuse-overlayfs smoke Pod
+      ansible.builtin.command:
+        cmd: k3s kubectl delete pod kata-fuse-smoke --ignore-not-found --wait=true
+      changed_when: false
+
+    - name: Create Kata Docker Pod using fuse-overlayfs
+      ansible.builtin.shell:
+        cmd: |
+          set -o pipefail
+          k3s kubectl apply -f - <<'EOF'
+          apiVersion: v1
+          kind: Pod
+          metadata:
+            name: kata-fuse-smoke
+          spec:
+            runtimeClassName: kata
+            restartPolicy: Never
+            containers:
+              - name: dockerd
+                image: docker.io/library/docker:27-dind
+                securityContext:
+                  privileged: true
+                env:
+                  - name: DOCKER_TLS_CERTDIR
+                    value: ""
+                command:
+                  - sh
+                  - -c
+                  - |
+                    apk add --no-cache fuse-overlayfs
+                    test -e /dev/fuse || mknod /dev/fuse c 10 229
+                    chmod 0666 /dev/fuse
+                    exec dockerd-entrypoint.sh --storage-driver=fuse-overlayfs
+              - name: client
+                image: docker.io/library/docker:27-cli
+                env:
+                  - name: DOCKER_HOST
+                    value: tcp://127.0.0.1:2375
+                command:
+                  - sh
+                  - -c
+                  - |
+                    until docker info >/dev/null 2>&1; do sleep 1; done
+                    docker info --format 'storage-driver={{ "{{" }}.Driver{{ "}}" }}'
+                    docker run --rm docker.io/library/busybox:1.37 echo fuse-nested-docker-ok
+          EOF
+        executable: /bin/bash
+      changed_when: true
+
+    - name: Wait for the Docker client to finish
+      ansible.builtin.command:
+        cmd: >-
+          k3s kubectl wait --for=jsonpath='{.status.containerStatuses[?(@.name=="client")].state.terminated.exitCode}'=0
+          pod/kata-fuse-smoke --timeout=10m
+      changed_when: false
+
+    - name: Read Docker client result
+      ansible.builtin.command:
+        cmd: k3s kubectl logs kata-fuse-smoke -c client
+      register: kata_fuse_smoke_log
+      changed_when: false
+
+    - name: Require fuse-overlayfs and nested Docker
+      ansible.builtin.assert:
+        that:
+          - "'storage-driver=fuse-overlayfs' in kata_fuse_smoke_log.stdout"
+          - "'fuse-nested-docker-ok' in kata_fuse_smoke_log.stdout"
+        success_msg: Docker used fuse-overlayfs and completed a nested container inside Kata
@@ -0,0 +1,3 @@
+---
+# Reuse the guest-kernel and Docker sidecar smoke tests against the LXC worker.
+- import_playbook: ../../kata-lab/ansible/verify.yml