Establish clean homelab infrastructure baseline

Reorganize the brownfield repository, remove retired and generated artifacts, harden ignore rules, and record the GitOps/IaC redesign.
2026-09-09 16:47:20 +00:00
commit 88a02ababa
418 changed files with 50579 additions and 0 deletions
@@ -0,0 +1,168 @@
+---
+# LINSTOR cluster: controller on one node, satellites everywhere, storage pools,
+# resource group, and the PVE storage entry.
+#
+# All linstor CLI calls run ONCE against the controller — LINSTOR is itself a
+# cluster-wide database, so repeating them per-host would just race.
+
+- name: Enable and start the satellite on every node
+  ansible.builtin.systemd_service:
+    name: linstor-satellite
+    enabled: true
+    state: started
+
+- name: Enable and start the controller
+  ansible.builtin.systemd_service:
+    name: linstor-controller
+    enabled: true
+    state: started
+  when: inventory_hostname == pve_linstor_controller
+
+- name: Point the client at the controller
+  # Without this, `linstor` talks to localhost and fails on the satellites.
+  ansible.builtin.copy:
+    dest: /etc/linstor/linstor-client.conf
+    mode: "0644"
+    content: |
+      # Managed by Ansible (role pve_linstor).
+      [global]
+      controllers=linstor://{{ hostvars[pve_linstor_controller].ansible_host }}
+
+- name: Wait for the controller API
+  ansible.builtin.command:
+    cmd: linstor node list
+  register: _lin_ready
+  until: _lin_ready.rc == 0
+  retries: 24
+  delay: 5
+  changed_when: false
+  run_once: true
+  delegate_to: "{{ pve_linstor_controller }}"
+
+# ── nodes ─────────────────────────────────────────────────────────────────
+- name: Register each node with LINSTOR
+  ansible.builtin.command:
+    cmd: >-
+      linstor node create {{ item }} {{ hostvars[item].ansible_host }}
+      --node-type {{ 'Combined' if item == pve_linstor_controller else 'Satellite' }}
+  loop: "{{ groups['pve'] }}"
+  register: _node_create
+  changed_when: "'successfully' in (_node_create.stdout | default('') | lower)"
+  failed_when:
+    - _node_create.rc != 0
+    - "'already exists' not in (_node_create.stdout | default('') + _node_create.stderr | default('')) | lower"
+  run_once: true
+  delegate_to: "{{ pve_linstor_controller }}"
+
+- name: Wait for all nodes to come ONLINE
+  ansible.builtin.shell:
+    cmd: |
+      set -o pipefail
+      linstor -m --output-version v1 node list \
+        | python3 -c "import json,sys; print(all(n['connection_status']=='ONLINE' for n in json.load(sys.stdin)[0]))"
+    executable: /bin/bash
+  register: _nodes_online
+  until: "'True' in _nodes_online.stdout"
+  retries: 24
+  delay: 5
+  changed_when: false
+  run_once: true
+  delegate_to: "{{ pve_linstor_controller }}"
+
+# ── storage pools ─────────────────────────────────────────────────────────
+- name: Create the storage pools
+  ansible.builtin.command:
+    cmd: >-
+      linstor storage-pool create lvm {{ item.0 }} {{ item.1.pool }} {{ item.1.vg }}
+  loop: "{{ groups['pve'] | product([
+              {'pool': pve_linstor_ssd_pool, 'vg': pve_linstor_ssd_vg},
+              {'pool': pve_linstor_hdd_pool, 'vg': pve_linstor_hdd_vg}]) | list }}"
+  register: _sp
+  changed_when: "'successfully' in (_sp.stdout | default('') | lower)"
+  failed_when:
+    - _sp.rc != 0
+    - "'already exists' not in (_sp.stdout | default('') + _sp.stderr | default('')) | lower"
+  run_once: true
+  delegate_to: "{{ pve_linstor_controller }}"
+
+# ── resource groups ───────────────────────────────────────────────────────
+# One per storage pool — a resource group binds to exactly one pool, so the hdd
+# pool needs its own or that capacity is unusable from Proxmox.
+- name: Create the resource groups
+  ansible.builtin.command:
+    cmd: >-
+      linstor resource-group create {{ item.name }}
+      --storage-pool {{ item.pool }}
+      --place-count {{ item.place_count | default(pve_linstor_place_count) }}
+  loop: "{{ pve_linstor_resource_groups }}"
+  loop_control:
+    label: "{{ item.name }} -> {{ item.pool }}"
+  register: _rg
+  changed_when: "'successfully' in (_rg.stdout | default('') | lower)"
+  failed_when:
+    - _rg.rc != 0
+    - "'already exists' not in (_rg.stdout | default('') + _rg.stderr | default('')) | lower"
+  run_once: true
+  delegate_to: "{{ pve_linstor_controller }}"
+
+- name: Read existing volume-group definitions
+  # `linstor volume-group create` is NOT idempotent: every call APPENDS another
+  # volume number. Running the play three times left pve-rg with VlmNrs 0,1,2, so
+  # PVE then failed every disk create with "has 3 Volume groups, but only 1 sizes
+  # were provided". Check before creating.
+  ansible.builtin.shell:
+    # Count table rows. NOTE the leading char class: linstor draws its tables
+    # with BOX-DRAWING pipes (U+2502), not ASCII '|', so a '^\\|' pattern never
+    # matches and the count is always 0 -- which silently appends a NEW volume
+    # group on every run.
+    cmd: "linstor volume-group list {{ item.name }} 2>/dev/null | grep -cE '^.[[:space:]]*[0-9]+[[:space:]]' || true"
+  loop: "{{ pve_linstor_resource_groups }}"
+  loop_control:
+    label: "{{ item.name }}"
+  register: _vg_count
+  changed_when: false
+  check_mode: false
+  run_once: true
+  delegate_to: "{{ pve_linstor_controller }}"
+
+- name: Create a volume group where none exists
+  ansible.builtin.command:
+    cmd: "linstor volume-group create {{ item.item.name }}"
+  loop: "{{ _vg_count.results }}"
+  loop_control:
+    label: "{{ item.item.name }} (has {{ item.stdout | default('?') | trim }})"
+  when: (item.stdout | default('0') | trim | int) == 0
+  run_once: true
+  delegate_to: "{{ pve_linstor_controller }}"
+
+# ── PVE storage entries ───────────────────────────────────────────────────
+- name: Register each resource group as PVE storage
+  ansible.builtin.command:
+    cmd: >-
+      pvesm add drbd {{ item.name }}
+      --content {{ item.content | default('images,rootdir') }}
+      --controller {{ hostvars[pve_linstor_controller].ansible_host }}
+      --resourcegroup {{ item.name }}
+  loop: "{{ pve_linstor_resource_groups }}"
+  loop_control:
+    label: "{{ item.name }}"
+  register: _pvesm
+  changed_when: _pvesm.rc == 0
+  failed_when:
+    - _pvesm.rc != 0
+    - "'already defined' not in (_pvesm.stderr | default('') + _pvesm.stdout | default(''))"
+  run_once: true
+  delegate_to: "{{ pve_linstor_controller }}"
+
+- name: Report LINSTOR state
+  ansible.builtin.shell:
+    cmd: "linstor storage-pool list; echo; linstor resource-group list; echo; pvesm status"
+  register: _lin_state
+  changed_when: false
+  run_once: true
+  delegate_to: "{{ pve_linstor_controller }}"
+
+- name: Show it
+  ansible.builtin.debug:
+    msg: "{{ _lin_state.stdout_lines }}"
+  run_once: true
@@ -0,0 +1,15 @@
+---
+# LINSTOR/DRBD hyperconverged storage. Split into stages so the destructive part
+# (disk wiping) is separately gated and cannot run by accident.
+
+- name: Install packages
+  ansible.builtin.import_tasks: packages.yml
+  tags: [linstor_pkgs]
+
+- name: Prepare backing storage (LVM)
+  ansible.builtin.import_tasks: storage.yml
+  tags: [linstor_storage]
+
+- name: Configure the LINSTOR cluster
+  ansible.builtin.import_tasks: linstor.yml
+  tags: [linstor_config]
@@ -0,0 +1,112 @@
+---
+# LINBIT's public repo. Free and needs no subscription — the subscription only
+# buys support and their prebuilt kernel modules; drbd-dkms builds locally.
+
+- name: Install the LINBIT signing key
+  ansible.builtin.get_url:
+    url: "{{ pve_linstor_repo_key_url }}"
+    dest: /tmp/linbit-pubkey.asc
+    mode: "0644"
+  retries: 3
+  delay: 10
+
+- name: Convert the key to a keyring
+  ansible.builtin.shell:
+    cmd: "gpg --dearmor < /tmp/linbit-pubkey.asc > {{ pve_linstor_keyring }}"
+    creates: "{{ pve_linstor_keyring }}"
+
+- name: Add the LINBIT repository
+  ansible.builtin.deb822_repository:
+    name: linbit
+    types: [deb]
+    uris: "{{ pve_linstor_repo_url }}"
+    suites: "{{ pve_linstor_repo_suite }}"
+    components: [drbd-9]
+    signed_by: "{{ pve_linstor_keyring }}"
+    enabled: true
+    state: present
+  register: _linbit_repo
+
+- name: Update apt cache
+  ansible.builtin.apt:
+    update_cache: true
+  register: _apt
+  retries: 3
+  delay: 15
+  until: _apt is succeeded
+
+- name: Install DRBD + LINSTOR packages
+  # drbd-dkms COMPILES a kernel module against the running kernel, so the
+  # headers must match. This is the step most likely to fail on a flaky uplink,
+  # hence the retries.
+  ansible.builtin.apt:
+    name:
+      - "pve-headers-{{ ansible_facts['kernel'] }}"
+      - drbd-dkms
+      - drbd-utils
+      - linstor-controller
+      - linstor-satellite
+      - linstor-client
+      - linstor-proxmox
+    state: present
+  register: _linstor_pkgs
+  retries: 2
+  delay: 30
+  until: _linstor_pkgs is succeeded
+
+- name: Load the DRBD module
+  ansible.builtin.command:
+    cmd: modprobe drbd
+  changed_when: false
+
+- name: Check which DRBD version is actually LOADED
+  ansible.builtin.shell:
+    cmd: "cat /proc/drbd 2>/dev/null | head -1"
+  register: _drbd_ver
+  changed_when: false
+
+# The kernel ships an IN-TREE drbd 8.4. If anything loaded it before drbd-dkms
+# was installed, modprobe is a no-op and the running module stays 8.4 even
+# though dkms built 9.x correctly (modinfo will happily report 9.x from
+# .../updates/dkms/drbd.ko). LINSTOR needs 9. Swap it live when nothing is using
+# it; refuse and demand a reboot when something is.
+- name: Check the module refcount before swapping
+  ansible.builtin.shell:
+    cmd: "lsmod | awk '$1==\"drbd\" {print $3}' | head -1"
+  register: _drbd_refs
+  changed_when: false
+  when: "'version: 9' not in _drbd_ver.stdout"
+
+- name: Refuse to swap a DRBD module that is in use
+  ansible.builtin.fail:
+    msg: >-
+      In-tree DRBD {{ _drbd_ver.stdout }} is loaded and IN USE
+      (refcount {{ _drbd_refs.stdout | default('?') }}). Reboot
+      {{ inventory_hostname }} to pick up the dkms-built DRBD 9.
+  when:
+    - "'version: 9' not in _drbd_ver.stdout"
+    - (_drbd_refs.stdout | default('0') | trim | int) > 0
+
+- name: Swap the in-tree DRBD 8.4 for the dkms-built DRBD 9
+  ansible.builtin.shell:
+    cmd: |
+      set -e
+      modprobe -r drbd_transport_tcp 2>/dev/null || true
+      modprobe -r drbd
+      depmod -a
+      modprobe drbd
+  when:
+    - "'version: 9' not in _drbd_ver.stdout"
+    - (_drbd_refs.stdout | default('0') | trim | int) == 0
+
+- name: Re-read the DRBD version after the swap
+  ansible.builtin.shell:
+    cmd: "cat /proc/drbd 2>/dev/null | head -1"
+  register: _drbd_ver
+  changed_when: false
+
+- name: Assert DRBD 9
+  ansible.builtin.assert:
+    that: "'version: 9' in _drbd_ver.stdout"
+    fail_msg: "Expected DRBD 9, got: {{ _drbd_ver.stdout }}"
+    quiet: true
@@ -0,0 +1,176 @@
+---
+# LVM volume groups that back the LINSTOR pools.
+#
+# DESTRUCTIVE. Everything that destroys data is gated behind
+# pve_linstor_wipe_hdd and guarded by an explicit "is this really free?" check.
+
+# ── SSD pool: the space the installer left beyond the 60G `pve` VG ─────────
+- name: Find the SSD holding the pve VG
+  ansible.builtin.shell:
+    cmd: |
+      set -o pipefail
+      pvs --noheadings -o pv_name --select 'vg_name=pve' | tr -d ' ' | head -1
+    executable: /bin/bash
+  register: _pve_pv
+  changed_when: false
+  check_mode: false          # read-only discovery; must run in --check
+
+- name: Derive the parent disk holding the pve PV
+  # DO NOT use `lsblk -no PKNAME <pv>`: without --nodeps lsblk also lists the LVs
+  # stacked on the partition, whose PKNAME is the PARTITION itself, so `head -1`
+  # can return e.g. "nvme0n1p3" instead of "nvme0n1". That mistake made sgdisk
+  # write a GPT INTO the LVM PV holding root on pve2/pve3 and destroyed their
+  # `pve` VG metadata (2026-07-25). sysfs is unambiguous: the parent of
+  # /sys/class/block/<part> IS the disk.
+  ansible.builtin.shell:
+    cmd: |
+      set -o pipefail
+      PV="{{ _pve_pv.stdout | trim }}"
+      PART="$(basename "$PV")"
+      DISK="$(basename "$(readlink -f "/sys/class/block/${PART}/..")")"
+      echo "/dev/${DISK}"
+    executable: /bin/bash
+  register: _ssd_disk
+  changed_when: false
+  check_mode: false          # read-only discovery; must run in --check
+
+- name: Refuse to proceed unless that really is a whole disk
+  # Last line of defence: sgdisk against a partition is destructive, so verify
+  # the derived device is TYPE=disk and not a partition before touching it.
+  ansible.builtin.shell:
+    cmd: "lsblk -dno TYPE {{ _ssd_disk.stdout | trim }}"
+  register: _ssd_type
+  changed_when: false
+  check_mode: false          # read-only discovery; must run in --check
+
+- name: Assert it is a disk
+  ansible.builtin.assert:
+    that: "(_ssd_type.stdout | trim) == 'disk'"
+    fail_msg: >-
+      Derived SSD device {{ _ssd_disk.stdout | trim }} is TYPE
+      '{{ _ssd_type.stdout | trim }}', not 'disk'. Refusing to partition it —
+      running sgdisk against a partition destroys whatever is on it.
+    quiet: true
+
+- name: Check whether the SSD VG already exists
+  ansible.builtin.command:
+    cmd: "vgs {{ pve_linstor_ssd_vg }}"
+  register: _ssd_vg
+  changed_when: false
+  failed_when: false
+
+- name: Create a partition from the SSD's free tail
+  # sgdisk -N uses ALL remaining free space for the next partition number.
+  # Only touches unallocated space; the existing pve partitions are untouched.
+  ansible.builtin.command:
+    cmd: "sgdisk -N 0 -t 0:8e00 -c 0:linstor-ssd {{ _ssd_disk.stdout | trim }}"
+  when: _ssd_vg.rc != 0
+  register: _ssd_part
+
+- name: Re-read the partition table
+  # partx (util-linux) rather than partprobe: `parted` is NOT installed on a
+  # stock PVE 9 node, so partprobe fails with "No such file or directory".
+  ansible.builtin.shell:
+    cmd: |
+      set -e
+      partx -u "{{ _ssd_disk.stdout | trim }}"
+      udevadm settle
+    executable: /bin/bash
+  when: _ssd_part is changed
+  changed_when: true
+
+- name: Find the new SSD partition
+  ansible.builtin.shell:
+    cmd: |
+      set -o pipefail
+      lsblk -rno NAME,PARTLABEL "{{ _ssd_disk.stdout | trim }}" \
+        | awk '$2=="linstor-ssd" {print "/dev/"$1}' | head -1
+    executable: /bin/bash
+  register: _ssd_partdev
+  changed_when: false
+
+- name: Create the SSD volume group
+  ansible.builtin.command:
+    cmd: "vgcreate {{ pve_linstor_ssd_vg }} {{ _ssd_partdev.stdout | trim }}"
+  when:
+    - _ssd_vg.rc != 0
+    - (_ssd_partdev.stdout | trim) | length > 0
+
+# ── HDD pool: the whole 1TB spindle ───────────────────────────────────────
+- name: Confirm the HDD target is a whole disk
+  # Same guard as the SSD path. wipefs/vgcreate against a partition by mistake is
+  # how the pve VG on pve2/pve3 got destroyed on 2026-07-25; assert the device
+  # type rather than trusting the variable.
+  ansible.builtin.shell:
+    cmd: "lsblk -dno TYPE {{ pve_linstor_hdd_disk }}"
+  register: _hdd_type
+  changed_when: false
+  check_mode: false          # read-only discovery; must run in --check
+
+- name: Assert the HDD target is a disk
+  ansible.builtin.assert:
+    that: "(_hdd_type.stdout | trim) == 'disk'"
+    fail_msg: >-
+      {{ pve_linstor_hdd_disk }} is TYPE '{{ _hdd_type.stdout | trim }}', not
+      'disk'. Refusing to wipe it.
+    quiet: true
+
+- name: Inspect the HDD's current volume group
+  ansible.builtin.shell:
+    cmd: |
+      set -o pipefail
+      pvs --noheadings -o vg_name {{ pve_linstor_hdd_disk }} 2>/dev/null | tr -d ' '
+    executable: /bin/bash
+  register: _hdd_vg
+  changed_when: false
+  check_mode: false          # read-only discovery; must run in --check
+  failed_when: false
+
+- name: Refuse to wipe a HDD carrying anything other than a stale Ceph VG
+  # The only VGs we expect here are the leftover ceph-<uuid> ones from the old
+  # cluster. Anything else means this disk is not what we think it is.
+  ansible.builtin.fail:
+    msg: >-
+      {{ pve_linstor_hdd_disk }} on {{ inventory_hostname }} holds VG
+      '{{ _hdd_vg.stdout | trim }}', which is not a stale ceph-* VG.
+      Refusing to wipe. Inspect it by hand.
+  when:
+    - (_hdd_vg.stdout | trim) | length > 0
+    - not (_hdd_vg.stdout | trim).startswith('ceph-')
+    - (_hdd_vg.stdout | trim) != pve_linstor_hdd_vg
+
+- name: Remove the stale Ceph volume group
+  ansible.builtin.command:
+    cmd: "vgremove -f {{ _hdd_vg.stdout | trim }}"
+  when:
+    - pve_linstor_wipe_hdd | bool
+    - (_hdd_vg.stdout | trim).startswith('ceph-')
+
+- name: Wipe the HDD's signatures
+  ansible.builtin.command:
+    cmd: "wipefs -a {{ pve_linstor_hdd_disk }}"
+  when:
+    - pve_linstor_wipe_hdd | bool
+    - (_hdd_vg.stdout | trim).startswith('ceph-')
+
+- name: Check whether the HDD VG already exists
+  ansible.builtin.command:
+    cmd: "vgs {{ pve_linstor_hdd_vg }}"
+  register: _hdd_vg_now
+  changed_when: false
+  failed_when: false
+
+- name: Create the HDD volume group
+  ansible.builtin.command:
+    cmd: "vgcreate {{ pve_linstor_hdd_vg }} {{ pve_linstor_hdd_disk }}"
+  when: _hdd_vg_now.rc != 0
+
+- name: Report the resulting volume groups
+  ansible.builtin.command:
+    cmd: "vgs --noheadings -o vg_name,vg_size,vg_free"
+  register: _vgs
+  changed_when: false
+
+- name: Show them
+  ansible.builtin.debug:
+    msg: "{{ _vgs.stdout_lines }}"