Establish clean homelab infrastructure baseline

Reorganize the brownfield repository, remove retired and generated artifacts, harden ignore rules, and record the GitOps/IaC redesign.

infrastructure/openbao/ansible/roles/openbao_bootstrap/tasks/auth_kubernetes.yml

@@ -0,0 +1,52 @@
+---
+# Kubernetes auth — in-cluster agents (e.g. hermes) authenticate with their SA JWT.
+# External bao, so we must supply the cluster host, CA cert and a reviewer JWT.
+# Get them from the cluster:
+#   kubectl -n agents create sa bao-reviewer
+#   kubectl create clusterrolebinding bao-reviewer --clusterrole=system:auth-delegator \
+#     --serviceaccount=agents:bao-reviewer
+#   kubectl -n agents create token bao-reviewer --duration=87600h   → openbao_k8s_reviewer_jwt
+#   kubectl get cm kube-root-ca.crt -o jsonpath='{.data.ca\.crt}' > k8s-ca.crt  → on the bao host
+
+- name: Assert Kubernetes auth inputs are provided
+  ansible.builtin.assert:
+    that:
+      - openbao_k8s_reviewer_jwt | length > 0
+      - openbao_k8s_host | length > 0
+    fail_msg: "Set openbao_k8s_host and vault_openbao_k8s_reviewer_jwt, and place the cluster CA at openbao_k8s_ca_cert_file."
+
+- name: Enable the Kubernetes auth method
+  ansible.builtin.command: "bao auth enable kubernetes"
+  environment: "{{ openbao_cli_env }}"
+  register: k8s_enable
+  changed_when: k8s_enable.rc == 0
+  failed_when:
+    - k8s_enable.rc != 0
+    - "'already in use' not in (k8s_enable.stderr | default('')) + (k8s_enable.stdout | default(''))"
+  no_log: "{{ openbao_no_log }}"
+
+# RECONCILE ACTION — always reports "changed". `bao write` returns 0 whether or
+# not anything differed, and detecting a real diff would mean reading the config
+# back, which never returns token_reviewer_jwt. So a second run showing changed=2
+# for this file means "re-applied", NOT "drift was found". See CLAUDE.md on
+# idempotency being the acceptance test, and the exception for reconcile actions.
+- name: Configure the Kubernetes auth method
+  ansible.builtin.command: >-
+    bao write auth/kubernetes/config
+    kubernetes_host={{ openbao_k8s_host }}
+    kubernetes_ca_cert=@{{ openbao_k8s_ca_cert_file }}
+    token_reviewer_jwt={{ openbao_k8s_reviewer_jwt }}
+    disable_local_ca_jwt=true
+  environment: "{{ openbao_cli_env }}"
+  register: k8s_config
+  changed_when: k8s_config.rc == 0
+  no_log: true    # carries the reviewer JWT
+
+- name: Create/update the ai-agent Kubernetes role
+  ansible.builtin.command: "bao write auth/kubernetes/role/ai-agent -"
+  args:
+    stdin: "{{ lookup('template', 'k8s-ai-agent-role.json.j2') }}"
+  environment: "{{ openbao_cli_env }}"
+  register: k8s_role
+  changed_when: k8s_role.rc == 0
+  no_log: "{{ openbao_no_log }}"

infrastructure/openbao/ansible/roles/openbao_bootstrap/tasks/auth_oidc.yml

@@ -0,0 +1,44 @@
+---
+# OIDC auth via Authelia — human login (bao login -method=oidc / UI).
+# The Authelia 'openbao' client must already exist (authelia/values.yaml).
+
+- name: Assert the OIDC client secret is provided
+  ansible.builtin.assert:
+    that:
+      - openbao_oidc_client_secret | length > 0
+    fail_msg: "Set vault_openbao_oidc_client_secret (plaintext of the Authelia openbao client)."
+
+- name: Enable the OIDC auth method
+  ansible.builtin.command: "bao auth enable -path=oidc oidc"
+  environment: "{{ openbao_cli_env }}"
+  register: oidc_enable
+  changed_when: oidc_enable.rc == 0
+  failed_when:
+    - oidc_enable.rc != 0
+    - "'already in use' not in (oidc_enable.stderr | default('')) + (oidc_enable.stdout | default(''))"
+  no_log: "{{ openbao_no_log }}"
+  # Terraform owns this (../terraform). See openbao_config_managed_by_terraform.
+  when: not openbao_config_managed_by_terraform | bool
+
+- name: Configure the OIDC provider (Authelia)
+  ansible.builtin.command: >-
+    bao write auth/oidc/config
+    oidc_discovery_url={{ openbao_oidc_discovery_url }}
+    oidc_client_id={{ openbao_oidc_client_id }}
+    oidc_client_secret={{ openbao_oidc_client_secret }}
+    default_role={{ openbao_oidc_default_role }}
+  environment: "{{ openbao_cli_env }}"
+  register: oidc_config
+  changed_when: oidc_config.rc == 0
+  no_log: true    # carries the client secret — always hidden
+
+- name: Create/update the admin OIDC role (restricted to the admin AD group)
+  ansible.builtin.command: "bao write auth/oidc/role/{{ openbao_oidc_default_role }} -"
+  args:
+    stdin: "{{ lookup('template', 'oidc-admin-role.json.j2') }}"
+  environment: "{{ openbao_cli_env }}"
+  register: oidc_role
+  changed_when: oidc_role.rc == 0
+  no_log: "{{ openbao_no_log }}"
+  # Terraform owns this (../terraform). See openbao_config_managed_by_terraform.
+  when: not openbao_config_managed_by_terraform | bool

infrastructure/openbao/ansible/roles/openbao_bootstrap/tasks/kv.yml

@@ -0,0 +1,14 @@
+---
+# KV v2 engine for static agent secrets.
+
+- name: Enable KV v2 at {{ openbao_kv_path }}/
+  ansible.builtin.command: "bao secrets enable -path={{ openbao_kv_path }} -version=2 kv"
+  environment: "{{ openbao_cli_env }}"
+  register: kv_enable
+  changed_when: kv_enable.rc == 0
+  failed_when:
+    - kv_enable.rc != 0
+    - "'already in use' not in (kv_enable.stderr | default('')) + (kv_enable.stdout | default(''))"
+  no_log: "{{ openbao_no_log }}"
+  # Terraform owns this (../terraform). See openbao_config_managed_by_terraform.
+  when: not openbao_config_managed_by_terraform | bool

infrastructure/openbao/ansible/roles/openbao_bootstrap/tasks/main.yml

@@ -0,0 +1,59 @@
+---
+# Orchestrator. Preflight, then each concern as its own tagged task file.
+
+- name: Assert a bao token is provided
+  ansible.builtin.assert:
+    that:
+      - openbao_token | length > 0
+    fail_msg: >-
+      No token. Decrypt the PGP-wrapped root token and export it:
+      echo "<b64>" | base64 -d | gpg -dq  →  export BAO_TOKEN=<plaintext>
+
+- name: Preflight — bao is reachable, initialized and UNSEALED
+  ansible.builtin.command: "bao status -format=json"
+  environment:
+    BAO_ADDR: "{{ openbao_addr }}"
+    BAO_CACERT: "{{ openbao_tls_dir }}/cert.pem"
+  register: bao_status
+  changed_when: false
+  failed_when: bao_status.rc != 0   # 0 = unsealed; 2 = sealed → unseal first
+
+- name: Preflight — the provided token is valid
+  ansible.builtin.command: "bao token lookup"
+  environment: "{{ openbao_cli_env }}"
+  register: bao_tok
+  changed_when: false
+  failed_when: bao_tok.rc != 0
+  no_log: "{{ openbao_no_log }}"
+
+- name: Policies
+  # Terraform owns ALL policies (../terraform/policies.tf + policies/*.hcl).
+  # Kept for a Terraform-less bootstrap; see openbao_config_managed_by_terraform.
+  ansible.builtin.import_tasks: policies.yml
+  tags: [policies]
+  when: not openbao_config_managed_by_terraform | bool
+
+- name: KV v2 engine
+  ansible.builtin.import_tasks: kv.yml
+  tags: [kv]
+
+- name: SSH certificate authority
+  ansible.builtin.import_tasks: ssh_ca.yml
+  tags: [ssh_ca]
+
+- name: OIDC auth (Authelia)
+  ansible.builtin.import_tasks: auth_oidc.yml
+  tags: [oidc]
+
+- name: Kubernetes auth
+  ansible.builtin.import_tasks: auth_kubernetes.yml
+  tags: [k8s]
+  when: openbao_enable_k8s_auth | bool
+
+- name: PKI engine
+  ansible.builtin.import_tasks: pki.yml
+  tags: [pki]
+
+- name: Raft snapshot timer
+  ansible.builtin.import_tasks: snapshots.yml
+  tags: [snapshots]

infrastructure/openbao/ansible/roles/openbao_bootstrap/tasks/pki.yml

@@ -0,0 +1,106 @@
+---
+# PKI engine — internal TLS CA. Optionally re-issues bao's own listener cert.
+
+- name: Enable the PKI secrets engine at {{ openbao_pki_mount }}/
+  ansible.builtin.command: "bao secrets enable -path={{ openbao_pki_mount }} pki"
+  environment: "{{ openbao_cli_env }}"
+  register: pki_enable
+  changed_when: pki_enable.rc == 0
+  failed_when:
+    - pki_enable.rc != 0
+    - "'already in use' not in (pki_enable.stderr | default('')) + (pki_enable.stdout | default(''))"
+  no_log: "{{ openbao_no_log }}"
+  # Terraform owns this (../terraform). See openbao_config_managed_by_terraform.
+  when: not openbao_config_managed_by_terraform | bool
+
+- name: Tune PKI max lease TTL
+  ansible.builtin.command: "bao secrets tune -max-lease-ttl={{ openbao_pki_max_lease_ttl }} {{ openbao_pki_mount }}"
+  environment: "{{ openbao_cli_env }}"
+  register: pki_tune
+  changed_when: pki_tune.rc == 0
+  no_log: "{{ openbao_no_log }}"
+  # Terraform owns this (../terraform). See openbao_config_managed_by_terraform.
+  when: not openbao_config_managed_by_terraform | bool
+
+- name: Check whether the root CA already exists
+  ansible.builtin.command: "bao read -field=certificate {{ openbao_pki_mount }}/cert/ca"
+  environment: "{{ openbao_cli_env }}"
+  register: pki_ca_check
+  changed_when: false
+  failed_when: false
+  no_log: "{{ openbao_no_log }}"
+
+- name: Generate the internal root CA (once)
+  ansible.builtin.command: >-
+    bao write {{ openbao_pki_mount }}/root/generate/internal
+    common_name="{{ openbao_pki_ca_cn }}" ttl={{ openbao_pki_max_lease_ttl }}
+  environment: "{{ openbao_cli_env }}"
+  when: "'BEGIN CERTIFICATE' not in (pki_ca_check.stdout | default(''))"
+  register: pki_root
+  changed_when: pki_root.rc == 0
+  no_log: "{{ openbao_no_log }}"
+
+- name: Configure issuing/CRL URLs
+  ansible.builtin.command: >-
+    bao write {{ openbao_pki_mount }}/config/urls
+    issuing_certificates={{ openbao_addr }}/v1/{{ openbao_pki_mount }}/ca
+    crl_distribution_points={{ openbao_addr }}/v1/{{ openbao_pki_mount }}/crl
+  environment: "{{ openbao_cli_env }}"
+  register: pki_urls
+  changed_when: pki_urls.rc == 0
+  no_log: "{{ openbao_no_log }}"
+  # Terraform owns this (../terraform). See openbao_config_managed_by_terraform.
+  when: not openbao_config_managed_by_terraform | bool
+
+- name: Create/update the server-cert role
+  ansible.builtin.command: >-
+    bao write {{ openbao_pki_mount }}/roles/{{ openbao_pki_server_role }}
+    allowed_domains={{ openbao_pki_allowed_domains }}
+    allow_subdomains=true allow_ip_sans=true max_ttl=8760h
+  environment: "{{ openbao_cli_env }}"
+  register: pki_role
+  changed_when: pki_role.rc == 0
+  no_log: "{{ openbao_no_log }}"
+  # Terraform owns this (../terraform). See openbao_config_managed_by_terraform.
+  when: not openbao_config_managed_by_terraform | bool
+
+# --- GATED: replace the self-signed listener cert with a PKI-issued one ----------
+- name: Replace bao's listener cert from PKI
+  when: openbao_pki_replace_listener_cert | bool
+  block:
+    - name: Issue a listener certificate
+      ansible.builtin.command: >-
+        bao write -format=json {{ openbao_pki_mount }}/issue/{{ openbao_pki_server_role }}
+        common_name={{ openbao_fqdn }}
+        ip_sans={{ openbao_lan_ip }},127.0.0.1
+        ttl=8760h
+      environment: "{{ openbao_cli_env }}"
+      register: pki_issue
+      changed_when: true
+      no_log: true
+
+    - name: Install listener private key
+      ansible.builtin.copy:
+        content: "{{ (pki_issue.stdout | from_json).data.private_key }}\n"
+        dest: "{{ openbao_tls_dir }}/key.pem"
+        owner: openbao
+        group: openbao
+        mode: "0640"
+      no_log: true
+      notify: restart openbao
+
+    - name: Install listener certificate (leaf + issuing CA chain)
+      ansible.builtin.copy:
+        content: "{{ (pki_issue.stdout | from_json).data.certificate }}\n{{ (pki_issue.stdout | from_json).data.issuing_ca }}\n"
+        dest: "{{ openbao_tls_dir }}/cert.pem"
+        owner: openbao
+        group: openbao
+        mode: "0644"
+      notify: restart openbao
+
+    - name: Print the PKI root CA (clients must trust this)
+      ansible.builtin.debug:
+        msg: >-
+          Listener now uses a PKI-issued cert. Distribute the root CA to clients:
+          bao read -field=certificate {{ openbao_pki_mount }}/cert/ca
+          (NOTE: a Shamir-sealed node re-seals on restart — unseal it afterwards.)

infrastructure/openbao/ansible/roles/openbao_bootstrap/tasks/policies.yml

@@ -0,0 +1,20 @@
+---
+# Policies. Declaratively applied each run (bao policy write is an idempotent overwrite).
+
+- name: Write the ai-agent-ssh policy (sign SSH certs, nothing else)
+  ansible.builtin.command: "bao policy write ai-agent-ssh -"
+  args:
+    stdin: "{{ lookup('template', 'ai-agent-ssh-policy.hcl.j2') }}"
+  environment: "{{ openbao_cli_env }}"
+  register: pol_agent
+  changed_when: pol_agent.rc == 0
+  no_log: "{{ openbao_no_log }}"
+
+- name: Write the admin policy (human OIDC logins)
+  ansible.builtin.command: "bao policy write admin -"
+  args:
+    stdin: "{{ lookup('template', 'admin-policy.hcl.j2') }}"
+  environment: "{{ openbao_cli_env }}"
+  register: pol_admin
+  changed_when: pol_admin.rc == 0
+  no_log: "{{ openbao_no_log }}"

infrastructure/openbao/ansible/roles/openbao_bootstrap/tasks/snapshots.yml

@@ -0,0 +1,70 @@
+---
+# Automated Raft snapshots via a systemd timer + a scoped periodic token.
+
+- name: Write the snapshot policy (read raft snapshots only)
+  ansible.builtin.command: "bao policy write snapshot -"
+  args:
+    stdin: "{{ lookup('template', 'snapshot-policy.hcl.j2') }}"
+  environment: "{{ openbao_cli_env }}"
+  register: snap_pol
+  changed_when: snap_pol.rc == 0
+  no_log: "{{ openbao_no_log }}"
+  # Terraform owns this (../terraform). See openbao_config_managed_by_terraform.
+  when: not openbao_config_managed_by_terraform | bool
+
+- name: Check whether the snapshot token already exists
+  ansible.builtin.stat:
+    path: /etc/openbao/snapshot.token
+  register: snap_tok_stat
+
+- name: Create a periodic snapshot token (once)
+  ansible.builtin.command: "bao token create -policy=snapshot -period=768h -orphan -field=token"
+  environment: "{{ openbao_cli_env }}"
+  register: snap_tok_new
+  when: not snap_tok_stat.stat.exists
+  changed_when: snap_tok_new.rc == 0
+  no_log: true
+
+- name: Store the snapshot token (root-only)
+  ansible.builtin.copy:
+    content: "{{ snap_tok_new.stdout }}"
+    dest: /etc/openbao/snapshot.token
+    owner: root
+    group: root
+    mode: "0600"
+  when: not snap_tok_stat.stat.exists
+  no_log: true
+
+- name: Ensure the snapshot output directory exists
+  ansible.builtin.file:
+    path: "{{ openbao_snapshot_dir }}"
+    state: directory
+    owner: root
+    group: root
+    mode: "0700"
+
+- name: Install the snapshot script
+  ansible.builtin.template:
+    src: bao-snapshot.sh.j2
+    dest: /usr/local/bin/bao-snapshot.sh
+    owner: root
+    group: root
+    mode: "0755"
+
+- name: Install the snapshot systemd service + timer
+  ansible.builtin.template:
+    src: "{{ item }}.j2"
+    dest: "/etc/systemd/system/{{ item }}"
+    owner: root
+    group: root
+    mode: "0644"
+  loop:
+    - openbao-snapshot.service
+    - openbao-snapshot.timer
+
+- name: Enable and start the snapshot timer
+  ansible.builtin.systemd:
+    name: openbao-snapshot.timer
+    state: started
+    enabled: true
+    daemon_reload: true

infrastructure/openbao/ansible/roles/openbao_bootstrap/tasks/ssh_ca.yml

@@ -0,0 +1,62 @@
+---
+# SSH certificate authority: sign short-lived client certs for the ai-agent role.
+
+- name: Enable the SSH secrets engine at {{ openbao_ssh_mount }}/
+  ansible.builtin.command: "bao secrets enable -path={{ openbao_ssh_mount }} ssh"
+  environment: "{{ openbao_cli_env }}"
+  register: ssh_enable
+  changed_when: ssh_enable.rc == 0
+  failed_when:
+    - ssh_enable.rc != 0
+    - "'already in use' not in (ssh_enable.stderr | default('')) + (ssh_enable.stdout | default(''))"
+  no_log: "{{ openbao_no_log }}"
+  # Terraform owns this (../terraform). See openbao_config_managed_by_terraform.
+  when: not openbao_config_managed_by_terraform | bool
+
+- name: Check whether the SSH CA signing key already exists
+  ansible.builtin.command: "bao read -field=public_key {{ openbao_ssh_mount }}/config/ca"
+  environment: "{{ openbao_cli_env }}"
+  register: ssh_ca_check
+  changed_when: false
+  failed_when: false
+  no_log: "{{ openbao_no_log }}"
+
+- name: Generate the SSH CA signing key (once)
+  ansible.builtin.command: "bao write {{ openbao_ssh_mount }}/config/ca generate_signing_key=true"
+  environment: "{{ openbao_cli_env }}"
+  when: ssh_ca_check.rc != 0
+  register: ssh_ca_gen
+  changed_when: ssh_ca_gen.rc == 0
+  no_log: "{{ openbao_no_log }}"
+
+- name: Create/update the ai-agent signing role
+  ansible.builtin.command: "bao write {{ openbao_ssh_mount }}/roles/ai-agent -"
+  args:
+    stdin: "{{ lookup('template', 'ssh-ai-agent-role.json.j2') }}"
+  environment: "{{ openbao_cli_env }}"
+  register: ssh_role
+  changed_when: ssh_role.rc == 0
+  no_log: "{{ openbao_no_log }}"
+  # Terraform owns this (../terraform). See openbao_config_managed_by_terraform.
+  when: not openbao_config_managed_by_terraform | bool
+
+- name: Fetch the SSH CA public key
+  ansible.builtin.command: "bao read -field=public_key {{ openbao_ssh_mount }}/config/ca"
+  environment: "{{ openbao_cli_env }}"
+  register: ssh_ca_public
+  changed_when: false
+  no_log: "{{ openbao_no_log }}"
+
+- name: Save the SSH CA public key on the bao host (for cert-authority lines)
+  ansible.builtin.copy:
+    content: "{{ ssh_ca_public.stdout }}\n"
+    dest: "{{ openbao_ssh_ca_pub_path }}"
+    owner: root
+    group: root
+    mode: "0644"
+
+- name: Show the cert-authority line for no-root target hosts
+  ansible.builtin.debug:
+    msg: >-
+      Add to ~/.ssh/authorized_keys on each target (scope per node):
+      cert-authority,principals="<node>",restrict,pty {{ ssh_ca_public.stdout }}

infrastructure/openbao/ansible/roles/openbao_bootstrap/tasks/verify.yml

@@ -0,0 +1,45 @@
+---
+# Smoke tests — ansible-playbook bootstrap-openbao.yml --tags verify
+# Confirms the engines/auth are mounted and the SSH CA actually signs.
+
+- name: Secrets engines are mounted
+  ansible.builtin.command: "bao secrets list -format=json"
+  environment: "{{ openbao_cli_env }}"
+  register: v_secrets
+  changed_when: false
+  no_log: "{{ openbao_no_log }}"
+
+- name: Auth methods are enabled
+  ansible.builtin.command: "bao auth list -format=json"
+  environment: "{{ openbao_cli_env }}"
+  register: v_auth
+  changed_when: false
+  no_log: "{{ openbao_no_log }}"
+
+- name: Assert expected mounts exist
+  ansible.builtin.assert:
+    that:
+      - "'{{ openbao_kv_path }}/' in (v_secrets.stdout | from_json)"
+      - "'{{ openbao_ssh_mount }}/' in (v_secrets.stdout | from_json)"
+      - "'{{ openbao_pki_mount }}/' in (v_secrets.stdout | from_json)"
+      - "'oidc/' in (v_auth.stdout | from_json)"
+    fail_msg: "Expected mounts missing — check the bootstrap run."
+
+- name: SSH CA signs a throwaway key (end-to-end)
+  ansible.builtin.shell: >-
+    set -o pipefail;
+    ssh-keygen -t ed25519 -f /tmp/bao-verify -N '' -q -C verify <<<y >/dev/null 2>&1;
+    bao write -field=signed_key {{ openbao_ssh_mount }}/sign/ai-agent
+    public_key=@/tmp/bao-verify.pub valid_principals={{ openbao_ssh_default_user }};
+    rm -f /tmp/bao-verify /tmp/bao-verify.pub
+  args:
+    executable: /bin/bash
+  environment: "{{ openbao_cli_env }}"
+  register: v_sign
+  changed_when: false
+  failed_when: "'ssh-ed25519-cert' not in (v_sign.stdout | default('')) and 'ssh-rsa-cert' not in (v_sign.stdout | default(''))"
+  no_log: "{{ openbao_no_log }}"
+
+- name: Report
+  ansible.builtin.debug:
+    msg: "Bootstrap verified: kv/ ssh-client-signer/ pki/ + oidc auth mounted; SSH CA signed a test cert."