stack-orchestrator/playbooks/fix-pv-mounts.yml

---
# Verify PV hostPaths match expected kind-node paths, fix if wrong.
#
# Checks each PV's hostPath against the expected path derived from the
# spec.yml volume mapping through the kind extraMounts. If any PV has a
# wrong path, fails unless -e fix=true is passed.
#
# Does NOT touch the deployment.
#
# Usage:
#   # Check only (fails if mounts are bad)
#   ansible-playbook -i biscayne.vaasl.io, playbooks/fix-pv-mounts.yml
#
#   # Fix stale PVs
#   ansible-playbook -i biscayne.vaasl.io, playbooks/fix-pv-mounts.yml -e fix=true
#
- name: Verify and fix PV mount paths
  hosts: all
  gather_facts: false
  environment:
    KUBECONFIG: /home/rix/.kube/config
  vars:
    kind_cluster: laconic-70ce4c4b47e23b85
    k8s_namespace: "laconic-{{ kind_cluster }}"
    fix: false
    volumes:
      - name: validator-snapshots
        host_path: /mnt/solana/snapshots
        capacity: 200Gi
      - name: validator-ledger
        host_path: /mnt/solana/ledger
        capacity: 2Ti
      - name: validator-accounts
        host_path: /mnt/solana/ramdisk/accounts
        capacity: 800Gi
      - name: validator-log
        host_path: /mnt/solana/log
        capacity: 10Gi

  tasks:
    - name: Read current PV hostPaths
      ansible.builtin.command: >
        kubectl get pv {{ kind_cluster }}-{{ item.name }}
        -o jsonpath='{.spec.hostPath.path}'
      register: current_paths
      loop: "{{ volumes }}"
      failed_when: false
      changed_when: false

    - name: Build path comparison
      ansible.builtin.set_fact:
        path_mismatches: "{{ current_paths.results | selectattr('stdout', 'ne', '') | rejectattr('stdout', 'equalto', item.host_path) | list }}"
        path_missing: "{{ current_paths.results | selectattr('stdout', 'equalto', '') | list }}"
      loop: "{{ volumes }}"
      loop_control:
        label: "{{ item.name }}"

    - name: Show current vs expected paths
      ansible.builtin.debug:
        msg: >-
          {{ item.item.name }}:
          current={{ item.stdout if item.stdout else 'NOT FOUND' }}
          expected={{ item.item.host_path }}
          {{ 'OK' if item.stdout == item.item.host_path else 'NEEDS FIX' }}
      loop: "{{ current_paths.results }}"
      loop_control:
        label: "{{ item.item.name }}"

    - name: Check for mismatched PVs
      ansible.builtin.fail:
        msg: >-
          PV {{ item.item.name }} has wrong hostPath:
          {{ item.stdout if item.stdout else 'NOT FOUND' }}
          (expected {{ item.item.host_path }}).
          Run with -e fix=true to delete and recreate.
      when: item.stdout != item.item.host_path and not fix | bool
      loop: "{{ current_paths.results }}"
      loop_control:
        label: "{{ item.item.name }}"

    # ---- Fix mode ---------------------------------------------------------
    - name: Delete stale PVCs
      ansible.builtin.command: >
        kubectl delete pvc {{ kind_cluster }}-{{ item.item.name }}
        -n {{ k8s_namespace }} --timeout=60s
      when: fix | bool and item.stdout != item.item.host_path
      loop: "{{ current_paths.results }}"
      loop_control:
        label: "{{ item.item.name }}"
      failed_when: false
      changed_when: true

    - name: Delete stale PVs
      ansible.builtin.command: >
        kubectl delete pv {{ kind_cluster }}-{{ item.item.name }}
        --timeout=60s
      when: fix | bool and item.stdout != item.item.host_path
      loop: "{{ current_paths.results }}"
      loop_control:
        label: "{{ item.item.name }}"
      failed_when: false
      changed_when: true

    - name: Create PVs with correct hostPaths
      ansible.builtin.command: >
        kubectl apply -f -
      args:
        stdin: |
          apiVersion: v1
          kind: PersistentVolume
          metadata:
            name: {{ kind_cluster }}-{{ item.item.name }}
          spec:
            capacity:
              storage: {{ item.item.capacity }}
            accessModes:
              - ReadWriteOnce
            persistentVolumeReclaimPolicy: Retain
            storageClassName: manual
            hostPath:
              path: {{ item.item.host_path }}
      when: fix | bool and item.stdout != item.item.host_path
      loop: "{{ current_paths.results }}"
      loop_control:
        label: "{{ item.item.name }}"
      changed_when: true

    - name: Create PVCs
      ansible.builtin.command: >
        kubectl apply -f -
      args:
        stdin: |
          apiVersion: v1
          kind: PersistentVolumeClaim
          metadata:
            name: {{ kind_cluster }}-{{ item.item.name }}
            namespace: {{ k8s_namespace }}
          spec:
            accessModes:
              - ReadWriteOnce
            storageClassName: manual
            volumeName: {{ kind_cluster }}-{{ item.item.name }}
            resources:
              requests:
                storage: {{ item.item.capacity }}
      when: fix | bool and item.stdout != item.item.host_path
      loop: "{{ current_paths.results }}"
      loop_control:
        label: "{{ item.item.name }}"
      changed_when: true

    # ---- Final verify -----------------------------------------------------
    - name: Verify PV paths
      ansible.builtin.command: >
        kubectl get pv {{ kind_cluster }}-{{ item.name }}
        -o jsonpath='{.spec.hostPath.path}'
      register: final_paths
      loop: "{{ volumes }}"
      changed_when: false
      when: fix | bool

    - name: Assert all PV paths correct
      ansible.builtin.assert:
        that: item.stdout == item.item.host_path
        fail_msg: "{{ item.item.name }}: {{ item.stdout }} != {{ item.item.host_path }}"
        success_msg: "{{ item.item.name }}: {{ item.stdout }} OK"
      loop: "{{ final_paths.results }}"
      loop_control:
        label: "{{ item.item.name }}"
      when: fix | bool