stack-orchestrator/ansible/biscayne-redeploy.yml

---
# Redeploy agave-stack on biscayne with aria2c snapshot pre-download
#
# Usage:
#   # Standard redeploy (download snapshot, preserve accounts + ledger)
#   ansible-playbook -i biscayne.vaasl.io, ansible/biscayne-redeploy.yml
#
#   # Full wipe (accounts + ledger) — slow rebuild
#   ansible-playbook -i biscayne.vaasl.io, ansible/biscayne-redeploy.yml \
#     -e wipe_accounts=true -e wipe_ledger=true
#
#   # Skip snapshot download (use existing)
#   ansible-playbook -i biscayne.vaasl.io, ansible/biscayne-redeploy.yml \
#     -e skip_snapshot=true
#
#   # Pass extra args to snapshot-download.py
#   ansible-playbook -i biscayne.vaasl.io, ansible/biscayne-redeploy.yml \
#     -e 'snapshot_args=--version 2.2 --min-download-speed 50'
#
#   # Snapshot only (no redeploy)
#   ansible-playbook -i biscayne.vaasl.io, ansible/biscayne-redeploy.yml --tags snapshot
#
- name: Redeploy agave validator on biscayne
  hosts: all
  gather_facts: false
  vars:
    deployment_dir: /srv/deployments/agave
    laconic_so: /home/rix/.local/bin/laconic-so
    kind_cluster: laconic-70ce4c4b47e23b85
    k8s_namespace: "laconic-{{ kind_cluster }}"
    snapshot_dir: /srv/solana/snapshots
    ledger_dir: /srv/solana/ledger
    accounts_dir: /srv/solana/ramdisk/accounts
    ramdisk_mount: /srv/solana/ramdisk
    ramdisk_device: /dev/ram0
    snapshot_script_local: "{{ playbook_dir }}/../scripts/snapshot-download.py"
    snapshot_script: /tmp/snapshot-download.py
    # Flags — non-destructive by default
    wipe_accounts: false
    wipe_ledger: false
    skip_snapshot: false
    snapshot_args: ""

  tasks:
    # --- Snapshot download (runs while validator is still up) ---
    - name: Verify aria2c installed
      command: which aria2c
      changed_when: false
      when: not skip_snapshot | bool
      tags: [snapshot]

    - name: Copy snapshot script to remote
      copy:
        src: "{{ snapshot_script_local }}"
        dest: "{{ snapshot_script }}"
        mode: "0755"
      when: not skip_snapshot | bool
      tags: [snapshot]

    - name: Download snapshot via aria2c
      command: >
        python3 {{ snapshot_script }}
        -o {{ snapshot_dir }}
        {{ snapshot_args }}        
      become: true
      register: snapshot_result
      when: not skip_snapshot | bool
      timeout: 3600
      tags: [snapshot]

    - name: Show snapshot download result
      debug:
        msg: "{{ snapshot_result.stdout_lines | default(['skipped']) }}"
      tags: [snapshot]

    # --- Teardown (namespace only, preserve kind cluster) ---
    - name: Delete deployment namespace
      command: >
        kubectl delete namespace {{ k8s_namespace }} --timeout=120s        
      register: ns_delete
      failed_when: false
      tags: [teardown]

    - name: Wait for namespace to terminate
      command: >
        kubectl get namespace {{ k8s_namespace }}
        -o jsonpath='{.status.phase}'        
      register: ns_status
      retries: 30
      delay: 5
      until: ns_status.rc != 0
      failed_when: false
      when: ns_delete.rc == 0
      tags: [teardown]

    # --- Data wipe (opt-in) ---
    - name: Wipe ledger data
      shell: rm -rf {{ ledger_dir }}/*
      become: true
      when: wipe_ledger | bool
      tags: [wipe]

    - name: Wipe accounts ramdisk (umount + mkfs + mount)
      shell: |
        mountpoint -q {{ ramdisk_mount }} && umount {{ ramdisk_mount }} || true
        mkfs.ext4 -q {{ ramdisk_device }}
        mount {{ ramdisk_device }} {{ ramdisk_mount }}
        mkdir -p {{ accounts_dir }}
        chown solana:solana {{ ramdisk_mount }} {{ accounts_dir }}        
      become: true
      when: wipe_accounts | bool
      tags: [wipe]

    - name: Clean old snapshots (keep newest full + incremental)
      shell: |
        cd {{ snapshot_dir }} || exit 0
        newest=$(ls -t snapshot-*.tar.* 2>/dev/null | head -1)
        if [ -n "$newest" ]; then
          newest_inc=$(ls -t incremental-snapshot-*.tar.* 2>/dev/null | head -1)
          find . -maxdepth 1 -name '*.tar.*' \
            ! -name "$newest" \
            ! -name "${newest_inc:-__none__}" \
            -delete
        fi        
      become: true
      when: not skip_snapshot | bool
      tags: [wipe]

    # --- Deploy ---
    - name: Verify kind-config.yml has unified mount root
      command: "grep -c 'containerPath: /mnt$' {{ deployment_dir }}/kind-config.yml"
      register: mount_root_check
      failed_when: mount_root_check.stdout | int < 1
      tags: [deploy]

    - name: Start deployment
      command: "{{ laconic_so }} deployment --dir {{ deployment_dir }} start"
      timeout: 600
      tags: [deploy]

    - name: Wait for pod to be running
      command: >
        kubectl get pods -n {{ k8s_namespace }}
        -o jsonpath='{.items[0].status.phase}'        
      register: pod_status
      retries: 60
      delay: 10
      until: pod_status.stdout == "Running"
      tags: [deploy]

    # --- Verify ---
    - name: Verify unified mount inside kind node
      command: "docker exec {{ kind_cluster }}-control-plane ls /mnt/solana/"
      register: mount_check
      tags: [verify]

    - name: Show mount contents
      debug:
        msg: "{{ mount_check.stdout_lines }}"
      tags: [verify]

    - name: Check validator log file is being written
      command: >
        kubectl exec -n {{ k8s_namespace }}
        deployment/{{ kind_cluster }}-deployment
        -c agave-validator -- test -f /data/log/validator.log        
      retries: 12
      delay: 10
      until: log_file_check.rc == 0
      register: log_file_check
      failed_when: false
      tags: [verify]

    - name: Check RPC health
      uri:
        url: http://127.0.0.1:8899/health
        return_content: true
      register: rpc_health
      retries: 6
      delay: 10
      until: rpc_health.status == 200
      failed_when: false
      delegate_to: "{{ inventory_hostname }}"
      tags: [verify]

    - name: Report status
      debug:
        msg: >-
          Deployment complete.
          Log: {{ 'writing' if log_file_check.rc == 0 else 'not yet created' }}.
          RPC: {{ rpc_health.content | default('not responding') }}.
          Wiped: ledger={{ wipe_ledger }}, accounts={{ wipe_accounts }}.          
      tags: [verify]