194 lines
6.1 KiB
YAML
194 lines
6.1 KiB
YAML
|
---
|
||
|
|
# Redeploy agave-stack on biscayne with aria2c snapshot pre-download
|
||
|
|
#
|
||
|
|
# Usage:
|
||
|
|
# # Standard redeploy (download snapshot, preserve accounts + ledger)
|
||
|
|
# ansible-playbook -i biscayne.vaasl.io, ansible/biscayne-redeploy.yml
|
||
|
|
#
|
||
|
|
# # Full wipe (accounts + ledger) — slow rebuild
|
||
|
|
# ansible-playbook -i biscayne.vaasl.io, ansible/biscayne-redeploy.yml \
|
||
|
|
# -e wipe_accounts=true -e wipe_ledger=true
|
||
|
|
#
|
||
|
|
# # Skip snapshot download (use existing)
|
||
|
|
# ansible-playbook -i biscayne.vaasl.io, ansible/biscayne-redeploy.yml \
|
||
|
|
# -e skip_snapshot=true
|
||
|
|
#
|
||
|
|
# # Pass extra args to snapshot-download.py
|
||
|
|
# ansible-playbook -i biscayne.vaasl.io, ansible/biscayne-redeploy.yml \
|
||
|
|
# -e 'snapshot_args=--version 2.2 --min-download-speed 50'
|
||
|
|
#
|
||
|
|
# # Snapshot only (no redeploy)
|
||
|
|
# ansible-playbook -i biscayne.vaasl.io, ansible/biscayne-redeploy.yml --tags snapshot
|
||
|
|
#
|
||
|
|
- name: Redeploy agave validator on biscayne
|
||
|
|
hosts: all
|
||
|
|
gather_facts: false
|
||
|
|
vars:
|
||
|
|
deployment_dir: /srv/deployments/agave
|
||
|
|
laconic_so: /home/rix/.local/bin/laconic-so
|
||
|
|
kind_cluster: laconic-70ce4c4b47e23b85
|
||
|
|
k8s_namespace: "laconic-{{ kind_cluster }}"
|
||
|
|
snapshot_dir: /srv/solana/snapshots
|
||
|
|
ledger_dir: /srv/solana/ledger
|
||
|
|
accounts_dir: /srv/solana/ramdisk/accounts
|
||
|
|
ramdisk_mount: /srv/solana/ramdisk
|
||
|
|
ramdisk_device: /dev/ram0
|
||
|
|
snapshot_script_local: "{{ playbook_dir }}/../scripts/snapshot-download.py"
|
||
|
|
snapshot_script: /tmp/snapshot-download.py
|
||
|
|
# Flags — non-destructive by default
|
||
|
|
wipe_accounts: false
|
||
|
|
wipe_ledger: false
|
||
|
|
skip_snapshot: false
|
||
|
|
snapshot_args: ""
|
||
|
|
|
||
|
|
tasks:
|
||
|
|
# --- Snapshot download (runs while validator is still up) ---
|
||
|
|
- name: Verify aria2c installed
|
||
|
|
command: which aria2c
|
||
|
|
changed_when: false
|
||
|
|
when: not skip_snapshot | bool
|
||
|
|
tags: [snapshot]
|
||
|
|
|
||
|
|
- name: Copy snapshot script to remote
|
||
|
|
copy:
|
||
|
|
src: "{{ snapshot_script_local }}"
|
||
|
|
dest: "{{ snapshot_script }}"
|
||
|
|
mode: "0755"
|
||
|
|
when: not skip_snapshot | bool
|
||
|
|
tags: [snapshot]
|
||
|
|
|
||
|
|
- name: Download snapshot via aria2c
|
||
|
|
command: >
|
||
|
|
python3 {{ snapshot_script }}
|
||
|
|
-o {{ snapshot_dir }}
|
||
|
|
{{ snapshot_args }}
|
||
|
|
become: true
|
||
|
|
register: snapshot_result
|
||
|
|
when: not skip_snapshot | bool
|
||
|
|
timeout: 3600
|
||
|
|
tags: [snapshot]
|
||
|
|
|
||
|
|
- name: Show snapshot download result
|
||
|
|
debug:
|
||
|
|
msg: "{{ snapshot_result.stdout_lines | default(['skipped']) }}"
|
||
|
|
tags: [snapshot]
|
||
|
|
|
||
|
|
# --- Teardown (namespace only, preserve kind cluster) ---
|
||
|
|
- name: Delete deployment namespace
|
||
|
|
command: >
|
||
|
|
kubectl delete namespace {{ k8s_namespace }} --timeout=120s
|
||
|
|
register: ns_delete
|
||
|
|
failed_when: false
|
||
|
|
tags: [teardown]
|
||
|
|
|
||
|
|
- name: Wait for namespace to terminate
|
||
|
|
command: >
|
||
|
|
kubectl get namespace {{ k8s_namespace }}
|
||
|
|
-o jsonpath='{.status.phase}'
|
||
|
|
register: ns_status
|
||
|
|
retries: 30
|
||
|
|
delay: 5
|
||
|
|
until: ns_status.rc != 0
|
||
|
|
failed_when: false
|
||
|
|
when: ns_delete.rc == 0
|
||
|
|
tags: [teardown]
|
||
|
|
|
||
|
|
# --- Data wipe (opt-in) ---
|
||
|
|
- name: Wipe ledger data
|
||
|
|
shell: rm -rf {{ ledger_dir }}/*
|
||
|
|
become: true
|
||
|
|
when: wipe_ledger | bool
|
||
|
|
tags: [wipe]
|
||
|
|
|
||
|
|
- name: Wipe accounts ramdisk (umount + mkfs + mount)
|
||
|
|
shell: |
|
||
|
|
mountpoint -q {{ ramdisk_mount }} && umount {{ ramdisk_mount }} || true
|
||
|
|
mkfs.ext4 -q {{ ramdisk_device }}
|
||
|
|
mount {{ ramdisk_device }} {{ ramdisk_mount }}
|
||
|
|
mkdir -p {{ accounts_dir }}
|
||
|
|
chown solana:solana {{ ramdisk_mount }} {{ accounts_dir }}
|
||
|
|
become: true
|
||
|
|
when: wipe_accounts | bool
|
||
|
|
tags: [wipe]
|
||
|
|
|
||
|
|
- name: Clean old snapshots (keep newest full + incremental)
|
||
|
|
shell: |
|
||
|
|
cd {{ snapshot_dir }} || exit 0
|
||
|
|
newest=$(ls -t snapshot-*.tar.* 2>/dev/null | head -1)
|
||
|
|
if [ -n "$newest" ]; then
|
||
|
|
newest_inc=$(ls -t incremental-snapshot-*.tar.* 2>/dev/null | head -1)
|
||
|
|
find . -maxdepth 1 -name '*.tar.*' \
|
||
|
|
! -name "$newest" \
|
||
|
|
! -name "${newest_inc:-__none__}" \
|
||
|
|
-delete
|
||
|
|
fi
|
||
|
|
become: true
|
||
|
|
when: not skip_snapshot | bool
|
||
|
|
tags: [wipe]
|
||
|
|
|
||
|
|
# --- Deploy ---
|
||
|
|
- name: Verify kind-config.yml has unified mount root
|
||
|
|
command: "grep -c 'containerPath: /mnt$' {{ deployment_dir }}/kind-config.yml"
|
||
|
|
register: mount_root_check
|
||
|
|
failed_when: mount_root_check.stdout | int < 1
|
||
|
|
tags: [deploy]
|
||
|
|
|
||
|
|
- name: Start deployment
|
||
|
|
command: "{{ laconic_so }} deployment --dir {{ deployment_dir }} start"
|
||
|
|
timeout: 600
|
||
|
|
tags: [deploy]
|
||
|
|
|
||
|
|
- name: Wait for pod to be running
|
||
|
|
command: >
|
||
|
|
kubectl get pods -n {{ k8s_namespace }}
|
||
|
|
-o jsonpath='{.items[0].status.phase}'
|
||
|
|
register: pod_status
|
||
|
|
retries: 60
|
||
|
|
delay: 10
|
||
|
|
until: pod_status.stdout == "Running"
|
||
|
|
tags: [deploy]
|
||
|
|
|
||
|
|
# --- Verify ---
|
||
|
|
- name: Verify unified mount inside kind node
|
||
|
|
command: "docker exec {{ kind_cluster }}-control-plane ls /mnt/solana/"
|
||
|
|
register: mount_check
|
||
|
|
tags: [verify]
|
||
|
|
|
||
|
|
- name: Show mount contents
|
||
|
|
debug:
|
||
|
|
msg: "{{ mount_check.stdout_lines }}"
|
||
|
|
tags: [verify]
|
||
|
|
|
||
|
|
- name: Check validator log file is being written
|
||
|
|
command: >
|
||
|
|
kubectl exec -n {{ k8s_namespace }}
|
||
|
|
deployment/{{ kind_cluster }}-deployment
|
||
|
|
-c agave-validator -- test -f /data/log/validator.log
|
||
|
|
retries: 12
|
||
|
|
delay: 10
|
||
|
|
until: log_file_check.rc == 0
|
||
|
|
register: log_file_check
|
||
|
|
failed_when: false
|
||
|
|
tags: [verify]
|
||
|
|
|
||
|
|
- name: Check RPC health
|
||
|
|
uri:
|
||
|
|
url: http://127.0.0.1:8899/health
|
||
|
|
return_content: true
|
||
|
|
register: rpc_health
|
||
|
|
retries: 6
|
||
|
|
delay: 10
|
||
|
|
until: rpc_health.status == 200
|
||
|
|
failed_when: false
|
||
|
|
delegate_to: "{{ inventory_hostname }}"
|
||
|
|
tags: [verify]
|
||
|
|
|
||
|
|
- name: Report status
|
||
|
|
debug:
|
||
|
|
msg: >-
|
||
|
|
Deployment complete.
|
||
|
|
Log: {{ 'writing' if log_file_check.rc == 0 else 'not yet created' }}.
|
||
|
|
RPC: {{ rpc_health.content | default('not responding') }}.
|
||
|
|
Wiped: ledger={{ wipe_ledger }}, accounts={{ wipe_accounts }}.
|
||
|
|
tags: [verify]
|