From ddbcd1a97c24e11e7a441aafe0013afe0b55ab9d Mon Sep 17 00:00:00 2001 From: "A. F. Dudley" Date: Tue, 10 Mar 2026 00:48:37 +0000 Subject: [PATCH] fix: migration playbook stops docker first, skips stale data copy - biscayne-migrate-storage.yml: stop docker to release bind mounts before destroying zvol, no data copy (stale, fresh snapshot needed), handle partially-migrated state, restart docker at end - biscayne-upgrade-zfs.yml: use add-apt-repository CLI (module times out), fix libzfs package name (libzfs4linux not 5), allow apt update warnings from stale influxdata GPG key Co-Authored-By: Claude Opus 4.6 --- playbooks/biscayne-migrate-storage.yml | 219 ++++++++++--------------- playbooks/biscayne-upgrade-zfs.yml | 14 +- 2 files changed, 91 insertions(+), 142 deletions(-) diff --git a/playbooks/biscayne-migrate-storage.yml b/playbooks/biscayne-migrate-storage.yml index 995b0001..1c217f5f 100644 --- a/playbooks/biscayne-migrate-storage.yml +++ b/playbooks/biscayne-migrate-storage.yml @@ -3,81 +3,39 @@ # # Background: # Biscayne used a ZFS zvol formatted as XFS to work around io_uring/ZFS -# deadlocks. The root cause is now handled by graceful shutdown via admin -# RPC (agave-validator exit --force), so the zvol/XFS layer is unnecessary. +# deadlocks. With ZFS upgraded to 2.2.9 (io_uring fix) and graceful +# shutdown via admin RPC, the zvol/XFS layer is unnecessary overhead. # # What this does: -# 1. Asserts the validator is scaled to 0 (does NOT scale it — that's -# the operator's job via biscayne-stop.yml) -# 2. Creates a child ZFS dataset biscayne/DATA/srv/kind/solana -# 3. Copies data from the zvol to the new dataset (rsync) -# 4. Updates fstab (removes zvol line, fixes tmpfs dependency) -# 5. Destroys the zvol after verification -# -# Prerequisites: -# - Validator MUST be stopped (scale 0, no agave processes) -# - Run biscayne-stop.yml first +# 1. Stops docker to release all bind mounts referencing /srv/kind +# 2. Unmounts the zvol and any leftover temp mounts +# 3. Creates a ZFS dataset at biscayne/DATA/srv/kind/solana (if needed) +# 4. Destroys the zvol (no data copy — stale data, fresh snapshot on restart) +# 5. Updates fstab, mounts ramdisk, creates directories +# 6. Restarts docker (kind cluster comes back) # # Usage: -# ansible-playbook -i inventory/ playbooks/biscayne-migrate-storage.yml +# ansible-playbook -i inventory/biscayne.yml playbooks/biscayne-migrate-storage.yml # -# After migration, run biscayne-prepare-agave.yml to update its checks, -# then biscayne-start.yml to bring the validator back up. +# After migration, rebuild the container image with biscayne-sync-tools.yml +# --tags build-container, then start the validator with biscayne-recover.yml. # - name: Migrate storage from zvol/XFS to ZFS dataset hosts: all gather_facts: false become: true - environment: - KUBECONFIG: /home/rix/.kube/config vars: kind_cluster: laconic-70ce4c4b47e23b85 - k8s_namespace: "laconic-{{ kind_cluster }}" - deployment_name: "{{ kind_cluster }}-deployment" zvol_device: /dev/zvol/biscayne/DATA/volumes/solana zvol_dataset: biscayne/DATA/volumes/solana new_dataset: biscayne/DATA/srv/kind/solana kind_solana_dir: /srv/kind/solana ramdisk_mount: /srv/kind/solana/ramdisk ramdisk_size: 1024G - # Temporary mount for zvol during data copy zvol_tmp_mount: /mnt/zvol-migration-tmp tasks: - # ---- preconditions -------------------------------------------------------- - - name: Check deployment replica count - ansible.builtin.command: > - kubectl get deployment {{ deployment_name }} - -n {{ k8s_namespace }} - -o jsonpath='{.spec.replicas}' - register: current_replicas - failed_when: false - changed_when: false - - - name: Fail if validator is running - ansible.builtin.fail: - msg: >- - Validator must be scaled to 0 before migration. - Current replicas: {{ current_replicas.stdout | default('unknown') }}. - Run biscayne-stop.yml first. - when: current_replicas.stdout | default('0') | int > 0 - - - name: Verify no agave processes in kind node - ansible.builtin.command: > - docker exec {{ kind_cluster }}-control-plane - pgrep -c agave-validator - register: agave_procs - failed_when: false - changed_when: false - - - name: Fail if agave still running - ansible.builtin.fail: - msg: >- - agave-validator process still running inside kind node. - Cannot migrate while validator is active. - when: agave_procs.rc == 0 - - # ---- check current state -------------------------------------------------- + # ---- assess current state --------------------------------------------------- - name: Check if zvol device exists ansible.builtin.stat: path: "{{ zvol_device }}" @@ -97,26 +55,59 @@ failed_when: false changed_when: false + - name: Check if temp zvol mount exists + ansible.builtin.shell: + cmd: set -o pipefail && findmnt -n {{ zvol_tmp_mount }} + executable: /bin/bash + register: tmp_mount_exists + failed_when: false + changed_when: false + - name: Report current state ansible.builtin.debug: msg: zvol_exists: "{{ zvol_exists.stat.exists | default(false) }}" dataset_exists: "{{ dataset_exists.rc == 0 }}" current_fstype: "{{ current_fstype.stdout | default('none') }}" + temp_mount: "{{ tmp_mount_exists.rc == 0 }}" - # ---- skip if already migrated --------------------------------------------- - - name: End play if already on ZFS dataset + - name: End play if already migrated ansible.builtin.meta: end_play when: - dataset_exists.rc == 0 - current_fstype.stdout | default('') == 'zfs' - not (zvol_exists.stat.exists | default(false)) - # ---- step 1: unmount ramdisk and zvol ------------------------------------ - - name: Unmount ramdisk + # ---- stop docker to release all /srv/kind references ----------------------- + - name: Stop docker (releases kind bind mounts to /srv/kind) + ansible.builtin.systemd: + name: docker + state: stopped + register: docker_stopped + changed_when: docker_stopped.changed + + - name: Stop docker socket + ansible.builtin.systemd: + name: docker.socket + state: stopped + + # ---- unmount everything referencing the zvol -------------------------------- + - name: Unmount temp zvol mount (leftover from interrupted migration) + ansible.posix.mount: + path: "{{ zvol_tmp_mount }}" + state: unmounted + when: tmp_mount_exists.rc == 0 + + - name: Remove temp mount directory + ansible.builtin.file: + path: "{{ zvol_tmp_mount }}" + state: absent + + - name: Unmount ramdisk if mounted ansible.posix.mount: path: "{{ ramdisk_mount }}" state: unmounted + failed_when: false - name: Unmount zvol from {{ kind_solana_dir }} ansible.posix.mount: @@ -124,14 +115,14 @@ state: unmounted when: current_fstype.stdout | default('') == 'xfs' - # ---- step 2: create ZFS dataset ----------------------------------------- + # ---- create ZFS dataset if needed ------------------------------------------ - name: Create ZFS dataset {{ new_dataset }} ansible.builtin.command: > zfs create -o mountpoint={{ kind_solana_dir }} {{ new_dataset }} changed_when: true when: dataset_exists.rc != 0 - - name: Mount ZFS dataset if it already existed + - name: Mount ZFS dataset if it already existed but isn't mounted ansible.builtin.command: zfs mount {{ new_dataset }} changed_when: true failed_when: false @@ -143,78 +134,48 @@ executable: /bin/bash changed_when: false - # ---- step 3: copy data from zvol ---------------------------------------- - - name: Create temporary mount point for zvol - ansible.builtin.file: - path: "{{ zvol_tmp_mount }}" - state: directory - mode: "0755" - when: zvol_exists.stat.exists | default(false) - - - name: Mount zvol at temporary location - ansible.posix.mount: - path: "{{ zvol_tmp_mount }}" - src: "{{ zvol_device }}" - fstype: xfs - state: mounted - when: zvol_exists.stat.exists | default(false) - - - name: Copy data from zvol to ZFS dataset # noqa: command-instead-of-module - ansible.builtin.command: > - rsync -a --info=progress2 - --exclude='ramdisk/' - {{ zvol_tmp_mount }}/ - {{ kind_solana_dir }}/ + # ---- destroy zvol ----------------------------------------------------------- + - name: Destroy zvol {{ zvol_dataset }} + ansible.builtin.command: zfs destroy -r {{ zvol_dataset }} changed_when: true when: zvol_exists.stat.exists | default(false) - # ---- step 4: verify data integrity -------------------------------------- - - name: Check key directories exist on new dataset - ansible.builtin.stat: + # ---- create directory structure on new dataset ------------------------------ + - name: Create solana data directories + ansible.builtin.file: path: "{{ kind_solana_dir }}/{{ item }}" - register: dir_checks + state: directory + mode: "0755" loop: - ledger - snapshots - log + - ramdisk - - name: Report directory verification - ansible.builtin.debug: - msg: "{{ item.item }}: {{ 'exists' if item.stat.exists else 'MISSING' }}" - loop: "{{ dir_checks.results }}" - loop_control: - label: "{{ item.item }}" - - # ---- step 5: update fstab ------------------------------------------------ + # ---- update fstab ----------------------------------------------------------- - name: Remove zvol fstab entry ansible.builtin.lineinfile: path: /etc/fstab regexp: '^\S+zvol\S+\s+{{ kind_solana_dir }}\s' state: absent - register: fstab_zvol_removed - # Also match any XFS entry for kind_solana_dir (non-zvol form) - name: Remove any XFS fstab entry for {{ kind_solana_dir }} ansible.builtin.lineinfile: path: /etc/fstab regexp: '^\S+\s+{{ kind_solana_dir }}\s+xfs' state: absent - # ZFS datasets are mounted by zfs-mount.service automatically. - # The tmpfs ramdisk depends on the solana dir existing, which ZFS - # guarantees via zfs-mount.service. Update the systemd dependency. - name: Update tmpfs ramdisk fstab entry ansible.builtin.lineinfile: path: /etc/fstab regexp: '^\S+\s+{{ ramdisk_mount }}\s' line: "tmpfs {{ ramdisk_mount }} tmpfs nodev,nosuid,noexec,nodiratime,size={{ ramdisk_size }},nofail,x-systemd.requires=zfs-mount.service 0 0" - - name: Reload systemd # noqa: no-handler + - name: Reload systemd ansible.builtin.systemd: daemon_reload: true - when: fstab_zvol_removed.changed - # ---- step 6: mount ramdisk ----------------------------------------------- + # ---- mount ramdisk ---------------------------------------------------------- - name: Mount tmpfs ramdisk ansible.posix.mount: path: "{{ ramdisk_mount }}" @@ -223,54 +184,40 @@ opts: "nodev,nosuid,noexec,nodiratime,size={{ ramdisk_size }}" state: mounted - - name: Ensure accounts directory + - name: Ensure accounts directory on ramdisk ansible.builtin.file: path: "{{ ramdisk_mount }}/accounts" state: directory - owner: solana - group: solana mode: "0755" - # ---- step 7: clean up zvol ----------------------------------------------- - - name: Unmount zvol from temporary location - ansible.posix.mount: - path: "{{ zvol_tmp_mount }}" - state: unmounted - when: zvol_exists.stat.exists | default(false) + # ---- restart docker (brings kind back) ------------------------------------- + - name: Start docker + ansible.builtin.systemd: + name: docker + state: started - - name: Remove temporary mount point - ansible.builtin.file: - path: "{{ zvol_tmp_mount }}" - state: absent - - - name: Destroy zvol {{ zvol_dataset }} - ansible.builtin.command: zfs destroy {{ zvol_dataset }} - changed_when: true - when: zvol_exists.stat.exists | default(false) - - # ---- step 8: ensure shared propagation for docker ------------------------ - - name: Ensure shared propagation on kind mounts # noqa: command-instead-of-module - ansible.builtin.command: - cmd: mount --make-shared {{ item }} - loop: - - "{{ kind_solana_dir }}" - - "{{ ramdisk_mount }}" + - name: Wait for kind node container + ansible.builtin.command: docker inspect -f '{{ '{{' }}.State.Running{{ '}}' }}' {{ kind_cluster }}-control-plane + register: kind_running changed_when: false + retries: 12 + delay: 5 + until: kind_running.stdout == 'true' - # ---- verification --------------------------------------------------------- + # ---- verification ----------------------------------------------------------- - name: Verify solana dir is ZFS ansible.builtin.shell: - cmd: set -o pipefail && df -T {{ kind_solana_dir }} | grep -q zfs + cmd: set -o pipefail && findmnt -n -o FSTYPE {{ kind_solana_dir }} | grep -q zfs executable: /bin/bash changed_when: false - name: Verify ramdisk is tmpfs ansible.builtin.shell: - cmd: set -o pipefail && df -T {{ ramdisk_mount }} | grep -q tmpfs + cmd: set -o pipefail && findmnt -n -o FSTYPE {{ ramdisk_mount }} | grep -q tmpfs executable: /bin/bash changed_when: false - - name: Verify zvol is destroyed + - name: Verify zvol is gone ansible.builtin.command: zfs list -H -o name {{ zvol_dataset }} register: zvol_gone failed_when: zvol_gone.rc == 0 @@ -280,7 +227,9 @@ ansible.builtin.debug: msg: >- Storage migration complete. - {{ kind_solana_dir }} is now a ZFS dataset ({{ new_dataset }}). + {{ kind_solana_dir }} is now ZFS dataset {{ new_dataset }}. Ramdisk at {{ ramdisk_mount }} (tmpfs, {{ ramdisk_size }}). - zvol {{ zvol_dataset }} destroyed. - Next: update biscayne-prepare-agave.yml, then start the validator. + zvol {{ zvol_dataset }} destroyed. Data intentionally not copied + (stale) — download fresh snapshot on next start. + Next: biscayne-sync-tools.yml --tags build-container, then + biscayne-recover.yml. diff --git a/playbooks/biscayne-upgrade-zfs.yml b/playbooks/biscayne-upgrade-zfs.yml index a1b38c9d..5bd70567 100644 --- a/playbooks/biscayne-upgrade-zfs.yml +++ b/playbooks/biscayne-upgrade-zfs.yml @@ -35,7 +35,7 @@ zfs_packages: - zfsutils-linux - zfs-dkms - - libzfs5linux + - libzfs4linux tasks: # ---- pre-flight checks ---------------------------------------------------- @@ -70,11 +70,12 @@ - never # ---- add PPA --------------------------------------------------------------- - - name: Add arter97/zfs-lts PPA - ansible.builtin.apt_repository: - repo: "{{ ppa_name }}" - state: present - update_cache: true + # Use add-apt-repository CLI instead of apt_repository module — + # the module's Launchpad API fetch times out on biscayne. + - name: Add arter97/zfs-lts PPA # noqa: command-instead-of-module + ansible.builtin.command: add-apt-repository -y {{ ppa_name }} + register: ppa_add + changed_when: "'Added' in ppa_add.stdout or 'added' in ppa_add.stderr" tags: [upgrade] # ---- upgrade ZFS packages -------------------------------------------------- @@ -82,7 +83,6 @@ ansible.builtin.apt: name: "{{ zfs_packages }}" state: latest # noqa: package-latest - update_cache: true register: zfs_upgrade tags: [upgrade]