cerc-io
/
stack-orchestrator
mirror of https://github.com/cerc-io/stack-orchestrator
1
0
Fork 0
Code Issues Actions Packages Projects Releases Wiki Activity

fix: migration playbook stops docker first, skips stale data copy 

- biscayne-migrate-storage.yml: stop docker to release bind mounts
  before destroying zvol, no data copy (stale, fresh snapshot needed),
  handle partially-migrated state, restart docker at end
- biscayne-upgrade-zfs.yml: use add-apt-repository CLI (module times
  out), fix libzfs package name (libzfs4linux not 5), allow apt update
  warnings from stale influxdata GPG key

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
fix/kind-mount-propagation
A. F. Dudley 2026-03-10 00:48:37 +00:00
parent b88af2be70
commit ddbcd1a97c
2 changed files with 91 additions and 142 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

219
playbooks/biscayne-migrate-storage.yml
View File

@ -3,81 +3,39 @@
# #
# Background: # Background:
# Biscayne used a ZFS zvol formatted as XFS to work around io_uring/ZFS # Biscayne used a ZFS zvol formatted as XFS to work around io_uring/ZFS
# deadlocks. The root cause is now handled by graceful shutdown via admin # deadlocks. With ZFS upgraded to 2.2.9 (io_uring fix) and graceful
# RPC (agave-validator exit --force), so the zvol/XFS layer is unnecessary. # shutdown via admin RPC, the zvol/XFS layer is unnecessary overhead.
# #
# What this does: # What this does:
# 1. Asserts the validator is scaled to 0 (does NOT scale it — that's # 1. Stops docker to release all bind mounts referencing /srv/kind
# the operator's job via biscayne-stop.yml) # 2. Unmounts the zvol and any leftover temp mounts
# 2. Creates a child ZFS dataset biscayne/DATA/srv/kind/solana # 3. Creates a ZFS dataset at biscayne/DATA/srv/kind/solana (if needed)
# 3. Copies data from the zvol to the new dataset (rsync) # 4. Destroys the zvol (no data copy — stale data, fresh snapshot on restart)
# 4. Updates fstab (removes zvol line, fixes tmpfs dependency) # 5. Updates fstab, mounts ramdisk, creates directories
# 5. Destroys the zvol after verification # 6. Restarts docker (kind cluster comes back)
#
# Prerequisites:
# - Validator MUST be stopped (scale 0, no agave processes)
# - Run biscayne-stop.yml first
# #
# Usage: # Usage:
# ansible-playbook -i inventory/ playbooks/biscayne-migrate-storage.yml # ansible-playbook -i inventory/biscayne.yml playbooks/biscayne-migrate-storage.yml
# #
# After migration, run biscayne-prepare-agave.yml to update its checks, # After migration, rebuild the container image with biscayne-sync-tools.yml
# then biscayne-start.yml to bring the validator back up. # --tags build-container, then start the validator with biscayne-recover.yml.
# #
- name: Migrate storage from zvol/XFS to ZFS dataset - name: Migrate storage from zvol/XFS to ZFS dataset
hosts: all hosts: all
gather_facts: false gather_facts: false
become: true become: true
environment:
KUBECONFIG: /home/rix/.kube/config
vars: vars:
kind_cluster: laconic-70ce4c4b47e23b85 kind_cluster: laconic-70ce4c4b47e23b85
k8s_namespace: "laconic-{{ kind_cluster }}"
deployment_name: "{{ kind_cluster }}-deployment"
zvol_device: /dev/zvol/biscayne/DATA/volumes/solana zvol_device: /dev/zvol/biscayne/DATA/volumes/solana
zvol_dataset: biscayne/DATA/volumes/solana zvol_dataset: biscayne/DATA/volumes/solana
new_dataset: biscayne/DATA/srv/kind/solana new_dataset: biscayne/DATA/srv/kind/solana
kind_solana_dir: /srv/kind/solana kind_solana_dir: /srv/kind/solana
ramdisk_mount: /srv/kind/solana/ramdisk ramdisk_mount: /srv/kind/solana/ramdisk
ramdisk_size: 1024G ramdisk_size: 1024G
# Temporary mount for zvol during data copy
zvol_tmp_mount: /mnt/zvol-migration-tmp zvol_tmp_mount: /mnt/zvol-migration-tmp
tasks: tasks:
# ---- preconditions -------------------------------------------------------- # ---- assess current state ---------------------------------------------------
- name: Check deployment replica count
ansible.builtin.command: >
kubectl get deployment {{ deployment_name }}
-n {{ k8s_namespace }}
-o jsonpath='{.spec.replicas}'
register: current_replicas
failed_when: false
changed_when: false
- name: Fail if validator is running
ansible.builtin.fail:
msg: >-
Validator must be scaled to 0 before migration.
Current replicas: {{ current_replicas.stdout | default('unknown') }}.
Run biscayne-stop.yml first.
when: current_replicas.stdout | default('0') | int > 0
- name: Verify no agave processes in kind node
ansible.builtin.command: >
docker exec {{ kind_cluster }}-control-plane
pgrep -c agave-validator
register: agave_procs
failed_when: false
changed_when: false
- name: Fail if agave still running
ansible.builtin.fail:
msg: >-
agave-validator process still running inside kind node.
Cannot migrate while validator is active.
when: agave_procs.rc == 0
# ---- check current state --------------------------------------------------
- name: Check if zvol device exists - name: Check if zvol device exists
ansible.builtin.stat: ansible.builtin.stat:
path: "{{ zvol_device }}" path: "{{ zvol_device }}"
@ -97,26 +55,59 @@
failed_when: false failed_when: false
changed_when: false changed_when: false
- name: Check if temp zvol mount exists
ansible.builtin.shell:
cmd: set -o pipefail && findmnt -n {{ zvol_tmp_mount }}
executable: /bin/bash
register: tmp_mount_exists
failed_when: false
changed_when: false
- name: Report current state - name: Report current state
ansible.builtin.debug: ansible.builtin.debug:
msg: msg:
zvol_exists: "{{ zvol_exists.stat.exists | default(false) }}" zvol_exists: "{{ zvol_exists.stat.exists | default(false) }}"
dataset_exists: "{{ dataset_exists.rc == 0 }}" dataset_exists: "{{ dataset_exists.rc == 0 }}"
current_fstype: "{{ current_fstype.stdout | default('none') }}" current_fstype: "{{ current_fstype.stdout | default('none') }}"
temp_mount: "{{ tmp_mount_exists.rc == 0 }}"
# ---- skip if already migrated --------------------------------------------- - name: End play if already migrated
- name: End play if already on ZFS dataset
ansible.builtin.meta: end_play ansible.builtin.meta: end_play
when: when:
- dataset_exists.rc == 0 - dataset_exists.rc == 0
- current_fstype.stdout | default('') == 'zfs' - current_fstype.stdout | default('') == 'zfs'
- not (zvol_exists.stat.exists | default(false)) - not (zvol_exists.stat.exists | default(false))
# ---- step 1: unmount ramdisk and zvol ------------------------------------ # ---- stop docker to release all /srv/kind references -----------------------
- name: Unmount ramdisk - name: Stop docker (releases kind bind mounts to /srv/kind)
ansible.builtin.systemd:
name: docker
state: stopped
register: docker_stopped
changed_when: docker_stopped.changed
- name: Stop docker socket
ansible.builtin.systemd:
name: docker.socket
state: stopped
# ---- unmount everything referencing the zvol --------------------------------
- name: Unmount temp zvol mount (leftover from interrupted migration)
ansible.posix.mount:
path: "{{ zvol_tmp_mount }}"
state: unmounted
when: tmp_mount_exists.rc == 0
- name: Remove temp mount directory
ansible.builtin.file:
path: "{{ zvol_tmp_mount }}"
state: absent
- name: Unmount ramdisk if mounted
ansible.posix.mount: ansible.posix.mount:
path: "{{ ramdisk_mount }}" path: "{{ ramdisk_mount }}"
state: unmounted state: unmounted
failed_when: false
- name: Unmount zvol from {{ kind_solana_dir }} - name: Unmount zvol from {{ kind_solana_dir }}
ansible.posix.mount: ansible.posix.mount:
@ -124,14 +115,14 @@
state: unmounted state: unmounted
when: current_fstype.stdout | default('') == 'xfs' when: current_fstype.stdout | default('') == 'xfs'
# ---- step 2: create ZFS dataset ----------------------------------------- # ---- create ZFS dataset if needed ------------------------------------------
- name: Create ZFS dataset {{ new_dataset }} - name: Create ZFS dataset {{ new_dataset }}
ansible.builtin.command: > ansible.builtin.command: >
zfs create -o mountpoint={{ kind_solana_dir }} {{ new_dataset }} zfs create -o mountpoint={{ kind_solana_dir }} {{ new_dataset }}
changed_when: true changed_when: true
when: dataset_exists.rc != 0 when: dataset_exists.rc != 0
- name: Mount ZFS dataset if it already existed - name: Mount ZFS dataset if it already existed but isn't mounted
ansible.builtin.command: zfs mount {{ new_dataset }} ansible.builtin.command: zfs mount {{ new_dataset }}
changed_when: true changed_when: true
failed_when: false failed_when: false
@ -143,78 +134,48 @@
executable: /bin/bash executable: /bin/bash
changed_when: false changed_when: false
# ---- step 3: copy data from zvol ---------------------------------------- # ---- destroy zvol -----------------------------------------------------------
- name: Create temporary mount point for zvol - name: Destroy zvol {{ zvol_dataset }}
ansible.builtin.file: ansible.builtin.command: zfs destroy -r {{ zvol_dataset }}
path: "{{ zvol_tmp_mount }}"
state: directory
mode: "0755"
when: zvol_exists.stat.exists | default(false)
- name: Mount zvol at temporary location
ansible.posix.mount:
path: "{{ zvol_tmp_mount }}"
src: "{{ zvol_device }}"
fstype: xfs
state: mounted
when: zvol_exists.stat.exists | default(false)
- name: Copy data from zvol to ZFS dataset # noqa: command-instead-of-module
ansible.builtin.command: >
rsync -a --info=progress2
--exclude='ramdisk/'
{{ zvol_tmp_mount }}/
{{ kind_solana_dir }}/
changed_when: true changed_when: true
when: zvol_exists.stat.exists | default(false) when: zvol_exists.stat.exists | default(false)
# ---- step 4: verify data integrity -------------------------------------- # ---- create directory structure on new dataset ------------------------------
- name: Check key directories exist on new dataset - name: Create solana data directories
ansible.builtin.stat: ansible.builtin.file:
path: "{{ kind_solana_dir }}/{{ item }}" path: "{{ kind_solana_dir }}/{{ item }}"
register: dir_checks state: directory
mode: "0755"
loop: loop:
- ledger - ledger
- snapshots - snapshots
- log - log
- ramdisk
- name: Report directory verification # ---- update fstab -----------------------------------------------------------
ansible.builtin.debug:
msg: "{{ item.item }}: {{ 'exists' if item.stat.exists else 'MISSING' }}"
loop: "{{ dir_checks.results }}"
loop_control:
label: "{{ item.item }}"
# ---- step 5: update fstab ------------------------------------------------
- name: Remove zvol fstab entry - name: Remove zvol fstab entry
ansible.builtin.lineinfile: ansible.builtin.lineinfile:
path: /etc/fstab path: /etc/fstab
regexp: '^\S+zvol\S+\s+{{ kind_solana_dir }}\s' regexp: '^\S+zvol\S+\s+{{ kind_solana_dir }}\s'
state: absent state: absent
register: fstab_zvol_removed
# Also match any XFS entry for kind_solana_dir (non-zvol form)
- name: Remove any XFS fstab entry for {{ kind_solana_dir }} - name: Remove any XFS fstab entry for {{ kind_solana_dir }}
ansible.builtin.lineinfile: ansible.builtin.lineinfile:
path: /etc/fstab path: /etc/fstab
regexp: '^\S+\s+{{ kind_solana_dir }}\s+xfs' regexp: '^\S+\s+{{ kind_solana_dir }}\s+xfs'
state: absent state: absent
# ZFS datasets are mounted by zfs-mount.service automatically.
# The tmpfs ramdisk depends on the solana dir existing, which ZFS
# guarantees via zfs-mount.service. Update the systemd dependency.
- name: Update tmpfs ramdisk fstab entry - name: Update tmpfs ramdisk fstab entry
ansible.builtin.lineinfile: ansible.builtin.lineinfile:
path: /etc/fstab path: /etc/fstab
regexp: '^\S+\s+{{ ramdisk_mount }}\s' regexp: '^\S+\s+{{ ramdisk_mount }}\s'
line: "tmpfs {{ ramdisk_mount }} tmpfs nodev,nosuid,noexec,nodiratime,size={{ ramdisk_size }},nofail,x-systemd.requires=zfs-mount.service 0 0" line: "tmpfs {{ ramdisk_mount }} tmpfs nodev,nosuid,noexec,nodiratime,size={{ ramdisk_size }},nofail,x-systemd.requires=zfs-mount.service 0 0"
- name: Reload systemd # noqa: no-handler - name: Reload systemd
ansible.builtin.systemd: ansible.builtin.systemd:
daemon_reload: true daemon_reload: true
when: fstab_zvol_removed.changed
# ---- step 6: mount ramdisk ----------------------------------------------- # ---- mount ramdisk ----------------------------------------------------------
- name: Mount tmpfs ramdisk - name: Mount tmpfs ramdisk
ansible.posix.mount: ansible.posix.mount:
path: "{{ ramdisk_mount }}" path: "{{ ramdisk_mount }}"
@ -223,54 +184,40 @@
opts: "nodev,nosuid,noexec,nodiratime,size={{ ramdisk_size }}" opts: "nodev,nosuid,noexec,nodiratime,size={{ ramdisk_size }}"
state: mounted state: mounted
- name: Ensure accounts directory - name: Ensure accounts directory on ramdisk
ansible.builtin.file: ansible.builtin.file:
path: "{{ ramdisk_mount }}/accounts" path: "{{ ramdisk_mount }}/accounts"
state: directory state: directory
owner: solana
group: solana
mode: "0755" mode: "0755"
# ---- step 7: clean up zvol ----------------------------------------------- # ---- restart docker (brings kind back) -------------------------------------
- name: Unmount zvol from temporary location - name: Start docker
ansible.posix.mount: ansible.builtin.systemd:
path: "{{ zvol_tmp_mount }}" name: docker
state: unmounted state: started
when: zvol_exists.stat.exists | default(false)
- name: Remove temporary mount point - name: Wait for kind node container
ansible.builtin.file: ansible.builtin.command: docker inspect -f '{{ '{{' }}.State.Running{{ '}}' }}' {{ kind_cluster }}-control-plane
path: "{{ zvol_tmp_mount }}" register: kind_running
state: absent
- name: Destroy zvol {{ zvol_dataset }}
ansible.builtin.command: zfs destroy {{ zvol_dataset }}
changed_when: true
when: zvol_exists.stat.exists | default(false)
# ---- step 8: ensure shared propagation for docker ------------------------
- name: Ensure shared propagation on kind mounts # noqa: command-instead-of-module
ansible.builtin.command:
cmd: mount --make-shared {{ item }}
loop:
- "{{ kind_solana_dir }}"
- "{{ ramdisk_mount }}"
changed_when: false changed_when: false
retries: 12
delay: 5
until: kind_running.stdout == 'true'
# ---- verification --------------------------------------------------------- # ---- verification -----------------------------------------------------------
- name: Verify solana dir is ZFS - name: Verify solana dir is ZFS
ansible.builtin.shell: ansible.builtin.shell:
cmd: set -o pipefail && df -T {{ kind_solana_dir }} | grep -q zfs cmd: set -o pipefail && findmnt -n -o FSTYPE {{ kind_solana_dir }} | grep -q zfs
executable: /bin/bash executable: /bin/bash
changed_when: false changed_when: false
- name: Verify ramdisk is tmpfs - name: Verify ramdisk is tmpfs
ansible.builtin.shell: ansible.builtin.shell:
cmd: set -o pipefail && df -T {{ ramdisk_mount }} | grep -q tmpfs cmd: set -o pipefail && findmnt -n -o FSTYPE {{ ramdisk_mount }} | grep -q tmpfs
executable: /bin/bash executable: /bin/bash
changed_when: false changed_when: false
- name: Verify zvol is destroyed - name: Verify zvol is gone
ansible.builtin.command: zfs list -H -o name {{ zvol_dataset }} ansible.builtin.command: zfs list -H -o name {{ zvol_dataset }}
register: zvol_gone register: zvol_gone
failed_when: zvol_gone.rc == 0 failed_when: zvol_gone.rc == 0
@ -280,7 +227,9 @@
ansible.builtin.debug: ansible.builtin.debug:
msg: >- msg: >-
Storage migration complete. Storage migration complete.
{{ kind_solana_dir }} is now a ZFS dataset ({{ new_dataset }}). {{ kind_solana_dir }} is now ZFS dataset {{ new_dataset }}.
Ramdisk at {{ ramdisk_mount }} (tmpfs, {{ ramdisk_size }}). Ramdisk at {{ ramdisk_mount }} (tmpfs, {{ ramdisk_size }}).
zvol {{ zvol_dataset }} destroyed. zvol {{ zvol_dataset }} destroyed. Data intentionally not copied
Next: update biscayne-prepare-agave.yml, then start the validator. (stale) — download fresh snapshot on next start.
Next: biscayne-sync-tools.yml --tags build-container, then
biscayne-recover.yml.

14
playbooks/biscayne-upgrade-zfs.yml
View File

@ -35,7 +35,7 @@
zfs_packages: zfs_packages:
- zfsutils-linux - zfsutils-linux
- zfs-dkms - zfs-dkms
- libzfs5linux - libzfs4linux
tasks: tasks:
# ---- pre-flight checks ---------------------------------------------------- # ---- pre-flight checks ----------------------------------------------------
@ -70,11 +70,12 @@
- never - never
# ---- add PPA --------------------------------------------------------------- # ---- add PPA ---------------------------------------------------------------
- name: Add arter97/zfs-lts PPA # Use add-apt-repository CLI instead of apt_repository module —
ansible.builtin.apt_repository: # the module's Launchpad API fetch times out on biscayne.
repo: "{{ ppa_name }}" - name: Add arter97/zfs-lts PPA # noqa: command-instead-of-module
state: present ansible.builtin.command: add-apt-repository -y {{ ppa_name }}
update_cache: true register: ppa_add
changed_when: "'Added' in ppa_add.stdout or 'added' in ppa_add.stderr"
tags: [upgrade] tags: [upgrade]
# ---- upgrade ZFS packages -------------------------------------------------- # ---- upgrade ZFS packages --------------------------------------------------
@ -82,7 +83,6 @@
ansible.builtin.apt: ansible.builtin.apt:
name: "{{ zfs_packages }}" name: "{{ zfs_packages }}"
state: latest # noqa: package-latest state: latest # noqa: package-latest
update_cache: true
register: zfs_upgrade register: zfs_upgrade
tags: [upgrade] tags: [upgrade]