#!/bin/bash

set -Eeuo pipefail

export PATH=/sbin:/bin:/usr/sbin:/usr/bin:/usr/local/sbin:/usr/local/bin
export XDG_RUNTIME_DIR="/run/user/$(id -u)"
mkdir -p "$XDG_RUNTIME_DIR"

# optional suffix from command-line, prepend dash if non-empty
SUFFIX="${1:-}"
SUFFIX="${SUFFIX:+-$SUFFIX}"

# define variables
DATASET="biscayne/DATA/deployments"
DEPLOYMENT_DIR="/srv/deployments/agave"
LOG_FILE="$HOME/.backlog_history"
ZFS_HOLD="backlog:pending"
SERVICE_STOP_TIMEOUT="300"
SNAPSHOT_RETENTION="6"
SNAPSHOT_PREFIX="backlog"
SNAPSHOT_TAG="$(date +%Y%m%d)${SUFFIX}"
SNAPSHOT="${DATASET}@${SNAPSHOT_PREFIX}-${SNAPSHOT_TAG}"

# remote replication targets
REMOTES=(
    "mysterio:edith/DATA/backlog/biscayne-main"
    "ardham:batterywharf/DATA/backlog/biscayne-main"
)

# log functions
log() {
    local time_fmt
    time_fmt=$(date -u +"%Y-%m-%dT%H:%M:%SZ")
    echo "[$time_fmt] $1" >> "$LOG_FILE"
}

log_close() {
    local end_time duration
    end_time=$(date +%s)
    duration=$((end_time - start_time))
    log "Backlog completed in ${duration}s"
    echo "" >> "$LOG_FILE"
}

# service controls
services() {
    local action="$1"

    case "$action" in
        stop)
            log "Stopping agave deployment..."
            laconic-so deployment --dir "$DEPLOYMENT_DIR" stop

            log "Waiting for services to fully stop..."
            local deadline=$(( $(date +%s) + SERVICE_STOP_TIMEOUT ))
            while true; do
                local running
                running=$(docker ps --filter "label=com.docker.compose.project.working_dir=$DEPLOYMENT_DIR" -q 2>/dev/null | wc -l)
                if [[ "$running" -eq 0 ]]; then
                    break
                fi
                if (( $(date +%s) >= deadline )); then
                    log "WARNING: Timeout waiting for services to stop; continuing."
                    break
                fi
                sleep 0.2
            done
            ;;
        start)
            log "Starting agave deployment..."
            laconic-so deployment --dir "$DEPLOYMENT_DIR" start
            ;;
        *)
            log "ERROR: Unknown action '$action' in services()"
            exit 2
            ;;
    esac
}

# send a snapshot to one remote
# args: snap remote_host remote_dataset
snapshot_send_one() {
    local snap="$1" remote_host="$2" remote_dataset="$3"

    log "Checking remote snapshots on $remote_host..."

    local -a local_snaps remote_snaps
    mapfile -t local_snaps < <(zfs list -H -t snapshot -o name -s creation -d1 "$DATASET" | grep -F "${DATASET}@${SNAPSHOT_PREFIX}-")
    mapfile -t remote_snaps < <(ssh "$remote_host" zfs list -H -t snapshot -o name -s creation "$remote_dataset" | grep -F "${remote_dataset}@${SNAPSHOT_PREFIX}-" || true)

    # find latest common snapshot
    local base=""
    local local_snap remote_snap remote_check
    for local_snap in "${local_snaps[@]}"; do
        remote_snap="${local_snap/$DATASET/$remote_dataset}"
        for remote_check in "${remote_snaps[@]}"; do
            if [[ "$remote_check" == "$remote_snap" ]]; then
                base="$local_snap"
                break
            fi
        done
    done

    if [[ -z "$base" && ${#remote_snaps[@]} -eq 0 ]]; then
        log "No remote snapshots found on $remote_host — sending full snapshot."
        if zfs send "$snap" | ssh "$remote_host" zfs receive -sF "$remote_dataset"; then
            log "Full send to $remote_host succeeded."
            return 0
        else
            log "ERROR: Full send to $remote_host failed."
            return 1
        fi
    elif [[ -n "$base" ]]; then
        log "Common base snapshot $base found — sending incremental to $remote_host."
        if zfs send -i "$base" "$snap" | ssh "$remote_host" zfs receive -sF "$remote_dataset"; then
            log "Incremental send to $remote_host succeeded."
            return 0
        else
            log "ERROR: Incremental send to $remote_host failed."
            return 1
        fi
    else
        log "STALE DESTINATION: $remote_host has snapshots but no common base with local — skipping."
        return 1
    fi
}

# send snapshot to all remotes
snapshot_send() {
    local snap="$1"
    local failure_count=0

    set +e
    local entry remote_host remote_dataset
    for entry in "${REMOTES[@]}"; do
        remote_host="${entry%%:*}"
        remote_dataset="${entry#*:}"
        if ! snapshot_send_one "$snap" "$remote_host" "$remote_dataset"; then
            failure_count=$((failure_count + 1))
        fi
    done
    set -e

    if [[ "$failure_count" -gt 0 ]]; then
        log "WARNING: $failure_count destination(s) failed or are out of sync."
        return 1
    fi
    return 0
}

# snapshot management
snapshot() {
    local action="$1"

    case "$action" in
        create)
            log "Creating snapshot: $SNAPSHOT"
            zfs snapshot "$SNAPSHOT"
            zfs hold "$ZFS_HOLD" "$SNAPSHOT" || log "ERROR: Failed to hold $SNAPSHOT"
            ;;
        send)
            log "Sending snapshot $SNAPSHOT..."
            if snapshot_send "$SNAPSHOT"; then
                log "Snapshot send completed. Releasing hold."
                zfs release "$ZFS_HOLD" "$SNAPSHOT" || log "ERROR: Failed to release hold on $SNAPSHOT"
            else
                log "WARNING: Snapshot send encountered errors. Hold retained on $SNAPSHOT."
            fi
            ;;
        prune)
            if [[ "$SNAPSHOT_RETENTION" -gt 0 ]]; then
                log "Pruning old snapshots in $DATASET (retaining $SNAPSHOT_RETENTION destroyable snapshots)..."

                local -a all_snaps destroyable
                mapfile -t all_snaps < <(zfs list -H -t snapshot -o name -s creation -d1 "$DATASET" | grep -F "${DATASET}@${SNAPSHOT_PREFIX}-")

                destroyable=()
                for snap in "${all_snaps[@]}"; do
                    if zfs destroy -n -- "$snap" &>/dev/null; then
                        destroyable+=("$snap")
                    else
                        log "Skipping $snap — snapshot not destroyable (likely held)"
                    fi
                done

                local count to_destroy
                count="${#destroyable[@]}"
                to_destroy=$((count - SNAPSHOT_RETENTION))

                if [[ "$to_destroy" -le 0 ]]; then
                    log "Nothing to prune — only $count destroyable snapshots exist"
                else
                    local i
                    for (( i=0; i<to_destroy; i++ )); do
                        snap="${destroyable[$i]}"
                        log "Destroying snapshot: $snap"
                        if ! zfs destroy -- "$snap"; then
                            log "WARNING: Failed to destroy $snap despite earlier check"
                        fi
                    done
                fi
            else
                log "Skipping pruning — retention is set to $SNAPSHOT_RETENTION"
            fi
            ;;
        *)
            log "ERROR: Snapshot unknown action: $action"
            exit 2
            ;;
    esac
}

# open logging and begin execution
mkdir -p "$(dirname -- "$LOG_FILE")"

start_time=$(date +%s)
exec >> "$LOG_FILE" 2>&1
trap 'log_close' EXIT
trap 'rc=$?; log "ERROR: command failed at line $LINENO (exit $rc)"; exit $rc' ERR

log "Backlog Started"

if zfs list -H -t snapshot -o name -d1 "$DATASET" | grep -qxF "$SNAPSHOT"; then
    log "WARNING: Snapshot $SNAPSHOT already exists. Exiting."
    exit 1
fi

services stop
snapshot create
services start
snapshot send
snapshot prune

# end