From 601f520a457cb83fc39c56082afbefac382a03bb Mon Sep 17 00:00:00 2001 From: "A. F. Dudley" Date: Mon, 9 Mar 2026 06:11:19 +0000 Subject: [PATCH] fix: add 30-min wall-clock timeout to incremental convergence loop Without a bound, the loop runs forever if sources never serve an incremental close enough to head (e.g. full snapshot base slot is too old). After 30 minutes, proceed with the best incremental available or none. Co-Authored-By: Claude Opus 4.6 --- scripts/agave-container/snapshot_download.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/scripts/agave-container/snapshot_download.py b/scripts/agave-container/snapshot_download.py index 0abaa02a..151b2f26 100644 --- a/scripts/agave-container/snapshot_download.py +++ b/scripts/agave-container/snapshot_download.py @@ -601,8 +601,18 @@ def download_best_snapshot( log.info("Rolling incremental download (base slot %d, convergence %d slots)...", full_snap_slot, convergence_slots) prev_inc_filename: str | None = None + loop_start: float = time.monotonic() + max_convergence_time: float = 1800.0 # 30 min wall-clock limit while True: + if time.monotonic() - loop_start > max_convergence_time: + if prev_inc_filename: + log.warning("Convergence timeout (%.0fs) — using %s", + max_convergence_time, prev_inc_filename) + else: + log.warning("Convergence timeout (%.0fs) — no incremental downloaded", + max_convergence_time) + break inc_fn, inc_mirrors = probe_incremental(fast_sources, full_snap_slot) if inc_fn is None: if prev_inc_filename is None: