From e271be56855027f013fd46b6cb8fe0f21ad9a3a0 Mon Sep 17 00:00:00 2001 From: Ronald Tse Date: Sat, 5 Sep 2026 01:23:21 +0200 Subject: [PATCH] fix(distill): evaluate timeout 2h -> 5h The s46-era Hebrew eval decodes a ByT5-base teacher plus the student over 1,864 long sentences; the 2h ceiling killed full-set evals mid-run and the watchdog restarted from scratch each time. --- src/gpu/modal_distill.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/gpu/modal_distill.py b/src/gpu/modal_distill.py index ddf5f9e..6024e8b 100644 --- a/src/gpu/modal_distill.py +++ b/src/gpu/modal_distill.py @@ -485,7 +485,10 @@ def _edit_distance(a, b) -> int: gpu="A10G", cpu=8, memory=32 * 1024, - timeout=2 * 3600, + # s46-era eval decodes teacher (ByT5-base) + student over 1,864 long + # sentences; measured >2h under A10G contention — the 2h ceiling + # killed full-set evals mid-run + timeout=5 * 3600, volumes={"/datasets": DATASETS, "/checkpoints": CHECKPOINTS}, ) def evaluate(spec_id: str = "heb-diac-small", limit: int = 0) -> dict: