From 06a414851011ad5fa1416c2424ba19fed1f5d95f Mon Sep 17 00:00:00 2001
From: Jammy2211 <JNightingale2211@gmail.com>
Date: Sat, 31 Oct 2026 12:12:16 +0000
Subject: [PATCH 1/2] fix: results index skips campaign task records; batch
 wait loop keeps the fit's exit status

- autofit.results index walked into output/.campaign_tasks/, so the
  pending/failed task records a campaign's stage entry writes before its
  fit starts showed up as phantom results. The directory is skipped (the
  campaign collector walks for them itself).
- The generated Slurm batch script's 'until wait' loop kept the 138 from a
  USR1-interrupted wait, so a fit that checkpointed on the pre-timeout
  signal and then finished cleanly exited 138. It now keeps only the fit's
  own status. The new test drives a real USR1 and fails on the old loop.

Fixes #1749.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
---
 autofit/campaign/scheduler/slurm.py |  9 +++--
 autofit/results/index.py            |  8 +++-
 test_autofit/campaign/test_slurm.py | 57 +++++++++++++++++++++++++++++
 test_autofit/results/test_index.py  | 13 +++++++
 4 files changed, 83 insertions(+), 4 deletions(-)

diff --git a/autofit/campaign/scheduler/slurm.py b/autofit/campaign/scheduler/slurm.py
index 40b789cca..b4f99f19f 100644
--- a/autofit/campaign/scheduler/slurm.py
+++ b/autofit/campaign/scheduler/slurm.py
@@ -107,9 +107,12 @@ def sbatch_script(request: SubmitRequest) -> str:
         "child=$!",
         "trap 'kill -USR1 \"$child\" 2>/dev/null || true' USR1",
         "trap 'kill -TERM \"$child\" 2>/dev/null || true' TERM",
-        "status=0",
-        'until wait "$child"; do',
-        "    status=$?",
+        # A trapped signal ends `wait` early (128 + signal); wait again until the fit
+        # itself has exited and keep only its own status, so a fit that checkpoints on
+        # USR1 and then finishes cleanly exits 0, not 138.
+        "while true; do",
+        "    status=0",
+        '    wait "$child" || status=$?',
         '    kill -0 "$child" 2>/dev/null || break',
         "done",
         'exit "$status"',
diff --git a/autofit/results/index.py b/autofit/results/index.py
index b96a6e381..ff1e2fa39 100644
--- a/autofit/results/index.py
+++ b/autofit/results/index.py
@@ -43,7 +43,13 @@
 reruns of that fit with another model or search setting (sibling hashes).
 """
 
-_SKIP_DIRS = {"files", "image", INDEX_DIRNAME}
+#: Batch-side task records (``<campaign>/<stage>/<index>.status.json``) a campaign's
+#: stage entry writes before its fit starts, so a task that fails before Python runs is
+#: still counted. They are the campaign collector's (``autofit.campaign.collector``
+#: walks for them itself), never results.
+CAMPAIGN_TASKS_DIRNAME = ".campaign_tasks"
+
+_SKIP_DIRS = {"files", "image", INDEX_DIRNAME, CAMPAIGN_TASKS_DIRNAME}
 
 _STATUS_SUFFIX = str(Product.STATUS)
 _ZIP_SUFFIX = str(Product.ZIP)
diff --git a/test_autofit/campaign/test_slurm.py b/test_autofit/campaign/test_slurm.py
index 7e59aed4e..329dbc541 100644
--- a/test_autofit/campaign/test_slurm.py
+++ b/test_autofit/campaign/test_slurm.py
@@ -78,6 +78,63 @@ def test_sbatch_script():
     assert "--gres" not in script
 
 
+def _run_generated_script(tmp_path, fit_body, signal_usr1):
+    """Run the generated batch script under bash, with a stand-in fit."""
+    import os
+    import shutil
+    import signal
+    import time
+
+    if shutil.which("bash") is None:
+        pytest.skip("needs bash")
+    ready = tmp_path / "ready"
+    fit = tmp_path / "fit.py"
+    fit.write_text(
+        "import os, signal, sys, time\n"
+        "got = []\n"
+        "signal.signal(signal.SIGUSR1, lambda *_: got.append(1))\n"
+        f"open({str(ready)!r}, 'w').close()\n" + fit_body
+    )
+    request = _request(cpus=1)
+    request.script = str(fit)
+    request.project_root = str(tmp_path)
+    request.env = {}
+    script = tmp_path / "job.sh"
+    script.write_text(sbatch_script(request))
+    # The script runs `python <script>`; point that name at this interpreter.
+    bin_dir = tmp_path / "bin"
+    bin_dir.mkdir()
+    (bin_dir / "python").symlink_to(sys.executable)
+    process = subprocess.Popen(
+        ["bash", str(script)],
+        env={**os.environ, "SLURM_ARRAY_TASK_ID": "0", "PATH": f"{bin_dir}:{os.environ['PATH']}"},
+        stdout=subprocess.PIPE,
+        stderr=subprocess.PIPE,
+    )
+    deadline = time.time() + 30
+    while not ready.exists() and time.time() < deadline:
+        time.sleep(0.05)
+    if signal_usr1:
+        process.send_signal(signal.SIGUSR1)
+    process.communicate(timeout=60)
+    return process.returncode
+
+
+def test_sbatch_script_fit_that_checkpoints_on_usr1_then_finishes_exits_0(tmp_path):
+    """The pre-timeout USR1 interrupts the batch shell's wait; the fit's own exit 0 must win."""
+    body = (
+        "while not got:\n"
+        "    time.sleep(0.05)\n"
+        "time.sleep(0.2)  # the checkpoint flush, after the first wait returned\n"
+        "sys.exit(0)\n"
+    )
+    assert _run_generated_script(tmp_path, body, signal_usr1=True) == 0
+
+
+def test_sbatch_script_exits_with_the_fits_failure(tmp_path):
+    assert _run_generated_script(tmp_path, "sys.exit(3)\n", signal_usr1=False) == 3
+
+
 def test_sbatch_script_passes_its_own_lint(ral):
     from autofit.campaign.lint import check_sbatch
 
diff --git a/test_autofit/results/test_index.py b/test_autofit/results/test_index.py
index bf87717b3..a5b333384 100644
--- a/test_autofit/results/test_index.py
+++ b/test_autofit/results/test_index.py
@@ -131,6 +131,19 @@ def test_nested_and_custom_layouts_are_found(results_root):
     assert all("files" not in Path(p).parts for p in found)
 
 
+def test_campaign_task_records_are_not_results(results_root):
+    write_zip(results_root / "lens_1", prefix="fit")
+    tasks = results_root / ".campaign_tasks" / "demo" / "source_lp"
+    tasks.mkdir(parents=True)
+    write_status(tasks / "0", "failed", failure_class="bad_input")
+    write_status(tasks / "1", "pending")
+
+    found = list(find_results(results_root))
+
+    assert len(found) == 1
+    assert ".campaign_tasks" not in found[0].parts
+
+
 def test_reindex_only_opens_new_and_changed_results(results_root, monkeypatch):
     for seed in range(30):
         write_zip(results_root, prefix=f"dataset_{seed}", seed=seed)

From 2caeaab3eb0beaa79505288eca9dc1aebd274514 Mon Sep 17 00:00:00 2001
From: Jammy2211 <JNightingale2211@gmail.com>
Date: Sat, 31 Oct 2026 12:13:30 +0000
Subject: [PATCH 2/2] test: the batch-script runner fails fast if the fit never
 starts, and kills on timeout (review nit)

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
---
 test_autofit/campaign/test_slurm.py | 18 ++++++++++++------
 1 file changed, 12 insertions(+), 6 deletions(-)

diff --git a/test_autofit/campaign/test_slurm.py b/test_autofit/campaign/test_slurm.py
index 329dbc541..5afb35838 100644
--- a/test_autofit/campaign/test_slurm.py
+++ b/test_autofit/campaign/test_slurm.py
@@ -111,12 +111,18 @@ def _run_generated_script(tmp_path, fit_body, signal_usr1):
         stdout=subprocess.PIPE,
         stderr=subprocess.PIPE,
     )
-    deadline = time.time() + 30
-    while not ready.exists() and time.time() < deadline:
-        time.sleep(0.05)
-    if signal_usr1:
-        process.send_signal(signal.SIGUSR1)
-    process.communicate(timeout=60)
+    try:
+        deadline = time.time() + 30
+        while not ready.exists() and time.time() < deadline:
+            time.sleep(0.05)
+        assert ready.exists(), "the stand-in fit never started"
+        if signal_usr1:
+            process.send_signal(signal.SIGUSR1)
+        process.communicate(timeout=60)
+    finally:
+        if process.poll() is None:
+            process.kill()
+            process.communicate()
     return process.returncode