fix: results index skips campaign task records; batch wait loop keeps the fit's exit status

autofit/campaign/scheduler/slurm.py

@@ -107,9 +107,12 @@ def sbatch_script(request: SubmitRequest) -> str:
         "child=$!",
         "trap 'kill -USR1 \"$child\" 2>/dev/null || true' USR1",
         "trap 'kill -TERM \"$child\" 2>/dev/null || true' TERM",
-        "status=0",
-        'until wait "$child"; do',
-        "    status=$?",
+        # A trapped signal ends `wait` early (128 + signal); wait again until the fit
+        # itself has exited and keep only its own status, so a fit that checkpoints on
+        # USR1 and then finishes cleanly exits 0, not 138.
+        "while true; do",
+        "    status=0",
+        '    wait "$child" || status=$?',
         '    kill -0 "$child" 2>/dev/null || break',
         "done",
         'exit "$status"',

autofit/results/index.py

@@ -43,7 +43,13 @@
 reruns of that fit with another model or search setting (sibling hashes).
 """
 
-_SKIP_DIRS = {"files", "image", INDEX_DIRNAME}
+#: Batch-side task records (``<campaign>/<stage>/<index>.status.json``) a campaign's
+#: stage entry writes before its fit starts, so a task that fails before Python runs is
+#: still counted. They are the campaign collector's (``autofit.campaign.collector``
+#: walks for them itself), never results.
+CAMPAIGN_TASKS_DIRNAME = ".campaign_tasks"
+
+_SKIP_DIRS = {"files", "image", INDEX_DIRNAME, CAMPAIGN_TASKS_DIRNAME}
 
 _STATUS_SUFFIX = str(Product.STATUS)
 _ZIP_SUFFIX = str(Product.ZIP)

test_autofit/campaign/test_slurm.py

@@ -78,6 +78,69 @@ def test_sbatch_script():
     assert "--gres" not in script
 
 
+def _run_generated_script(tmp_path, fit_body, signal_usr1):
+    """Run the generated batch script under bash, with a stand-in fit."""
+    import os
+    import shutil
+    import signal
+    import time
+
+    if shutil.which("bash") is None:
+        pytest.skip("needs bash")
+    ready = tmp_path / "ready"
+    fit = tmp_path / "fit.py"
+    fit.write_text(
+        "import os, signal, sys, time\n"
+        "got = []\n"
+        "signal.signal(signal.SIGUSR1, lambda *_: got.append(1))\n"
+        f"open({str(ready)!r}, 'w').close()\n" + fit_body
+    )
+    request = _request(cpus=1)
+    request.script = str(fit)
+    request.project_root = str(tmp_path)
+    request.env = {}
+    script = tmp_path / "job.sh"
+    script.write_text(sbatch_script(request))
+    # The script runs `python <script>`; point that name at this interpreter.
+    bin_dir = tmp_path / "bin"
+    bin_dir.mkdir()
+    (bin_dir / "python").symlink_to(sys.executable)
+    process = subprocess.Popen(
+        ["bash", str(script)],
+        env={**os.environ, "SLURM_ARRAY_TASK_ID": "0", "PATH": f"{bin_dir}:{os.environ['PATH']}"},
+        stdout=subprocess.PIPE,
+        stderr=subprocess.PIPE,
+    )
+    try:
+        deadline = time.time() + 30
+        while not ready.exists() and time.time() < deadline:
+            time.sleep(0.05)
+        assert ready.exists(), "the stand-in fit never started"
+        if signal_usr1:
+            process.send_signal(signal.SIGUSR1)
+        process.communicate(timeout=60)
+    finally:
+        if process.poll() is None:
+            process.kill()
+            process.communicate()
+    return process.returncode
+
+
+def test_sbatch_script_fit_that_checkpoints_on_usr1_then_finishes_exits_0(tmp_path):
+    """The pre-timeout USR1 interrupts the batch shell's wait; the fit's own exit 0 must win."""
+    body = (
+        "while not got:\n"
+        "    time.sleep(0.05)\n"
+        "time.sleep(0.2)  # the checkpoint flush, after the first wait returned\n"
+        "sys.exit(0)\n"
+    )
+    assert _run_generated_script(tmp_path, body, signal_usr1=True) == 0
+
+
+def test_sbatch_script_exits_with_the_fits_failure(tmp_path):
+    assert _run_generated_script(tmp_path, "sys.exit(3)\n", signal_usr1=False) == 3
+
+
 def test_sbatch_script_passes_its_own_lint(ral):
     from autofit.campaign.lint import check_sbatch
 

test_autofit/results/test_index.py

@@ -131,6 +131,19 @@ def test_nested_and_custom_layouts_are_found(results_root):
     assert all("files" not in Path(p).parts for p in found)
 
 
+def test_campaign_task_records_are_not_results(results_root):
+    write_zip(results_root / "lens_1", prefix="fit")
+    tasks = results_root / ".campaign_tasks" / "demo" / "source_lp"
+    tasks.mkdir(parents=True)
+    write_status(tasks / "0", "failed", failure_class="bad_input")
+    write_status(tasks / "1", "pending")
+
+    found = list(find_results(results_root))
+
+    assert len(found) == 1
+    assert ".campaign_tasks" not in found[0].parts
+
+
 def test_reindex_only_opens_new_and_changed_results(results_root, monkeypatch):
     for seed in range(30):
         write_zip(results_root, prefix=f"dataset_{seed}", seed=seed)