From f13e99f533f48a0a31d45ff8f36ec4caba0dc08e Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 21 Sep 2026 21:26:04 +0000 Subject: [PATCH] Report exit 5, not 4, when nothing was gathered and the run did not finish MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Exit 4 is a claim about the CATALOGUE — "we asked, and the answer was nothing" — so handing it to a run that never reached the site tells a pipeline the listing is empty when nothing was read at all. Measured across the family on 2026-09-21 by CALLING each repo's finish_run rather than grepping for the fix: 20 repos still returned 4, two returned 5, and seven had already fixed it using EXIT_PARTIAL with written reasoning. Three camps, none aware of the other two. This unifies on 5, taking the rule from the exit-6 camp and the code from the exit-5 camp. The rule is `if not complete` rather than a named list of stop reasons: a list cannot cover a reason nobody has added to it yet, so a new stop_reason would fall silently through to "the catalogue is empty" — the defect all three camps set out to fix. The code is 5 rather than 6 because EXIT_PARTIAL means "some rows were gathered and the output is incomplete", and a run holding nothing writes no output, so a consumer reading the file on a 6 finds either nothing or the PREVIOUS run's good data that save() deliberately leaves in place. 5 promises no file, and the family contract already reserves it for a transport failure. Rows gathered and THEN a failure is still exit 6, unchanged. Verified by invoking finish_run after the patch and requiring 5, with the repo's own suite green. Checks that pinned the old answer were rewritten in the same commit. Co-Authored-By: Claude Opus 5 (1M context) --- output_writer.py | 29 ++++++++++++++++++++++++++++- smoke_test.py | 10 +++++----- 2 files changed, 33 insertions(+), 6 deletions(-) diff --git a/output_writer.py b/output_writer.py index 6055b71..f9c8c18 100644 --- a/output_writer.py +++ b/output_writer.py @@ -308,6 +308,33 @@ def write_csv(rows: Sequence[Any], path: str, row_cls: Type = Product) -> None: EXIT_PARTIAL = 6 +# Exit code for a run that never GOT its pages: a navigation timeout, a dead +# or unauthenticated proxy, a DNS failure, or an edge answering with +# something that is not the page that was asked for. +# +# Distinct from EXIT_NO_PRODUCTS because those are opposite facts. Exit 4 is +# a statement about the CATALOGUE — "we asked, and the answer was nothing" — +# so handing it to a run that never reached the site tells a pipeline the +# listing is empty when nothing was read at all. +# +# 5 rather than a new number, and 5 rather than EXIT_PARTIAL: +# +# * this family's contract already reserves 5 for a transport failure +# (scraper_api_client has used it for a remote API error since it was +# written), so this needs no new code and no per-repo table for a caller +# driving more than one of these scrapers; +# * EXIT_PARTIAL (6) means "some rows were gathered and the output is +# incomplete". A run holding nothing writes no output at all, so a +# consumer that reads the file on a 6 finds either nothing or the +# PREVIOUS run's good data, which `save` deliberately does not +# overwrite. Exit 5 promises no file. +# +# Deliberately NOT applied when rows WERE gathered: a timeout on page 7 of +# 10 is a partial run (exit 6, output written), which is already right. This +# decides only what a run holding nothing reports. +EXIT_FETCH_FAILED = 5 + + def write_run_meta(out_prefix: str, meta: dict) -> str: """Write a run-metadata sidecar next to the output, return its path. @@ -504,7 +531,7 @@ def finish_run(rows: Sequence[Any], out_prefix: str, fmt: str, print(f"[!] Failed run: 0 of {pages_requested} page(s) were " f"fetched ({stop_reason}). This is NOT an empty result — " f"nothing was read from the site at all.") - return EXIT_PARTIAL + return EXIT_FETCH_FAILED return rc if not complete: print(f"[!] Partial run: stopped after {pages_completed} of " diff --git a/smoke_test.py b/smoke_test.py index 057f73a..5be8fba 100644 --- a/smoke_test.py +++ b/smoke_test.py @@ -73,7 +73,7 @@ import page_flow import product_parser from diff_runs import diff_products -from output_writer import (Post, Product, save, finish_run, write_csv, +from output_writer import (EXIT_FETCH_FAILED, Post, Product, save, finish_run, write_csv, run_meta, dedupe_by_key, dedupe_by_sku, ROW_CLASS_BY_MODE, UNIQUE_BY_SKU_MODES, COMPLETE_STOP_REASONS, EXIT_BLOCKED, @@ -1275,13 +1275,13 @@ def run(rows, stop_reason, blocked=False, allow_empty=False): # pipeline branching on the exit code, which is what this family # says exit codes are for, would have recorded an empty catalogue. code, meta = run([], "page_load_timeout", allow_empty=True) - ok &= check("0 rows because nothing was FETCHED is exit 6, not 4", - code == EXIT_PARTIAL) + ok &= check("0 rows because nothing was FETCHED is exit 5, not 4", + code == EXIT_FETCH_FAILED) ok &= check("and the sidecar says failed, not complete", meta is not None and meta["status"] == "failed") code, meta = run([], "next_batch_refused", allow_empty=True) - ok &= check("a refused batch with no rows is exit 6 too", - code == EXIT_PARTIAL) + ok &= check("a refused batch with no rows is exit 5 too", + code == EXIT_FETCH_FAILED) code, meta = run([], "blocked_cloudflare", blocked=True, allow_empty=True) ok &= check("but a BLOCK still outranks both, at exit 3", code == EXIT_BLOCKED)