Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
29 changes: 28 additions & 1 deletion output_writer.py
Original file line number Diff line number Diff line change
Expand Up @@ -308,6 +308,33 @@ def write_csv(rows: Sequence[Any], path: str, row_cls: Type = Product) -> None:
EXIT_PARTIAL = 6


# Exit code for a run that never GOT its pages: a navigation timeout, a dead
# or unauthenticated proxy, a DNS failure, or an edge answering with
# something that is not the page that was asked for.
#
# Distinct from EXIT_NO_PRODUCTS because those are opposite facts. Exit 4 is
# a statement about the CATALOGUE — "we asked, and the answer was nothing" —
# so handing it to a run that never reached the site tells a pipeline the
# listing is empty when nothing was read at all.
#
# 5 rather than a new number, and 5 rather than EXIT_PARTIAL:
#
# * this family's contract already reserves 5 for a transport failure
# (scraper_api_client has used it for a remote API error since it was
# written), so this needs no new code and no per-repo table for a caller
# driving more than one of these scrapers;
# * EXIT_PARTIAL (6) means "some rows were gathered and the output is
# incomplete". A run holding nothing writes no output at all, so a
# consumer that reads the file on a 6 finds either nothing or the
# PREVIOUS run's good data, which `save` deliberately does not
# overwrite. Exit 5 promises no file.
#
# Deliberately NOT applied when rows WERE gathered: a timeout on page 7 of
# 10 is a partial run (exit 6, output written), which is already right. This
# decides only what a run holding nothing reports.
EXIT_FETCH_FAILED = 5


def write_run_meta(out_prefix: str, meta: dict) -> str:
"""Write a run-metadata sidecar next to the output, return its path.

Expand Down Expand Up @@ -504,7 +531,7 @@ def finish_run(rows: Sequence[Any], out_prefix: str, fmt: str,
print(f"[!] Failed run: 0 of {pages_requested} page(s) were "
f"fetched ({stop_reason}). This is NOT an empty result — "
f"nothing was read from the site at all.")
return EXIT_PARTIAL
return EXIT_FETCH_FAILED
return rc
if not complete:
print(f"[!] Partial run: stopped after {pages_completed} of "
Expand Down
10 changes: 5 additions & 5 deletions smoke_test.py
Original file line number Diff line number Diff line change
Expand Up @@ -73,7 +73,7 @@
import page_flow
import product_parser
from diff_runs import diff_products
from output_writer import (Post, Product, save, finish_run, write_csv,
from output_writer import (EXIT_FETCH_FAILED, Post, Product, save, finish_run, write_csv,
run_meta, dedupe_by_key, dedupe_by_sku,
ROW_CLASS_BY_MODE, UNIQUE_BY_SKU_MODES,
COMPLETE_STOP_REASONS, EXIT_BLOCKED,
Expand Down Expand Up @@ -1275,13 +1275,13 @@ def run(rows, stop_reason, blocked=False, allow_empty=False):
# pipeline branching on the exit code, which is what this family
# says exit codes are for, would have recorded an empty catalogue.
code, meta = run([], "page_load_timeout", allow_empty=True)
ok &= check("0 rows because nothing was FETCHED is exit 6, not 4",
code == EXIT_PARTIAL)
ok &= check("0 rows because nothing was FETCHED is exit 5, not 4",
code == EXIT_FETCH_FAILED)
ok &= check("and the sidecar says failed, not complete",
meta is not None and meta["status"] == "failed")
code, meta = run([], "next_batch_refused", allow_empty=True)
ok &= check("a refused batch with no rows is exit 6 too",
code == EXIT_PARTIAL)
ok &= check("a refused batch with no rows is exit 5 too",
code == EXIT_FETCH_FAILED)
code, meta = run([], "blocked_cloudflare", blocked=True, allow_empty=True)
ok &= check("but a BLOCK still outranks both, at exit 3",
code == EXIT_BLOCKED)
Expand Down
Loading