diff --git a/.github/workflows/arxiv_refs.yml b/.github/workflows/arxiv_refs.yml
index e561eb8..063dbdc 100644
--- a/.github/workflows/arxiv_refs.yml
+++ b/.github/workflows/arxiv_refs.yml
@@ -16,6 +16,13 @@ name: arXiv refs backfill
# historical backlog over a few nights. Titles arXiv genuinely does not have are
# re-attempted each run — the deliberate cost of keeping no per-line failure
# state, and bounded by the same cap.
+#
+# An arXiv *outage* is not a non-match. The script retries 429/406/5xx along a
+# back-off ladder honouring Retry-After; a lookup that still gets no answer is
+# counted as an API failure, stops the run, is never NOTE-marked, and exits
+# non-zero. This job then commits whatever resolved before the failure and
+# goes red at the end — it ran green through every night of the 2026-09-14..18
+# outage because a bare `except` turned each 429 into "no unique match".
on:
schedule:
@@ -66,7 +73,7 @@ jobs:
DRY_RUN: ${{ inputs.dry_run }}
MARK_UNRESOLVED: ${{ inputs.mark_unresolved }}
run: |
- set -u
+ set -u -o pipefail
args="--limit ${LIMIT:-60}"
[ "${DRY_RUN:-false}" = "true" ] || args="$args --write"
# Never on the schedule: `inputs.*` is empty on a cron fire, so this
@@ -74,9 +81,20 @@ jobs:
[ "${MARK_UNRESOLVED:-false}" != "true" ] || args="$args --mark-unresolved"
# `-u`: unbuffered, so the log streams while the run is in flight
# instead of arriving in one block at the end.
- python -u scripts/backfill_arxiv_refs.py $args | tee /tmp/backfill.log
+ # The exit status is captured, not acted on here: a transport failure
+ # must still let the refs resolved before it be validated and
+ # committed. The last step turns it into a red job.
+ status=0
+ python -u scripts/backfill_arxiv_refs.py $args | tee /tmp/backfill.log || status=$?
+ echo "status=${status}" >> "$GITHUB_OUTPUT"
{
echo '## arXiv refs backfill'
+ if [ "$status" -ne 0 ]; then
+ echo
+ echo "> **arXiv API failure** — a lookup got no answer after the retry ladder (exit ${status})."
+ echo "> The run stopped there; nothing was NOTE-marked on a failed lookup."
+ echo
+ fi
echo '```'
cat /tmp/backfill.log
echo '```'
@@ -117,3 +135,13 @@ jobs:
- name: Refresh the dashboard
if: steps.backfill.outputs.changed == 'yes' && github.ref == 'refs/heads/main'
run: gh workflow run knowledge_board.yml --ref main
+
+ # Last, so partial progress is committed first: an arXiv outage turns the
+ # job red instead of reporting "N ref-less, 0 resolved" in green.
+ - name: Fail on arXiv API failure
+ if: steps.backfill.outputs.status != '0'
+ env:
+ STATUS: ${{ steps.backfill.outputs.status }}
+ run: |
+ echo "::error::arXiv refs backfill exited ${STATUS} — arXiv API transport failure (see the step summary)"
+ exit 1
diff --git a/scripts/arxiv_refs.py b/scripts/arxiv_refs.py
index 02a781a..ce1dbf1 100644
--- a/scripts/arxiv_refs.py
+++ b/scripts/arxiv_refs.py
@@ -31,6 +31,7 @@
from __future__ import annotations
+import http.client
import re
import subprocess
import sys
@@ -69,12 +70,22 @@
ACCEPT = "application/atom+xml"
RETRY_DELAYS = (5, 15, 45, 90, 180)
RETRY_STATUSES = {406, 429, 500, 502, 503, 504}
-TRANSIENT = (urllib.error.URLError, TimeoutError, ConnectionError)
+#: Failures of the *transport*, not answers from arXiv. ``HTTPError`` is a
+#: ``URLError`` subclass but is handled first, by status; ``HTTPException``
+#: covers a connection dropped mid-body (``IncompleteRead``).
+TRANSIENT = (urllib.error.URLError, TimeoutError, ConnectionError,
+ http.client.HTTPException)
_CURL_STATUS_MARKER = b"\n__PYAUTO_HTTP_STATUS__:"
class ArxivAPIError(RuntimeError):
- """The arXiv API could not answer a lookup reliably."""
+ """The arXiv API could not answer a lookup reliably.
+
+ Distinct from a ``None`` out of :func:`resolve_title`, which means arXiv
+ *did* answer and no unique title matched. Callers must never treat this as
+ a negative identification — in particular ``--mark-unresolved`` must not
+ NOTE-mark a line on the strength of it.
+ """
def arxiv_id(ref: str | None) -> str | None:
@@ -232,13 +243,21 @@ def _request_once(url: str) -> bytes:
return _curl_get(url)
-def _api_query(params: dict, *, delays: tuple | None = None) -> bytes:
+def _api_query(params: dict, *, delays: tuple | None = None,
+ sleep=time.sleep) -> bytes:
"""Query arXiv with retry/backoff and make transport failure explicit.
September 2026 exposed two independent failure modes: arXiv can throttle
shared runners with 429/406, and Fastly can reject Python's stdlib client
fingerprint with a persistent 406 while curl succeeds. The latter gets one
immediate alternate-transport attempt; real throttling still backs off.
+
+ The ladder is the same shape as PyAutoMind's ``arxiv_fetch.py:_get()``
+ (PyAutoMind#410), copied rather than imported: 429/406/5xx and transient
+ transport errors back off along `delays`, honouring ``Retry-After`` capped
+ at the ladder's longest step. 406 is arXiv's edge mitigation, not a
+ content-negotiation verdict, so it is retried like a 429. `sleep` is
+ injectable so the ladder can be tested without waiting it out.
"""
if delays is None:
delays = RETRY_DELAYS
@@ -269,7 +288,7 @@ def _api_query(params: dict, *, delays: tuple | None = None) -> bytes:
f"retrying in {wait}s",
file=sys.stderr,
)
- time.sleep(wait)
+ sleep(wait)
raise ArxivAPIError(
f"arXiv API unavailable after {attempts} attempt(s): {last_error}"
@@ -344,6 +363,12 @@ def resolve_title(title: str, max_results: int = 50,
query: object = _api_query) -> str | None:
"""Look `title` up on the arXiv API and return its id, or None.
+ ``None`` means exactly one thing: arXiv answered, and no unique title
+ matched. A lookup that never got a usable answer — HTTP error, DNS/network
+ failure, timeout, an unparseable body — raises :class:`ArxivAPIError`
+ instead, so the backfill can count it as an API failure rather than as
+ "no unique match".
+
Networked — the Dashboard never calls this; only the backfill does.
`query` is injected so the matching can be tested without arXiv.
@@ -360,10 +385,15 @@ def resolve_title(title: str, max_results: int = 50,
"start": 0, "max_results": max_results})
except ArxivAPIError:
raise
- except (urllib.error.URLError, TimeoutError, ConnectionError, OSError) as e:
+ except (*TRANSIENT, OSError) as e:
# Injected query functions in tests and alternate callers may not use
# _api_query(), so normalise their transport failures to the same
# explicit signal. None remains reserved for a real arXiv answer that
# contains no unique exact title match.
raise ArxivAPIError(f"arXiv API lookup failed: {e}") from e
- return match_entries(title, parse_api_entries(raw))
+ try:
+ entries = parse_api_entries(raw)
+ except ET.ParseError as e:
+ # An edge error page or a truncated body is not an answer either.
+ raise ArxivAPIError(f"arXiv API returned an unparseable body: {e}") from e
+ return match_entries(title, entries)
diff --git a/scripts/backfill_arxiv_refs.py b/scripts/backfill_arxiv_refs.py
index a666dd5..2364c4d 100644
--- a/scripts/backfill_arxiv_refs.py
+++ b/scripts/backfill_arxiv_refs.py
@@ -19,6 +19,14 @@
re-attempted each run; that is the cost of holding no per-line failure state,
and it is bounded by ``--limit``.
+A *transport* failure is a different thing from a non-match and is reported as
+one. ``arxiv_refs`` retries 429/406/5xx and network errors along a back-off
+ladder honouring ``Retry-After``; a lookup that still gets no answer raises
+``ArxivAPIError``, which this script counts as ``api_failed`` (never as
+``unmatched``), stops the run on, never NOTE-marks, and turns into a non-zero
+exit — so an arXiv outage goes red instead of logging "N ref-less, 0 resolved"
+(2026-09-14..18, when this job ran green every night through 429s and 406s).
+
Run by ``.github/workflows/arxiv_refs.yml`` (a checkout has no arXiv access
inside a sandboxed session; CI does). Locally:
@@ -118,12 +126,21 @@ def backfill(root: Path, *, write: bool, limit: int, mark_unresolved: bool = Fal
sleep=time.sleep, log=print) -> dict:
"""Resolve up to `limit` ref-less titles across the queue and the inbox.
- With `mark_unresolved`, a line that fails its lookup is also `NOTE `-marked
- so no later run asks about it again — a human-driven cleanup, never the
- nightly's own decision (see :func:`mark_unresolved_line`).
+ With `mark_unresolved`, a line that arXiv answered for but could not
+ uniquely identify is also `NOTE `-marked so no later run asks about it
+ again — a human-driven cleanup, never the nightly's own decision (see
+ :func:`mark_unresolved_line`). A line whose lookup failed on *transport*
+ is never marked: a NOTE-mark is permanent and must only ever follow an
+ actual answer from arXiv.
+
+ Stats: ``matched``/``unmatched`` count answers; ``api_failed`` counts
+ lookups that got no answer after the retry ladder. The first such failure
+ stops the run (every later lookup would hit the same dead endpoint), so
+ ``api_failed`` is 0 or 1 and ``unattempted`` counts the titles left behind
+ in that file within `limit` (later files are not opened).
"""
stats = {"scanned": 0, "matched": 0, "unmatched": 0, "api_failed": 0,
- "marked": 0, "files": []}
+ "unattempted": 0, "marked": 0, "files": []}
budget = limit
api_down = False
for name in TARGETS:
@@ -134,7 +151,7 @@ def backfill(root: Path, *, write: bool, limit: int, mark_unresolved: bool = Fal
todo = candidates("\n".join(lines), name)
stats["scanned"] += len(todo)
changed = file_marked = 0
- for idx, title in todo:
+ for n, (idx, title) in enumerate(todo):
if budget <= 0:
break
budget -= 1
@@ -146,8 +163,9 @@ def backfill(root: Path, *, write: bool, limit: int, mark_unresolved: bool = Fal
# --mark-unresolved must never turn an outage into permanent
# NOTE markers.
stats["api_failed"] += 1
+ stats["unattempted"] += min(len(todo) - n - 1, budget)
api_down = True
- log(f" ! arXiv API failure {title[:90]} ({e})")
+ log(f" ! arXiv API failure {title[:90]} ({e})")
break
if ref:
lines[idx] = apply_ref(lines[idx], ref)
@@ -173,7 +191,8 @@ def backfill(root: Path, *, write: bool, limit: int, mark_unresolved: bool = Fal
if write:
path.write_text("\n".join(lines) + "\n")
marked = f", {file_marked} NOTE-marked" if file_marked else ""
- failed = f", {stats['api_failed']} API failure" if stats["api_failed"] else ""
+ failed = (f", {stats['api_failed']} API failure — run stopped"
+ if api_down else "")
log(f"{name}: {len(todo)} ref-less, {changed - file_marked} resolved"
f"{marked}{failed}{'' if write else ' (dry run — nothing written)'}")
if api_down:
@@ -199,12 +218,20 @@ def main(argv: list[str] | None = None) -> int:
stats = backfill(args.root, write=args.write, limit=args.limit,
mark_unresolved=args.mark_unresolved)
marked = f", {stats['marked']} NOTE-marked" if stats["marked"] else ""
- failed = f", {stats['api_failed']} API failure" if stats["api_failed"] else ""
+ failed = (f", {stats['api_failed']} arXiv API failure "
+ f"({stats.get('unattempted', 0)} more not attempted)"
+ if stats["api_failed"] else "")
print(f"\n{stats['matched']} resolved, {stats['unmatched']} unresolved"
f"{marked}{failed}, {stats['scanned']} ref-less lines in total.")
+ if stats["api_failed"]:
+ print("ARXIV API FAILURE: a lookup got no answer after the retry ladder; "
+ "the run stopped there. Unresolved counts above are answers only "
+ "— nothing was NOTE-marked on a failed lookup.")
# A genuine no-match is normal and stays exit 0. A transport failure is a
- # broken run: after _api_query has already exhausted its retry ladder we
- # stop rather than misreporting the whole queue as unresolved.
+ # broken run: by the time it surfaces, _api_query has spent its whole retry
+ # ladder and the run has stopped, so transport failure is the outcome for
+ # every lookup after it — the dominant one. Exit non-zero so the job goes
+ # red; the workflow still commits whatever resolved before the failure.
return 1 if stats["api_failed"] else 0
diff --git a/tests/test_arxiv_refs.py b/tests/test_arxiv_refs.py
index e8b8cd6..37eed83 100644
--- a/tests/test_arxiv_refs.py
+++ b/tests/test_arxiv_refs.py
@@ -219,6 +219,139 @@ def rejected(*args, **kwargs):
assert len(seen) == 1
+def test_witness_a_429_is_an_api_failure_not_a_non_match(tmp_path):
+ """Witness for the 2026-09-14..18 outage: an arXiv 429 used to come back
+ from resolve_title() as None — "no unique match" — so the backfill logged a
+ cheerful non-match, exited 0, and --mark-unresolved would have NOTE-marked
+ the line for good. It must surface as a transport failure instead."""
+ def throttled(params):
+ raise urllib.error.HTTPError(
+ arxiv_refs.API_URL, 429, "Too Many Requests",
+ {"Retry-After": "30"}, None)
+
+ with pytest.raises(arxiv_refs.ArxivAPIError):
+ arxiv_refs.resolve_title("A Lensed Quasar Survey", query=throttled)
+
+ # …and through the backfill: counted as api_failed, never unmatched,
+ # never NOTE-marked, and the run exits non-zero.
+ root = _tree(tmp_path)
+ before = (root / "reading-queue.md").read_text()
+
+ def resolve(title):
+ return arxiv_refs.resolve_title(title, query=throttled)
+
+ stats = backfill.backfill(root, write=True, limit=10,
+ mark_unresolved=True, resolve=resolve,
+ sleep=lambda _: None, log=lambda *a: None)
+ assert stats["api_failed"] == 1
+ assert stats["unmatched"] == 0 and stats["marked"] == 0
+ assert (root / "reading-queue.md").read_text() == before
+
+
+def test_main_exits_nonzero_on_a_real_429_run(monkeypatch, tmp_path, capsys):
+ root = _tree(tmp_path)
+
+ def throttled(params):
+ raise _http_error(429)
+
+ real = backfill.backfill
+ monkeypatch.setattr(
+ backfill, "backfill",
+ lambda *a, **kw: real(
+ *a, resolve=lambda t: arxiv_refs.resolve_title(t, query=throttled),
+ sleep=lambda _: None, **kw))
+ assert backfill.main(["--root", str(root), "--write"]) == 1
+ out = capsys.readouterr().out
+ assert "ARXIV API FAILURE" in out
+ assert "0 unresolved, 1 arXiv API failure (1 more not attempted)" in out
+
+
+def _http_error(code, retry_after=None):
+ headers = {"Retry-After": retry_after} if retry_after else {}
+ return urllib.error.HTTPError(arxiv_refs.API_URL, code, "x", headers, None)
+
+
+def _scripted(monkeypatch, *outcomes):
+ """Replace the single HTTP attempt with a script of errors/bodies."""
+ queue = list(outcomes)
+ calls = []
+
+ def once(url):
+ calls.append(url)
+ out = queue.pop(0) if len(queue) > 1 else queue[0]
+ if isinstance(out, Exception):
+ raise out
+ return out
+
+ monkeypatch.setattr(arxiv_refs, "_request_once", once)
+ return calls
+
+
+def test_api_query_backs_off_on_429_and_honours_retry_after(monkeypatch):
+ """The retry ladder, without sleeping through it: `sleep` is injected."""
+ _scripted(monkeypatch, _http_error(429, "30"), _http_error(503), b"")
+ slept = []
+ assert arxiv_refs._api_query({"search_query": "all:x"},
+ delays=(5, 15, 45),
+ sleep=slept.append) == b""
+ # Retry-After 30 beats the 5 s step (capped at the ladder's longest, 45);
+ # the 503 carries none and takes the second step.
+ assert slept == [30, 15]
+
+
+def test_api_query_retries_406_as_edge_mitigation(monkeypatch):
+ # _request_once already tried curl for this 406; it is still throttling.
+ _scripted(monkeypatch, _http_error(406), b"")
+ slept = []
+ assert arxiv_refs._api_query({}, delays=(5,), sleep=slept.append) == \
+ b""
+ assert slept == [5]
+
+
+def test_api_query_caps_a_hostile_retry_after(monkeypatch):
+ _scripted(monkeypatch, _http_error(429, "99999"), b"")
+ slept = []
+ arxiv_refs._api_query({}, delays=(5, 15), sleep=slept.append)
+ assert slept == [15]
+
+
+def test_api_query_raises_api_error_once_the_ladder_is_spent(monkeypatch):
+ calls = _scripted(monkeypatch, _http_error(429))
+ slept = []
+ with pytest.raises(arxiv_refs.ArxivAPIError):
+ arxiv_refs._api_query({}, delays=(1, 2, 3), sleep=slept.append)
+ assert len(calls) == 4 and slept == [1, 2, 3]
+
+
+def test_api_query_does_not_retry_a_non_transient_status(monkeypatch):
+ calls = _scripted(monkeypatch, _http_error(400))
+ slept = []
+ with pytest.raises(arxiv_refs.ArxivAPIError):
+ arxiv_refs._api_query({}, delays=(1, 2), sleep=slept.append)
+ assert len(calls) == 1 and slept == []
+
+
+def test_api_query_retries_network_errors(monkeypatch):
+ _scripted(monkeypatch, urllib.error.URLError("dns"), TimeoutError("slow"),
+ b"")
+ slept = []
+ assert arxiv_refs._api_query({}, delays=(1, 2), sleep=slept.append) == \
+ b""
+ assert slept == [1, 2]
+
+
+def test_an_unparseable_body_is_an_api_failure_not_a_non_match():
+ with pytest.raises(arxiv_refs.ArxivAPIError):
+ arxiv_refs.resolve_title("A Lensed Quasar Survey",
+ query=lambda p: b"edge error")
+
+
+def test_none_still_means_arxiv_answered_with_no_unique_match():
+ assert arxiv_refs.resolve_title(
+ "A Lensed Quasar Survey",
+ query=lambda p: _atom(("2608.1", "Something Else"))) is None
+
+
# --- the backfill -------------------------------------------------------------
QUEUE = ("# Reading queue\n\nprose that is not a paper\n\n---\n\n"
"## Strong Lensing\n"