From d5d1aa4e638c21ff20df76633bd1cde047b9f539 Mon Sep 17 00:00:00 2001 From: Brett Heap <1513478+brettheap@users.noreply.github.com> Date: Sun, 27 Sep 2026 16:05:30 +0000 Subject: [PATCH 01/34] T050: the plain-documents fixture, spread across the six stations (plan 034) tests/fixtures/plain-documents/ carries eight .md documents: three sources (no `stage:` header, two sharing the phrase "rain barrel" so a future topic-based grouping pass has a pair to find) and one document each declaring `stage: grouping`, `stage: candidate`, `stage: selection`, `stage: submission` and `stage: completion` (RULED R1Q13 (a) with (c), openxFactory#656 comment 5850003126). Every document carries the small neutral field set the default adapter will require regardless of station (`title`, `summary`). tests/test_plain_documents_fixture.py is the vocabulary test T050 names as its falsifier: it asserts the fixture carries none of the eight controlled `Status:` words or the change/spec/delta nouns, that it covers all six stations with exactly one document per explicit station, and that at least two (but not all) sources share the topic fixture code will need to group on. Not yet wired into .github/workflows/validate.yml's explicit pytest list: phase2-ahead scope keeps this PR out of that file, the pin files and conftest.py/pyproject.toml/README.md, which the phase-1 chain still edits. This suite runs by node id today. Arc: neutral-product-standalone-operability Lane: openxfactory-4 (openXfactory-4-openDox_extraction) Co-Authored-By: Claude Opus 5.5 (1M context) --- .../candidate-toolshed-rebuild.md | 11 + .../completion-path-resurfacing.md | 11 + .../grouping-compost-corner.md | 12 + .../plain-documents/notes-rain-barrel-leak.md | 9 + .../notes-rain-barrel-overflow.md | 9 + .../notes-toolshed-inventory.md | 10 + .../selection-spring-planting-plan.md | 11 + .../submission-shed-door-repair.md | 11 + tests/test_plain_documents_fixture.py | 219 ++++++++++++++++++ 9 files changed, 303 insertions(+) create mode 100644 tests/fixtures/plain-documents/candidate-toolshed-rebuild.md create mode 100644 tests/fixtures/plain-documents/completion-path-resurfacing.md create mode 100644 tests/fixtures/plain-documents/grouping-compost-corner.md create mode 100644 tests/fixtures/plain-documents/notes-rain-barrel-leak.md create mode 100644 tests/fixtures/plain-documents/notes-rain-barrel-overflow.md create mode 100644 tests/fixtures/plain-documents/notes-toolshed-inventory.md create mode 100644 tests/fixtures/plain-documents/selection-spring-planting-plan.md create mode 100644 tests/fixtures/plain-documents/submission-shed-door-repair.md create mode 100644 tests/test_plain_documents_fixture.py diff --git a/tests/fixtures/plain-documents/candidate-toolshed-rebuild.md b/tests/fixtures/plain-documents/candidate-toolshed-rebuild.md new file mode 100644 index 00000000..67bbd952 --- /dev/null +++ b/tests/fixtures/plain-documents/candidate-toolshed-rebuild.md @@ -0,0 +1,11 @@ +stage: candidate +title: Rebuilding the toolshed door and frame +summary: A short write-up on replacing the toolshed's sagging door frame, put forward as one option worth picking. + +# Rebuilding the toolshed door and frame + +The toolshed door frame has sagged enough that the door no longer latches +without a shove. One option on the table is a full frame rebuild over a +weekend, using pressure-treated lumber and new hinges, rather than another +temporary shim. It would cost a Saturday and a modest supply run, and it +would settle a problem that keeps coming back every few months. diff --git a/tests/fixtures/plain-documents/completion-path-resurfacing.md b/tests/fixtures/plain-documents/completion-path-resurfacing.md new file mode 100644 index 00000000..2fb75dae --- /dev/null +++ b/tests/fixtures/plain-documents/completion-path-resurfacing.md @@ -0,0 +1,11 @@ +stage: completion +title: The gravel path resurfacing is done +summary: A short note closing out the gravel path work: what was laid, how much it took, and what is left over. + +# The gravel path resurfacing is done + +The path from the gate to the shared beds has a fresh layer of gravel now, +about two inches deep over the old base, and the low spot that used to +hold water after storms is filled and level. It took four wheelbarrow +loads more than expected, and the leftover gravel is stacked by the +toolshed for whoever tackles the side path next. diff --git a/tests/fixtures/plain-documents/grouping-compost-corner.md b/tests/fixtures/plain-documents/grouping-compost-corner.md new file mode 100644 index 00000000..bdce97e5 --- /dev/null +++ b/tests/fixtures/plain-documents/grouping-compost-corner.md @@ -0,0 +1,12 @@ +stage: grouping +title: Compost corner ideas, gathered in one place +summary: Three separate notes about turning food scraps into usable soil, pulled into a single set. + +# Compost corner ideas, gathered in one place + +Several notes about a compost corner behind the shared beds turned out to +be circling the same handful of questions: how often to turn the pile, +whether food scraps need to be covered with leaves right away, and where a +second bin would fit without blocking the wheelbarrow path. Gathering them +here makes it easier to see what they have in common before picking one to +work on next. diff --git a/tests/fixtures/plain-documents/notes-rain-barrel-leak.md b/tests/fixtures/plain-documents/notes-rain-barrel-leak.md new file mode 100644 index 00000000..88c6499e --- /dev/null +++ b/tests/fixtures/plain-documents/notes-rain-barrel-leak.md @@ -0,0 +1,9 @@ +title: A slow leak at the rain barrel +summary: The seal under the downspout drips after every storm and the ground underneath stays soft. + +# A slow leak at the rain barrel + +Water pools under the rain barrel near the garden gate after every storm. +The rubber seal where the downspout meets the lid looks cracked on one +side. Someone should check whether a new seal or a different downspout +fitting would keep the ground from staying soft all week. diff --git a/tests/fixtures/plain-documents/notes-rain-barrel-overflow.md b/tests/fixtures/plain-documents/notes-rain-barrel-overflow.md new file mode 100644 index 00000000..f464ddbb --- /dev/null +++ b/tests/fixtures/plain-documents/notes-rain-barrel-overflow.md @@ -0,0 +1,9 @@ +title: The rain barrel overflows before the storm ends +summary: A second barrel or an overflow hose would carry extra water to the garden beds instead of the path. + +# The rain barrel overflows before the storm ends + +The single rain barrel by the garden gate fills within twenty minutes of a +hard storm and then spills straight onto the gravel path. A short hose run +to the garden beds, or a second barrel chained to the first, would put that +extra water to use instead of letting it wash out the path edge. diff --git a/tests/fixtures/plain-documents/notes-toolshed-inventory.md b/tests/fixtures/plain-documents/notes-toolshed-inventory.md new file mode 100644 index 00000000..6eae560b --- /dev/null +++ b/tests/fixtures/plain-documents/notes-toolshed-inventory.md @@ -0,0 +1,10 @@ +title: What is actually in the toolshed +summary: A walk-through count of hand tools, hoses and pots, so nobody buys a third trowel. + +# What is actually in the toolshed + +A slow Saturday walk-through of the toolshed turned up four trowels, two +working wheelbarrows and one with a flat tire, a tangle of hoses that +mostly still hold water, and more clay pots stacked in the back corner than +anyone remembered buying. Worth doing again before the next round of seed +starting. diff --git a/tests/fixtures/plain-documents/selection-spring-planting-plan.md b/tests/fixtures/plain-documents/selection-spring-planting-plan.md new file mode 100644 index 00000000..613088f9 --- /dev/null +++ b/tests/fixtures/plain-documents/selection-spring-planting-plan.md @@ -0,0 +1,11 @@ +stage: selection +title: What we are planting behind the shared beds this spring +summary: The picked list of vegetables and flowers for the shared beds, chosen from several ideas floated over the winter. + +# What we are planting behind the shared beds this spring + +After looking at several ideas floated over the winter, the shared beds +this spring will carry tomatoes, bush beans, a border of marigolds, and a +short row of sunflowers along the back fence. Herbs stay in the raised +boxes near the gate, where they did well last year. This is the picked +list the bed captains are working from until the ground is turned. diff --git a/tests/fixtures/plain-documents/submission-shed-door-repair.md b/tests/fixtures/plain-documents/submission-shed-door-repair.md new file mode 100644 index 00000000..fcde0ac5 --- /dev/null +++ b/tests/fixtures/plain-documents/submission-shed-door-repair.md @@ -0,0 +1,11 @@ +stage: submission +title: Shed door repair, sent in for the work weekend +summary: A ready-to-go write-up of the shed door repair, sent in for the next work weekend's list. + +# Shed door repair, sent in for the work weekend + +This write-up covers the shed door repair end to end: new hinges, a +replacement latch, and enough pressure-treated lumber for the frame pieces +that have started to soften. It is sent in as one item for the next work +weekend's list, with the tools and the rough time it will take already +worked out, so the crew can pick it up without more back and forth first. diff --git a/tests/test_plain_documents_fixture.py b/tests/test_plain_documents_fixture.py new file mode 100644 index 00000000..6a0f23de --- /dev/null +++ b/tests/test_plain_documents_fixture.py @@ -0,0 +1,219 @@ +"""The plain-documents fixture: spread across the six stations, and clean of +the publishing repository's own vocabulary (plan 034, T050). + +`tests/fixtures/plain-documents/` is 5.0's fixture (plan 034 tasks.md § Phase +2, slice P2-F): a handful of `.md` documents a plain, ungoverned git +repository could contain, read by AT-R1 (spec.md § "AT-R1 — the release-1 +acceptance test", step 3(a)) and by F5.3, F7.2, F10.1 and F13.1 once those +falsifiers exist. None of it is wired to any `opendox` code yet — T052 and +T054 (the generator seam and the neutral projection) have not landed — so +this suite tests the fixture's own two guarantees rather than a projection +over it. + +THE TWO GUARANTEES, AND WHY. + +1. NEUTRAL VOCABULARY (T050's task line: *"It carries none of the declared + vocabulary: the eight `Status:` words and the change/spec/delta + nouns."*). A neutral product shipping its own publisher's governance + words in ITS TEST DATA is the same defect requirement 3's second scenario + refuses in RENDERED words and names + (`tests/test_default_profile.py`); this is the shipped-fixture half of + that same property, swept on its own footing because a fixture is prose + no engine renders and the rendering sweep never sees it. +2. SIX STATIONS, ONE GROUP (RULED R1Q13 (a) with (c), `openxFactory#656` + comment `5850003126`; spec.md line ~170, ~454-463). A neutral + front-matter key, `stage: `, names a document's station; a + document that declares no `stage:` line at all is a SOURCE. This fixture + carries one document per explicit station (`grouping`, `candidate`, + `selection`, `submission`, `completion`) and three sources, two of which + share the phrase "rain barrel" verbatim so a future topic-based grouping + pass (T054) has a pair to find — the fixture must yield at least one + group, so AT-R1 can open the chat pane from a grouping tile (spec.md + § AT-R1 steps 6-7). + +Every document also carries the small neutral field set the default adapter +will require regardless of station — `title` and `summary` — per T050's task +line and the answer's own example. + +NOT YET WIRED INTO `.github/workflows/validate.yml`'s explicit pytest list: +the phase-2 draft-ahead scope keeps this PR out of that file (conftest.py, +pyproject.toml, validate.yml and README.md are the phase-1 chain's), so this +suite runs by node id today, exactly as other narrowed-out suites in this +tree have (`tests/test_display_facet_leaves.py`'s own S7-residue history). +It joins the enumerated list whichever later task next touches it — most +likely F5.3/T056, or T049's own close. + +A CREATED file: no row in openxFactory's `docs/opendox-carve-manifest.yaml` +(RULED OQ-C: the manifest declares what LEAVES openxFactory, never what a +destination assembles). +""" + +from __future__ import annotations + +import re +import sys +from pathlib import Path + +import pytest + +ROOT = Path(__file__).resolve().parents[1] +FIXTURE_DIR = ROOT / "tests" / "fixtures" / "plain-documents" + +sys.path.insert(0, str(ROOT / "src")) +from opendox.display_profile import STAGE_ROLES # noqa: E402 + +#: Its STATUS TAXONOMY: the eight controlled `Status:` words this project's +#: own root `CLAUDE.md` names verbatim (*"brainstorm | staged | draft | +#: ratified | standard | superseded | retired | record"*), identical to the +#: eight `tests/test_default_profile.py`'s `STATUS_TAXONOMY`, +#: `tests/test_web_boundary.py`'s `_STATUS_WORDS` and +#: `tests/test_display_facet.py`'s `OPENXFACTORY_WORDS` all open with, at +#: openxFactory `133e37d9`. T050's own task line says "the eight", so this +#: suite does not also carry `test_default_profile.py`'s ninth, +#: `projection` — that word is `openxfactory-engineering.yaml`'s own +#: lifecycle addition, a different sweep than this one. +STATUS_WORDS = ( + "brainstorm", "staged", "draft", "ratified", "standard", + "superseded", "retired", "record", +) + +#: Its CHANGE/SPEC/DELTA NOUNS: the same family `tests/test_default_profile.py` +#: carries as `CHANGE_SPEC_DELTA_NOUNS` — the OpenSpec artifact a governed +#: change is, and the proposal that opens one. Copied rather than imported, +#: on the same footing as `STATUS_WORDS`: this fixture must be readable as +#: plain prose by something that never imports `opendox` at all (the +#: acceptance harness copies it into a bare `git init`), so the word list a +#: reader would check it against has to stand on its own too. +CHANGE_SPEC_DELTA_NOUNS = ( + "change", "changes", "spec", "specs", "delta", "deltas", + "openspec", "proposal", "proposals", +) + +DECLARED_VOCABULARY = frozenset(STATUS_WORDS + CHANGE_SPEC_DELTA_NOUNS) + +_WORD_PATTERN = re.compile( + r"\b(" + "|".join(re.escape(word) for word in DECLARED_VOCABULARY) + r")\b", + re.IGNORECASE, +) + +#: The neutral field set the default adapter will require of every document +#: regardless of station (T050's task line: *"e.g. title, summary"*). +REQUIRED_FIELDS = ("title", "summary") + +#: The phrase at least two SOURCE documents (no `stage:` header) share +#: verbatim, so a topic-based grouping pass has a pair to find (R1Q13 (a) +#: with (c)). +SHARED_SOURCE_TOPIC = "rain barrel" + +FIXTURE_FILES = sorted(FIXTURE_DIR.glob("*.md")) if FIXTURE_DIR.is_dir() else [] + + +def _header_and_body(text: str) -> tuple[dict[str, str], str]: + """The same convention `LocalGitCorpus._header_of` reads (`runtime/ + local_git_adapter.py`): the leading run of `Name: value` lines up to the + first blank line is the header, and everything after is the body. + Re-implemented rather than imported — this fixture must be readable as + plain text by something that never builds a corpus at all, exactly as + the acceptance harness (T095) and a human skimming the directory would + read it. + """ + lines = text.splitlines() + header: dict[str, str] = {} + body_start = len(lines) + for i, line in enumerate(lines): + if not line.strip(): + body_start = i + 1 + break + name, separator, value = line.partition(":") + if separator: + header[name.strip()] = value.strip() + return header, "\n".join(lines[body_start:]) + + +def _documents() -> dict[Path, tuple[dict[str, str], str]]: + return {path: _header_and_body(path.read_text(encoding="utf-8")) + for path in FIXTURE_FILES} + + +def test_fixture_directory_exists_and_is_not_empty() -> None: + assert FIXTURE_DIR.is_dir(), f"expected {FIXTURE_DIR} to exist" + assert FIXTURE_FILES, f"no .md fixture files found under {FIXTURE_DIR}" + + +@pytest.mark.parametrize("path", FIXTURE_FILES, ids=lambda p: p.name) +def test_no_declared_vocabulary_word(path: Path) -> None: + """Neither the header nor the body of any fixture document carries one + of the eight `Status:` words or a change/spec/delta noun.""" + text = path.read_text(encoding="utf-8") + hits = sorted({m.group(1).lower() for m in _WORD_PATTERN.finditer(text)}) + assert not hits, ( + f"{path.name} carries declared vocabulary word(s) {hits}; a neutral " + "fixture must not spell its publisher's governance words" + ) + + +@pytest.mark.parametrize("path", FIXTURE_FILES, ids=lambda p: p.name) +def test_required_fields_present(path: Path) -> None: + header, _ = _header_and_body(path.read_text(encoding="utf-8")) + missing = [field for field in REQUIRED_FIELDS if field not in header] + assert not missing, f"{path.name} is missing required field(s) {missing}" + + +def test_spread_across_the_six_stations() -> None: + """One document per explicit station, and every undeclared document is + a source (R1Q13 (a) with (c)).""" + by_role: dict[str, list[Path]] = {role: [] for role in STAGE_ROLES} + for path, (header, _) in _documents().items(): + stage = header.get("stage") + if stage is None: + by_role["source"].append(path) + continue + assert stage in STAGE_ROLES, ( + f"{path.name} declares stage {stage!r}, not one of {STAGE_ROLES}" + ) + by_role[stage].append(path) + + empty = [role for role, paths in by_role.items() if not paths] + assert not empty, f"no fixture document occupies station(s) {empty}" + + # "source" is where every undeclared document lands, so it alone may + # hold more than one; the five explicit stations carry exactly one each. + for role in STAGE_ROLES: + if role == "source": + continue + assert len(by_role[role]) == 1, ( + f"station {role!r} has {len(by_role[role])} document(s), want 1" + ) + + +def test_at_least_two_sources_share_a_topic() -> None: + """R1Q13 (a) with (c): groups derive from the topics sources share, and + the fixture must yield at least one group.""" + sharing = [ + path for path, (header, body) in _documents().items() + if "stage" not in header + and SHARED_SOURCE_TOPIC in " ".join( + (header.get("title", ""), header.get("summary", ""), body) + ).lower() + ] + assert len(sharing) >= 2, ( + f"expected >= 2 source documents sharing {SHARED_SOURCE_TOPIC!r}, " + f"found {[p.name for p in sharing]}" + ) + + +def test_at_least_one_source_shares_no_topic() -> None: + """Keeps the assertion above honest: a fixture where every source shared + one word would not exercise topic-based selection at all.""" + singleton = [ + path for path, (header, body) in _documents().items() + if "stage" not in header + and SHARED_SOURCE_TOPIC not in " ".join( + (header.get("title", ""), header.get("summary", ""), body) + ).lower() + ] + assert singleton, "every source shares the same topic phrase" + + +if __name__ == "__main__": + sys.exit(pytest.main([__file__, "-v"])) From 834f8ead4e4f333f39d07f740009d7bf85f47597 Mon Sep 17 00:00:00 2001 From: Brett Heap <1513478+brettheap@users.noreply.github.com> Date: Sun, 27 Sep 2026 16:39:22 +0000 Subject: [PATCH 02/34] T052: 5.4, declare the generator seam, and register openDox's own generator (plan 034) Box 5.4 of openxFactory's add-neutral-product-standalone-operability: "DECLARE THE GENERATOR SEAM - it does not exist and CorpusAdapter is not it." This declares it, in src/opendox/generator_seam.py, beside domain_profile.register(): - the operation handed over: generate(repo_root, repository, *, source_revision=None, generated_at=None, **inputs) -> the snapshot; - the registration point: register() for a host, and register_default(...) for an entry point, which registers only where nothing is registered; - the conformance a contributed generator must meet. It declares the contract it writes and every further input it reads, its operation takes the seam's call, and it answers a snapshot whose kind is that contract and whose schema_version is an integer. The seam checks the declaration when it is made and each snapshot when it comes back, and it refuses, never drops, an undeclared input. The conformance clause names T053's neutral snapshot kind, "opendox-snapshot" (openDox-spec#16), for openDox's own generator: src/opendox/default_generator.py's GENERATOR. The entry points (cli.build_parser, cli.main, serve.build_server, serve.main) register it where no host has (R1Q10 (a), openxFactory#656 comment 5850003126, in R1Q3 (a)'s pattern). A bare process still refuses, naming the seam and the call. A host replaces the default only before a snapshot has been generated from it. CorpusAdapter stays closed at six members. openDox's own generator refuses until T054 builds its projection. Plan 034 orders T052 before T054, and T054 edits neither cli.py nor serve.py (their single-writer order runs T052, then T055). No verb reaches the seam before T055 routes the generate verbs, which comes after T054. Falsifier: the seam tests, tests/test_generator_seam.py (59 cases). F5.2 is quoted by T059 and T061. Arc: neutral-product-standalone-operability Lane: openxfactory-4 (openXfactory-4-openDox_extraction) Co-Authored-By: Claude Opus 5.5 (1M context) --- src/opendox/cli.py | 9 + src/opendox/default_generator.py | 76 ++++ src/opendox/generator_seam.py | 515 +++++++++++++++++++++++++ src/opendox/serve.py | 9 + tests/test_generator_seam.py | 632 +++++++++++++++++++++++++++++++ 5 files changed, 1241 insertions(+) create mode 100644 src/opendox/default_generator.py create mode 100644 src/opendox/generator_seam.py create mode 100644 tests/test_generator_seam.py diff --git a/src/opendox/cli.py b/src/opendox/cli.py index 994111dc..cf4b9edc 100644 --- a/src/opendox/cli.py +++ b/src/opendox/cli.py @@ -128,6 +128,10 @@ # in (R1Q3 (a)). Importing either registers nothing: `build_parser()` and # `main()` below make the registration, and only where no host has made one. from opendox import default_profile, domain_profile # noqa: E402 +# openDox's OWN snapshot generator, and the generator seam an entry point +# registers it at (5.4; plan 034 T052). Importing either registers nothing: +# `build_parser()` and `main()` below register it, only where no host has. +from opendox import default_generator, generator_seam # noqa: E402 is_rfc3339_datetime = consumer_reach.is_rfc3339_datetime # noqa: E402 # ...and `SCANNED_ROOTS` keeps its NAME and its behaviour, not just its value: # :239 iterates it (`for root in SCANNED_ROOTS`) and that line is not one the @@ -951,6 +955,9 @@ def build_parser(*, subcommand_extensions: tuple = ()) -> argparse.ArgumentParse # `SUBCOMMAND_EXTENSIONS` is read a few lines below; the corpus # adapter's registration is read later, from `authoring.py`). corpus_adapter.register_default_home(_default_home_factory) + # AND openDox's OWN snapshot generator (5.4, T052; R1Q10 (a), in the same + # R1Q3 (a) pattern), registered only where no host has contributed one. + generator_seam.register_default(default_generator.GENERATOR) parser = argparse.ArgumentParser(prog="ideation-dashboard", description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) sub = parser.add_subparsers(dest="command", required=True) @@ -1054,6 +1061,8 @@ def main(argv: list[str] | None = None, *, # own match here: the OUTERMOST entry point states the contract on its # own, independent of what `build_parser()` does inside. corpus_adapter.register_default_home(_default_home_factory) + # AND openDox's own snapshot generator (5.4, T052), the same way. + generator_seam.register_default(default_generator.GENERATOR) args = build_parser( subcommand_extensions=subcommand_extensions).parse_args(argv) try: diff --git a/src/opendox/default_generator.py b/src/opendox/default_generator.py new file mode 100644 index 00000000..f2b9cb69 --- /dev/null +++ b/src/opendox/default_generator.py @@ -0,0 +1,76 @@ +"""openDox's OWN snapshot generator, declared as the generator seam's default. + +WHY THIS FILE EXISTS. openDox's entry points register openDox's own generator +where no host has contributed one (R1Q10 (a), `openxFactory#656` comment +`5850003126`, in R1Q3 (a)'s pattern). They do so beside the default profile and +the default home corpus, which they register the same way. This module is what +they register: `GENERATOR`, a `generator_seam.SnapshotGenerator` that writes +the neutral snapshot contract, `generator_seam.NEUTRAL_SNAPSHOT_KIND` (plan +034's T053; R1Q11 (a)), and takes no input beyond the operation's own four. + +ITS PROJECTION IS NOT BUILT YET, AND IT SAYS SO. Plan 034 orders the seam +before the projection. T054, openDox's small neutral projection over +`CorpusAdapter` (#1144's 5.1-5.3), comes after this task (T052). The entry +points' registration calls are this task's, because T054 edits neither +`cli.py` nor `serve.py`, whose single-writer order runs T052, then T055. So +`generate()` below refuses, naming itself and the task that builds it, and it +generates nothing. It never answers an empty snapshot, which would read exactly +like an honest one. + +NO VERB REACHES IT YET. The generate verbs still call the consumer's generator, +through `consumer_reach`, until T055 routes them through the seam, and T055 +comes after T054. The refusal is therefore reachable only by a library caller +that generates through the seam directly. T054 replaces it with the +projection, and the declaration below keeps its contract and its inputs. + +HOW IT IS REGISTERED: by the entry points, and never at import (R1Q3 (a)'s +pattern). `cli.build_parser()`, `cli.main()`, `serve.build_server()` and +`serve.main()` call `generator_seam.register_default(GENERATOR)`, which +registers it only where nothing is registered. Importing this module registers +nothing. + +IMPORT WEIGHT. `opendox.generator_seam` and the standard library. So this +module imports with no extra installed and no sibling present. + +A CREATED FILE: it has no row in openxFactory's +`docs/opendox-carve-manifest.yaml`, because the manifest declares what LEAVES +openxFactory and never what a destination assembles (RULED OQ-C). +""" + +from __future__ import annotations + +from pathlib import Path +from typing import Any + +from opendox import generator_seam + +__all__ = ["GENERATOR", "NeutralProjectionNotBuilt", "generate"] + + +class NeutralProjectionNotBuilt(generator_seam.GeneratorSeamError): + """openDox's own generator is declared, and its projection is not built. + + A subclass of the seam's own refusal, so a verb that reports the seam's + refusals reports this one too. It goes when plan 034's T054 lands the + projection.""" + + +def generate(repo_root: Path, repository: str, *, + source_revision: str | None = None, + generated_at: str | None = None) -> dict[str, Any]: + """The operation the seam hands over, for openDox's own generator. + + It REFUSES until plan 034's T054 builds the neutral projection. See the + module docstring for why the declaration lands first.""" + raise NeutralProjectionNotBuilt( + f"openDox's own generator is declared at the generator seam, writing " + f"{generator_seam.NEUTRAL_SNAPSHOT_KIND!r}, but its projection is not " + f"built at this commit (plan 034's T054 builds it), so nothing was " + f"generated for {repository!r} at {str(repo_root)!r}. A host that has " + f"a generator of its own registers it at process start with " + f"{generator_seam.REGISTRATION_CALL}.") + + +#: openDox's OWN generator, as the entry points register it where no host has. +GENERATOR = generator_seam.SnapshotGenerator( + contract=generator_seam.NEUTRAL_SNAPSHOT_KIND, generate=generate) diff --git a/src/opendox/generator_seam.py b/src/opendox/generator_seam.py new file mode 100644 index 00000000..0e8edcb2 --- /dev/null +++ b/src/opendox/generator_seam.py @@ -0,0 +1,515 @@ +"""THE GENERATOR SEAM: the one snapshot generator a process generates with, a +host's contribution or openDox's own default, held for late resolution. + +WHY THIS FILE EXISTS. Box 5.4 of openxFactory's +`add-neutral-product-standalone-operability` (RATIFIED, `openxFactory#656` +comment `5815412869`) reads *"DECLARE THE GENERATOR SEAM — it does not exist +and `CorpusAdapter` is not it"*. Its requirement 4 has openDox grow a neutral +generator of its own, while the consumer KEEPS its governed generator and +contributes it *"through a DECLARED GENERATOR SEAM"*. That seam *"SHALL BE +DECLARED BY THIS ARC, naming the operation it hands over, the registration +point, and what a conformant implementation must satisfy"*. `consumer_reach.py` +names the same gap from the other side. The injection that would retire its +`generator` reach, *"openDox naming a protocol and being handed an +implementation"*, *"does not exist yet and is BUILD-arc work"*. This module is +that declaration (plan 034's T052). + +IT ROUTES NOTHING YET. The generate verbs still call the consumer's generator +through `consumer_reach` until plan 034's T055 routes them through `generate()` +below. T055 comes after T054, which builds openDox's own projection. So from +the day a verb generates through this seam, openDox's own generator can +generate. + +`CorpusAdapter` IS NOT IT, AND STAYS CLOSED AT SIX MEMBERS. The corpus-read +interface (`corpus_adapter.py`) declares `resolve`, `list_documents`, `read`, +`classify`, `check` and `write_back`, and its own docstring says *"Nothing +else, ever"*. None of the six hands a generator over, and a seventh member +would be a seam that leaks. So a generator is handed over HERE, through a +registration of its own. A generator READS its corpus through that interface, +as openDox's own projection does (plan 034's T054). + +THE OPERATION HANDED OVER is one call. It answers the snapshot, a JSON-ready +`dict`: + + generate(repo_root, repository, *, + source_revision=None, generated_at=None, **inputs) + +* `repo_root` is the corpus checkout the generator scans, as a `Path`. +* `repository` is the repository id the snapshot is for. +* `source_revision` is the source anchor to pin, or `None`, which leaves the + generator to read the checkout itself. +* `generated_at` is the generation anchor, recorded verbatim when it is given. +* `**inputs` are the further inputs a generator DECLARES that it takes. Each is + passed only when the caller has a value for it. + +The four arguments are the ones openDox's generate verbs already hand a +generator. A governed generator's own extras, such as a project register, +travel as declared inputs. So the seam never drops one, and never names one +either. + +THE REGISTRATION POINT sits beside `domain_profile.register()`, in the shape +the product already uses (5.4's own words). A host calls +`opendox.generator_seam.register()` ONCE, at process +start. openDox's own entry points call `register_default(...)`, which registers +openDox's own generator only where nothing is registered. + +THE CONFORMANCE A CONTRIBUTED GENERATOR MUST MEET. `SnapshotGenerator` declares +it, and `generate()` enforces it. + +1. It DECLARES THE CONTRACT IT WRITES: the `kind` of every snapshot it answers. + openDox's own generator writes `NEUTRAL_SNAPSHOT_KIND`, the neutral + snapshot contract whose schema openDox's own spec leg owns (plan 034's + T053; R1Q11 (a), `openxFactory#656` comment `5850003126`). A consumer's + governed generator declares its own. So two generators that write two + contracts can never both claim to be the one conformant implementation, + which is the failure 5.4 was raised against. +2. Its operation TAKES THE SEAM'S CALL: the four arguments, and every input it + declares. This is checked when the declaration is made, wherever the + callable's signature can be read. +3. It DECLARES EVERY FURTHER INPUT it reads. An input that it does not declare + is refused before the generator is called, and never dropped. +4. It ANSWERS A SNAPSHOT OF ITS DECLARED CONTRACT: a `dict` whose `kind` is + that contract and whose `schema_version` is an integer, which is what a + validator picks a schema by. Anything else is refused, not handed on. +5. It is DETERMINISTIC over its inputs, as the generate verbs promise (*"the + deterministic snapshot"*): the same corpus at the same anchors answers the + same snapshot. Each generator's own suite proves that. The seam does not + check it again. + +REFUSAL, NOT A DEFAULT (4.2's discipline, and `domain_profile`'s). With nothing +registered, `current()` and `generate()` refuse, naming this seam and the +registration call. They never fall back to openDox's own generator, and they +never answer an empty snapshot, which would read exactly like an honest one. + +THE ENTRY POINTS REGISTER openDox's OWN GENERATOR WHERE NO HOST HAS (R1Q10 +(a), comment `5850003126`, in R1Q3 (a)'s pattern, comment `5817152735`). +`cli.build_parser()`, `cli.main()`, `serve.build_server()` and `serve.main()` +call `register_default(default_generator.GENERATOR)`, beside their +registrations of the default profile and the default home corpus. So, as for +the profile: + +* a process that builds nothing still meets `GeneratorNotRegistered`, which is + the library caller's case; +* a host registration made BEFORE anything is generated from the default + replaces it; +* AFTER a snapshot has been generated from the default, a host's registration + is refused as `GeneratorAlreadyRegistered`. One process would otherwise + write two contracts, and a reader could not tell which snapshot was which. + This is the same reason `domain_profile` refuses a swap after a build + (R1Q3 (ii); RN-1 (a), comment `5850003126`). + +ONE REGISTRATION. Registering the same declaration again is a no-op, so an +idempotent host start is not punished. A different declaration over a host's +is refused as `GeneratorAlreadyRegistered`. `unregister()` makes a deliberate +swap explicit. + +RESOLVE PER CALL. A caller generates through `generate()`, and does not hold +`current()`'s answer across calls. So every caller in a process answers from +the one registration that exists at the time. + +IMPORT WEIGHT. The standard library only. So this module imports with no extra +installed and no sibling present, and it names no sibling. + +A CREATED FILE: it has no row in openxFactory's +`docs/opendox-carve-manifest.yaml`, because the manifest declares what LEAVES +openxFactory and never what a destination assembles (RULED OQ-C). +""" + +from __future__ import annotations + +import inspect +import keyword +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Callable + +__all__ = [ + "GeneratorAlreadyRegistered", + "GeneratorInputRefused", + "GeneratorNotConformant", + "GeneratorNotRegistered", + "GeneratorSeamError", + "NEUTRAL_SNAPSHOT_KIND", + "OPERATION_ARGUMENTS", + "REGISTRATION_CALL", + "SnapshotGenerator", + "current", + "generate", + "is_registered", + "name_of", + "register", + "register_default", + "unregister", +] + +#: The contract openDox's OWN generator writes: the `kind` of the neutral +#: snapshot, whose schema openDox's spec leg owns (plan 034's T053; R1Q11 (a), +#: `openxFactory#656` comment `5850003126`). It is the `kind` const of that +#: schema, `contracts/schemas/opendox-snapshot.schema.yaml` (openDox-spec#16), +#: and it must stay equal to it. The value is a string in every neutral +#: snapshot, so F5.3's sweep of the snapshot's string values reads it too, and +#: it carries none of the words that sweep forbids. +NEUTRAL_SNAPSHOT_KIND = "opendox-snapshot" + +#: The operation's own arguments, in its call order. The first two are +#: positional, and the last two are keywords. A generator's declared inputs may +#: not reuse these names. +OPERATION_ARGUMENTS: tuple[str, ...] = ( + "repo_root", "repository", "source_revision", "generated_at") + +#: The ONE call a host makes, quoted verbatim in every refusal, so that a +#: refusal names its remedy rather than its symptom. +REGISTRATION_CALL = ( + "opendox.generator_seam.register()") + +#: How much of a `repr` a refusal quotes before it stops being read. +_NAME_LIMIT = 120 + + +class GeneratorSeamError(RuntimeError): + """A refusal of the generator seam. + + One base for every refusal below, so a verb that generates can report the + seam's refusals in one clause, as `cli.main()` reports its own.""" + + +class GeneratorNotRegistered(GeneratorSeamError): + """Nothing is registered: no host's generator, and no entry point's default. + + Raised instead of falling back to openDox's own generator. The default is a + registration an ENTRY POINT makes (R1Q10 (a), in R1Q3 (a)'s pattern), so + this is what a process meets when it generates without having built + anything through an entry point, which is the library caller's case. It is + also what a process meets after `unregister()`, because the refusal reports + what is registered NOW and keeps no history.""" + + +class GeneratorAlreadyRegistered(GeneratorSeamError): + """A second, different generator was registered over a first. + + ONE registration is the contract, as it is for the profile. A process whose + snapshots came from two generators would write two contracts under one + name. The entry point's default is replaceable only until a snapshot has + been generated from it. `unregister()` makes a deliberate swap explicit.""" + + +class GeneratorInputRefused(GeneratorSeamError): + """The registered generator was handed an input it does not declare. + + Refused before the generator is called. A generator that silently ignored + the input would write a snapshot that looks as though the input had been + read.""" + + +class GeneratorNotConformant(GeneratorSeamError): + """A generator broke the conformance this seam declares. + + It answered something other than a snapshot of its declared contract, or + it was offered as openDox's own default while declaring another contract. + What it answered is refused, not handed on: the validator and the views + read a snapshot by its `kind`.""" + + +def _qualified(value: Any) -> str: + """`module.qualname` for a callable, else a truncated `repr`. Never raises.""" + try: + module = getattr(value, "__module__", None) + qualname = getattr(value, "__qualname__", None) + if isinstance(module, str) and isinstance(qualname, str) and module and qualname: + return f"{module}.{qualname}" + except Exception: # noqa: BLE001 - naming must never out-raise + pass + try: + text = repr(value) + except Exception: # noqa: BLE001 + text = "" + if not text: + return "an unnameable callable" + return text if len(text) <= _NAME_LIMIT else text[:_NAME_LIMIT - 1] + "…" + + +@dataclass(frozen=True, eq=False) +class SnapshotGenerator: + """ONE generator's declaration: the contract it writes, its operation, and + the further inputs it takes. It is what `register()` and + `register_default()` hold. + + `contract` is the `kind` of every snapshot the generator answers. + `generate` is the operation this module's docstring declares. `inputs` + names the keyword inputs it reads beyond the operation's own four, and + defaults to none. + + Construction refuses a declaration the seam could not honour, rather than + registering one that fails at its first generation. That covers a contract + that is not a non-empty string, an operation that is not callable, an input + name that is not an identifier (or is a keyword, a duplicate, or one of the + operation's own four), and an operation whose signature cannot take the + seam's call. Frozen, and compared by identity, as `register()` compares + registrations.""" + + contract: str + generate: Callable[..., dict] + inputs: tuple[str, ...] = () + + def __post_init__(self) -> None: + if not isinstance(self.contract, str): + raise TypeError( + "a SnapshotGenerator declares the contract it writes as a " + f"string, the kind of every snapshot it answers, not " + f"{type(self.contract).__name__}") + if not self.contract or self.contract != self.contract.strip(): + raise ValueError( + "a SnapshotGenerator's contract is the kind its snapshots carry, " + f"a non-empty name with no surrounding space, not {self.contract!r}") + if not callable(self.generate): + raise TypeError( + "a SnapshotGenerator's generate is the operation the seam hands " + f"over, a callable, not {type(self.generate).__name__}") + if not isinstance(self.inputs, tuple) or not all( + isinstance(name, str) for name in self.inputs): + raise TypeError( + "a SnapshotGenerator declares its further inputs as a tuple of " + f"names, not {self.inputs!r}") + bad = sorted({name for name in self.inputs + if not name.isidentifier() or keyword.iskeyword(name)}) + reused = sorted(set(self.inputs) & set(OPERATION_ARGUMENTS)) + repeated = sorted({name for name in self.inputs + if self.inputs.count(name) > 1}) + if bad or reused or repeated: + raise ValueError( + "a SnapshotGenerator's inputs name keyword arguments its " + "operation takes beyond the operation's own four " + f"({', '.join(OPERATION_ARGUMENTS)}). Refused: " + f"not an identifier {bad}, one of the four {reused}, " + f"repeated {repeated}") + self._refuse_an_operation_that_cannot_take_the_call() + + def _refuse_an_operation_that_cannot_take_the_call(self) -> None: + try: + signature = inspect.signature(self.generate) + except (TypeError, ValueError): + # The signature cannot be read (some builtins). The seam's call is + # then checked when it is first made, and not here. + return + try: + signature.bind(Path("."), "repository", source_revision=None, + generated_at=None, + **{name: None for name in self.inputs}) + except TypeError as exc: + raise TypeError( + f"{_qualified(self.generate)} cannot take the generator seam's " + "call, generate(repo_root, repository, *, source_revision=, " + "generated_at=" + "".join(f", {name}=" for name in self.inputs) + + f"): {exc}. A contributed generator takes the operation's own " + "four arguments and every input it declares") from None + + +#: THE one registration, a host's or the entry point's default, or `None`. +_registered: SnapshotGenerator | None = None + +#: Whether `_registered` is the default an ENTRY POINT registered +#: (`register_default()`), rather than a host's own `register()`. +_is_default: bool = False + +#: Whether a snapshot has been generated from that default. Set by +#: `generate()`, before the default's operation is called. +_generated_from_default: bool = False + + +def name_of(generator: Any) -> str: + """A generator's most nameable name, with its contract, for a refusal. + + Public, as `domain_profile.name_of()` is, so a host's own refusal can name a + generator the same way. It never raises: a refusal that fails while + formatting itself replaces the reader's problem with a worse one.""" + if isinstance(generator, SnapshotGenerator): + return f"{_qualified(generator.generate)} (writing {generator.contract!r})" + return _qualified(generator) + + +def _require_a_declaration(generator: Any, call: str) -> None: + if not isinstance(generator, SnapshotGenerator): + raise TypeError( + f"{call} takes a SnapshotGenerator, the declaration of a generator's " + "contract, operation and inputs, not " + f"{type(generator).__name__}. A host with no generator of its own " + "does not register one: the seam then answers with openDox's own " + "generator where an entry point registered it, and refuses where " + "none did.") + + +def register(generator: SnapshotGenerator) -> SnapshotGenerator: + """THE one registration. A host calls this at process start. + + Returns the generator, so a host can register it and hold it in one + expression. + + Registering the SAME declaration again is a no-op. A DIFFERENT one over a + host's registration raises `GeneratorAlreadyRegistered`. Over the entry + point's default it REPLACES the default while nothing has been generated + from it, and is refused once something has. Either way the host ends up + holding the one registration, or knows why it does not.""" + global _registered, _is_default, _generated_from_default + _require_a_declaration(generator, "register()") + if _registered is generator: + # The same object again: a no-op, before any bookkeeping is touched. + return generator + if _registered is not None: + if not _is_default: + raise GeneratorAlreadyRegistered( + f"a host's generator is already registered at openDox's " + f"generator seam ({name_of(_registered)}), and " + f"{name_of(generator)} would replace it. Registration happens " + "ONCE, at process start: one process generating through two " + "generators would write two contracts, and a reader could not " + "tell which snapshot was which. Call " + "opendox.generator_seam.unregister() first if the swap is " + "deliberate.") + if _generated_from_default: + raise GeneratorAlreadyRegistered( + f"openDox's own default generator ({name_of(_registered)}) is " + "registered, because an entry point registered it where no host " + "had, and a snapshot has already been generated from it, so " + f"{name_of(generator)} cannot replace it now. A swap would leave " + "one process writing two contracts. A host's generator replaces " + "the default only BEFORE anything is generated from it, as a " + "host's profile replaces the default profile only before a build " + "(R1Q3 (ii), openxFactory#656 comment 5817152735; RN-1 (a), " + "comment 5850003126). So register the host's generator at " + "process start, ahead of the first generation. Call " + "opendox.generator_seam.unregister() first if the swap is " + "deliberate.") + _registered = generator + _is_default = False + _generated_from_default = False + return generator + + +def register_default(generator: SnapshotGenerator) -> SnapshotGenerator: + """AN ENTRY POINT'S registration of openDox's OWN generator (R1Q10 (a)). + + `cli.build_parser()`, `cli.main()`, `serve.build_server()` and + `serve.main()` call this with `default_generator.GENERATOR`. It registers + `generator` ONLY where nothing is registered, and leaves a host's + registration, or a default already registered, exactly as it is. Returns + whatever is registered afterwards. + + It is NOT for hosts. A host calls `register()`. The conformance clause holds + openDox's own generator to the neutral contract, so a declaration that + writes another contract is refused as `GeneratorNotConformant`, whether or + not anything is registered.""" + global _registered, _is_default, _generated_from_default + _require_a_declaration(generator, "register_default()") + if generator.contract != NEUTRAL_SNAPSHOT_KIND: + raise GeneratorNotConformant( + "register_default() registers openDox's OWN generator, which writes " + f"the neutral snapshot contract {NEUTRAL_SNAPSHOT_KIND!r}, and " + f"{name_of(generator)} declares another. A host's generator is " + f"registered with {REGISTRATION_CALL}.") + if _registered is None: + _registered = generator + _is_default = True + _generated_from_default = False + return _registered + + +def unregister() -> None: + """Drop the registration, a host's or the entry point's default. + + For test isolation and for a host tearing down. The record of a generation + from the default goes with it.""" + global _registered, _is_default, _generated_from_default + _registered = None + _is_default = False + _generated_from_default = False + + +def is_registered() -> bool: + """Is a generator registered? Answers without resolving or refusing.""" + return _registered is not None + + +def current() -> SnapshotGenerator: + """The registered generator, or a refusal naming this seam and its call. + + It answers what is registered: a host's generator, or the default an entry + point registered. It never falls back to the default itself, and asking + records nothing: only a generation closes the default's window.""" + if _registered is None: + raise GeneratorNotRegistered( + "no snapshot generator is registered at openDox's generator seam " + "(opendox.generator_seam), so there is nothing to generate a " + "snapshot with, and nothing is generated. openDox ships a generator " + "of its own, opendox.default_generator, but it is a registration an " + "ENTRY POINT makes and never a fallback here (R1Q10 (a), " + "openxFactory#656 comment 5850003126, in R1Q3 (a)'s pattern). " + "`cli.build_parser()`, `cli.main()`, `serve.build_server()` and " + "`serve.main()` register it where no host has. Nothing is " + "registered now, so either nothing in this process has been built " + "through one of them, or opendox.generator_seam.unregister() has " + "dropped the registration since. A host that contributes its own " + "generator registers it at process start with\n\n " + + REGISTRATION_CALL + "\n\nbefore the first generation.") + return _registered + + +def generate(repo_root: Path | str, repository: str, *, + source_revision: str | None = None, + generated_at: str | None = None, + **inputs: Any) -> dict: + """Generate a snapshot through the registered generator. This is THE + operation the seam hands over, called and checked. + + An input given as `None` means "not given", and it is not passed, so a verb + can hand over its options whether or not they were set. Any other input the + registered generator does not declare is refused before the generator is + called (`GeneratorInputRefused`). What the generator answers is handed back + only if it is a snapshot of the contract the generator declared + (`GeneratorNotConformant`). With nothing registered this refuses as + `current()` does.""" + global _generated_from_default + generator = current() + given = {name: value for name, value in inputs.items() if value is not None} + undeclared = sorted(set(given) - set(generator.inputs)) + if undeclared: + declared = ", ".join(generator.inputs) or "none" + raise GeneratorInputRefused( + f"{name_of(generator)} declares the inputs ({declared}) beyond the " + f"operation's own four, and was handed {', '.join(undeclared)}. An " + "input a generator does not declare is refused rather than dropped: " + "a generator that silently ignored it would write a snapshot that " + "looks as though the input had been read.") + if _is_default: + # Before the call, so a registration racing a first generation cannot + # slip in between them. + _generated_from_default = True + snapshot = generator.generate( + Path(repo_root), repository, source_revision=source_revision, + generated_at=generated_at, **given) + _refuse_a_snapshot_that_does_not_conform(generator, snapshot) + return snapshot + + +def _refuse_a_snapshot_that_does_not_conform(generator: SnapshotGenerator, + snapshot: Any) -> None: + if not isinstance(snapshot, dict): + raise GeneratorNotConformant( + f"{name_of(generator)} answered {type(snapshot).__name__}, not a " + "snapshot. The seam hands on only a snapshot, a dict, and an answer " + "it cannot read is refused rather than taken for one.") + kind = snapshot.get("kind") + if kind != generator.contract: + raise GeneratorNotConformant( + f"{name_of(generator)} declares that it writes " + f"{generator.contract!r}, and it answered a snapshot whose kind is " + f"{kind!r}. The seam hands back only a snapshot of the contract its " + "generator declared. The validator and the views read a snapshot " + "by its kind, so a generator whose output drifted from its " + "declaration would be two contracts under one name.") + version = snapshot.get("schema_version") + if type(version) is not int: + raise GeneratorNotConformant( + f"{name_of(generator)} answered a {generator.contract!r} snapshot " + f"whose schema_version is {version!r}, not an integer. A validator " + "picks a schema by kind and version, so a snapshot that does not " + "say which version of its contract it is cannot be checked.") diff --git a/src/opendox/serve.py b/src/opendox/serve.py index 9e3a2c03..a0cd5303 100644 --- a/src/opendox/serve.py +++ b/src/opendox/serve.py @@ -161,6 +161,10 @@ # `opendox.runtime.config` and `opendox.corpus_adapter` besides the stdlib. from opendox import corpus_adapter # noqa: E402 from opendox.runtime import local_git_adapter # noqa: E402 +# THE GENERATOR SEAM'S DEFAULT (5.4; plan 034 T052): openDox's own snapshot +# generator, which `build_server()` and `main()` register where no host has. +# Both modules are stdlib-only and name no sibling, so this adds no reach. +from opendox import default_generator, generator_seam # noqa: E402 registry_mod = consumer_reach.snapshot_registry # noqa: E402 # THE BY-FUNCTION SPLIT (`split-opendox-two-layer-product` § 2.4, PRs 2 and 3 @@ -1649,6 +1653,9 @@ def build_server( # registers ONLY where nothing already answers `corpus_adapter.home()`, # exactly as `domain_profile.register_default()` does for the profile. corpus_adapter.register_default_home(_default_home_factory) + # AND openDox's OWN snapshot generator (5.4, T052; R1Q10 (a), in the same + # R1Q3 (a) pattern), registered only where no host has contributed one. + generator_seam.register_default(default_generator.GENERATOR) from opendox import doxbench_turns # Imported HERE rather than at module scope, for the reason that is @@ -2147,6 +2154,8 @@ def main(argv: list[str] | None = None) -> int: # no-op once that runs; kept for the same reason T016 keeps its own # match here: the OUTERMOST entry point states the contract on its own. corpus_adapter.register_default_home(_default_home_factory) + # AND openDox's own snapshot generator (5.4, T052), the same way. + generator_seam.register_default(default_generator.GENERATOR) parser = argparse.ArgumentParser(prog="ideation-dashboard-serve", description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) parser.add_argument("--web-dir", default=str(Path(__file__).resolve().parent / "web"), diff --git a/tests/test_generator_seam.py b/tests/test_generator_seam.py new file mode 100644 index 00000000..d8e1bab6 --- /dev/null +++ b/tests/test_generator_seam.py @@ -0,0 +1,632 @@ +"""The generator seam, held to 5.4 of openxFactory's +`add-neutral-product-standalone-operability` (plan 034, T052). + +Box 5.4 reads *"DECLARE THE GENERATOR SEAM — it does not exist and +`CorpusAdapter` is not it"*. It asks for the operation handed over, the +registration point beside `domain_profile.register()`, and the conformance a +contributed generator must satisfy. Plan 034's T052 adds that the conformance +clause names T053's neutral snapshot kind for openDox's own generator, and that +the entry points register openDox's own generator where no host has (R1Q10 (a), +`openxFactory#656` comment `5850003126`, in R1Q3 (a)'s pattern). These are +T052's seam tests. + +WHAT IT ASSERTS, AND WHY EACH IS HERE + +1. NOTHING IS REGISTERED BY AN IMPORT, AND THE SEAM MAKES NO REACH. A fresh + process that imports both modules and both entry points registers nothing, + and it meets the refusal. The two modules import with every sibling + blocked, pull in no third-party module, and name no sibling in any import, + late or not. +2. NOTHING REGISTERED REFUSES, NAMING THE SEAM AND THE CALL (4.2's + discipline). It never falls back to openDox's own generator. +3. THE DECLARATION IS CHECKED WHEN IT IS MADE. A contract that is not a name, + an operation that is not callable or cannot take the seam's call, and an + input that is not a name beyond the operation's own four are refused before + anything is registered. +4. THE OPERATION IS HANDED OVER AND ITS ANSWER CHECKED. The registered + generator receives the four arguments and its declared inputs. An + undeclared input is refused before the call. A `None` input is not passed. + Only a snapshot of the declared contract comes back. +5. ONE REGISTRATION. The same declaration twice is a no-op, and a second host + is refused. +6. THE ENTRY POINTS' DEFAULT, in R1Q3 (a)'s pattern. openDox's own generator + writes the neutral kind and takes no input. `register_default()` registers + it only where nothing is, and holds it to the neutral contract. A host + replaces it before a generation, and is refused after one. Until T054 lands, + the default refuses, naming itself. +7. EACH ENTRY POINT REGISTERS IT. `cli.build_parser()` and `cli.main()` run for + real. `serve.build_server()` and `serve.main()` still cannot run in a lone + checkout (research R7), so their registration is executed from their own + source lines, as `tests/test_authoring_seam.py` does for the home corpus. +8. `CorpusAdapter` STAYS CLOSED AT SIX MEMBERS. + +`--noconftest` SAFE. The autouse fixture below saves and restores the three +registries the entry points write, so no case depends on the root conftest or +leaves a registration behind. + +A CREATED file: no carve-manifest row (RULED OQ-C). +""" + +from __future__ import annotations + +import ast +import inspect +import re +import subprocess +import sys +import textwrap +from pathlib import Path + +import pytest + +from opendox import corpus_adapter, default_generator, domain_profile +from opendox import generator_seam as gs + +ROOT = Path(__file__).resolve().parent.parent +SRC = ROOT / "src" +CLI = SRC / "opendox" / "cli.py" +SERVE = SRC / "opendox" / "serve.py" + +#: The seam's two modules. +SEAM_MODULES = {"opendox.generator_seam": SRC / "opendox" / "generator_seam.py", + "opendox.default_generator": SRC / "opendox" / "default_generator.py"} + +#: The four packages a neutral openDox must import without (#1144's F2.1). +SIBLINGS = ("openxdox", "ideation_dashboard", "doc_health", + "corpus_adapter_openxfactory") + +#: F5.3's declared vocabulary, verbatim from #1144's `tasks.md`. F5.3 reads +#: every string value of the neutral snapshot, and the snapshot's `kind` is one. +F5_3_WORDS = ("brainstorm", "staged", "draft", "ratified", "standard", + "superseded", "retired", "record", "openspec", "proposal.md", + "tasks.md", "design.md", "added requirements", + "modified requirements") + + +@pytest.fixture(autouse=True) +def _isolated_registries(): + """Every case starts with NO generator registered, and PUTS BACK what it found. + + The three registries an entry point writes are process-global by design: + the generator seam, the profile and the home corpus. Each is saved whole, + because each keeps more than its registration (whether it is the entry + point's default, and whether anything was generated or built from it), and + a restore through `register()` would hand an entry point's default back as + a host's. + """ + seam = (gs._registered, gs._is_default, gs._generated_from_default) + profile = (domain_profile._registered, domain_profile._is_default, + domain_profile._built_from_default) + home = corpus_adapter._home_factory + gs.unregister() + yield + (gs._registered, gs._is_default, gs._generated_from_default) = seam + (domain_profile._registered, domain_profile._is_default, + domain_profile._built_from_default) = profile + corpus_adapter._home_factory = home + + +def _declared(contract: str = "stand-in-snapshot", *, inputs: tuple = (), + answer=None): + """A stand-in generator's declaration, and the list its operation records. + + The operation takes exactly the seam's call. It answers `answer` when one + is given, and otherwise a minimal snapshot of `contract`.""" + calls: list[dict] = [] + + def operation(repo_root, repository, *, source_revision=None, + generated_at=None, **extra): + calls.append({"repo_root": repo_root, "repository": repository, + "source_revision": source_revision, + "generated_at": generated_at, **extra}) + if answer is not None: + return answer + return {"schema_version": 1, "kind": contract, "repository": repository} + + return gs.SnapshotGenerator(contract=contract, generate=operation, + inputs=inputs), calls + + +def _fresh_process(program: str) -> subprocess.CompletedProcess: + """`program` in a fresh interpreter, with this checkout's `src` first. The + registry is process-global, so a case about what a PROCESS meets runs in + one of its own.""" + return subprocess.run( + [sys.executable, "-c", f"import sys; sys.path.insert(0, {str(SRC)!r})\n" + + textwrap.dedent(program)], + capture_output=True, text=True, cwd=str(ROOT)) + + +# -------------------------------------------------------------------------- +# 1 — nothing is registered by an import, and the seam makes no reach +# -------------------------------------------------------------------------- + +def test_importing_the_seam_and_the_entry_points_registers_nothing() -> None: + """The library caller's case. A process that builds nothing still refuses, + because openDox's own generator is a registration an entry point MAKES.""" + done = _fresh_process(""" + import opendox.cli, opendox.serve, opendox.default_generator + from opendox import generator_seam as gs + assert gs.is_registered() is False, "an import registered a generator" + for call in (gs.current, lambda: gs.generate(".", "fixture")): + try: + call() + except gs.GeneratorNotRegistered: + continue + raise AssertionError(f"{call} answered with nothing registered") + print("refused") + """) + assert done.returncode == 0, done.stderr + assert done.stdout.strip() == "refused" + + +def test_the_seam_imports_with_no_sibling_and_no_third_party_module() -> None: + """Both modules import with the four siblings blocked at the finder, and + what they add to `sys.modules` is openDox's own or the standard library's.""" + block = "".join(f"sys.modules[{name!r}] = None\n" for name in SIBLINGS) + done = _fresh_process(block + textwrap.dedent(""" + before = set(sys.modules) + import opendox.generator_seam, opendox.default_generator + added = {name.split(".")[0] for name in set(sys.modules) - before} + print(sorted(added - set(sys.stdlib_module_names) - {"opendox"})) + """)) + assert done.returncode == 0, done.stderr + assert done.stdout.strip() == "[]", ( + f"the seam pulls in a module that is neither openDox's nor the " + f"standard library's: {done.stdout.strip()}") + + +def _named_imports(node: ast.AST) -> list[str]: + """The module names an import node, or an `import_module`/`__import__` + call with a literal name, reaches. This is #1144's F4.1 scan's own reading.""" + if isinstance(node, ast.Import): + return [alias.name for alias in node.names] + if isinstance(node, ast.ImportFrom): + return [node.module] if node.level == 0 and node.module else [] + if (isinstance(node, ast.Call) and node.args + and isinstance(node.args[0], ast.Constant) + and isinstance(node.args[0].value, str) + and getattr(node.func, "attr", getattr(node.func, "id", "")) + in ("import_module", "__import__")): + return [node.args[0].value] + return [] + + +@pytest.mark.parametrize("module", sorted(SEAM_MODULES)) +def test_the_seam_names_no_sibling_in_any_import_late_or_not(module: str) -> None: + """No import anywhere in either module, at module scope or inside a + function body, names a sibling. A late one is still a reach, and it is the + class F4.1's scan exists for.""" + tree = ast.parse(SEAM_MODULES[module].read_text(encoding="utf-8")) + reaches = sorted({name for node in ast.walk(tree) + for name in _named_imports(node) + if any(name == s or name.startswith(s + ".") + for s in SIBLINGS)}) + assert reaches == [], f"{module} reaches {reaches}" + + +# -------------------------------------------------------------------------- +# 2 — nothing registered: the seam refuses, naming itself and the call (4.2) +# -------------------------------------------------------------------------- + +def test_nothing_registered_refuses_naming_the_seam_and_the_call() -> None: + with pytest.raises(gs.GeneratorNotRegistered) as caught: + gs.current() + message = str(caught.value) + for expected in (gs.REGISTRATION_CALL, "opendox.generator_seam", + "opendox.default_generator", "ENTRY POINT", + "cli.build_parser()", "cli.main()", + "serve.build_server()", "serve.main()", + "R1Q10 (a)", "5850003126", "unregister()"): + assert expected in message, f"the refusal no longer says {expected!r}" + + +def test_generate_with_nothing_registered_refuses_as_the_seam(tmp_path) -> None: + """Never a fallback to openDox's own generator, and never an empty snapshot.""" + with pytest.raises(gs.GeneratorNotRegistered) as caught: + gs.generate(tmp_path, "fixture") + assert isinstance(caught.value, gs.GeneratorSeamError) + assert gs.is_registered() is False + + +# -------------------------------------------------------------------------- +# 3 — the declaration is checked when it is made +# -------------------------------------------------------------------------- + +@pytest.mark.parametrize("contract,error", ( + (None, TypeError), (3, TypeError), ("", ValueError), (" padded ", ValueError))) +def test_a_declaration_names_the_contract_it_writes(contract, error) -> None: + with pytest.raises(error): + gs.SnapshotGenerator(contract=contract, generate=default_generator.generate) + + +@pytest.mark.parametrize("operation", (None, "not callable", 3)) +def test_a_declaration_carries_a_callable_operation(operation) -> None: + with pytest.raises(TypeError): + gs.SnapshotGenerator(contract="stand-in-snapshot", generate=operation) + + +@pytest.mark.parametrize("inputs,error", ( + (["project_register_source"], TypeError), + ((1,), TypeError), + (("not an identifier",), ValueError), + (("class",), ValueError), + (("repository",), ValueError), + (("generated_at",), ValueError), + (("register", "register"), ValueError))) +def test_declared_inputs_are_names_beyond_the_operations_own_four(inputs, + error) -> None: + def operation(repo_root, repository, **kw): + return {} + + with pytest.raises(error): + gs.SnapshotGenerator(contract="stand-in-snapshot", generate=operation, + inputs=inputs) + + +def test_an_operation_that_cannot_take_the_seams_call_is_refused() -> None: + """Conformance clause 2, checked where the signature can be read.""" + with pytest.raises(TypeError) as caught: + gs.SnapshotGenerator(contract="stand-in-snapshot", + generate=lambda repo_root, repository: {}) + assert "cannot take the generator seam's call" in str(caught.value) + + def takes_the_four(repo_root, repository, *, source_revision=None, + generated_at=None): + return {} + + with pytest.raises(TypeError): + gs.SnapshotGenerator(contract="stand-in-snapshot", + generate=takes_the_four, + inputs=("project_register_source",)) + + +def test_a_governed_generators_shape_is_declarable() -> None: + """A consumer's generator keeps its own signature and declares its extras. + + This is the call shape of openXdox's `generator.generate_snapshot`, as T059 + will declare it, restated here because openDox may not import it. Its + keyword-only test hooks stay undeclared, and so the seam never passes them. + """ + def generate_snapshot(repo_root, repository, *, source_revision=None, + generated_at=None, git=None, generator_version="v", + project_register_source=None, possibles_source=None, + excluded_documents=None): + return {} + + declared = gs.SnapshotGenerator( + contract="governed-stand-in", generate=generate_snapshot, + inputs=("project_register_source", "possibles_source")) + assert declared.inputs == ("project_register_source", "possibles_source") + + +@pytest.mark.parametrize("candidate", (None, "a generator", default_generator.generate)) +def test_only_a_declaration_is_registered(candidate) -> None: + for call in (gs.register, gs.register_default): + with pytest.raises(TypeError): + call(candidate) + assert gs.is_registered() is False + + +# -------------------------------------------------------------------------- +# 4 — the operation handed over, and its answer checked +# -------------------------------------------------------------------------- + +def test_the_registered_generator_is_handed_the_operation(tmp_path) -> None: + declared, calls = _declared() + assert gs.register(declared) is declared + assert gs.current() is declared + snapshot = gs.generate(str(tmp_path), "fixture", source_revision="abc123", + generated_at="2026-09-27T00:00:00Z") + assert calls == [{"repo_root": tmp_path, "repository": "fixture", + "source_revision": "abc123", + "generated_at": "2026-09-27T00:00:00Z"}] + assert isinstance(calls[0]["repo_root"], Path), "repo_root is handed over as a Path" + assert snapshot == {"schema_version": 1, "kind": "stand-in-snapshot", + "repository": "fixture"} + + +def test_a_declared_input_is_passed_and_an_undeclared_one_refused(tmp_path) -> None: + declared, calls = _declared(inputs=("project_register_source",)) + gs.register(declared) + gs.generate(tmp_path, "fixture", + project_register_source=Path("register.yaml")) + assert calls[-1]["project_register_source"] == Path("register.yaml") + + with pytest.raises(gs.GeneratorInputRefused) as caught: + gs.generate(tmp_path, "fixture", possibles_source=Path("possibles.yaml")) + assert len(calls) == 1, "the generator was called with an undeclared input" + message = str(caught.value) + assert "possibles_source" in message and "project_register_source" in message + + +def test_an_input_given_as_none_is_not_passed(tmp_path) -> None: + """So a verb can hand over its options whether or not they were set.""" + declared, calls = _declared() + gs.register(declared) + gs.generate(tmp_path, "fixture", project_register_source=None, + possibles_source=None) + assert set(calls[-1]) == {"repo_root", "repository", "source_revision", + "generated_at"} + + +@pytest.mark.parametrize("answer", ( + ["not", "a", "snapshot"], + "a snapshot", + {"schema_version": 1, "kind": "another-snapshot"}, + {"schema_version": 1}, + {"kind": "stand-in-snapshot"}, + {"schema_version": "1", "kind": "stand-in-snapshot"}, + {"schema_version": True, "kind": "stand-in-snapshot"}, + {"schema_version": 1.0, "kind": "stand-in-snapshot"})) +def test_only_a_snapshot_of_the_declared_contract_comes_back(answer, + tmp_path) -> None: + """Conformance clause 4. Two generators writing two contracts can then + never both answer for one of them.""" + declared, calls = _declared(answer=answer) + gs.register(declared) + with pytest.raises(gs.GeneratorNotConformant) as caught: + gs.generate(tmp_path, "fixture") + assert calls, "the generator was never called" + assert "stand-in-snapshot" in str(caught.value) + + +# -------------------------------------------------------------------------- +# 5 — one registration +# -------------------------------------------------------------------------- + +def test_registering_the_same_declaration_twice_is_a_no_op() -> None: + declared, _ = _declared() + assert gs.register(declared) is declared + assert gs.register(declared) is declared + assert gs.current() is declared + + +def test_a_second_different_host_generator_is_refused() -> None: + first, _ = _declared("first-snapshot") + second, _ = _declared("second-snapshot") + gs.register(first) + with pytest.raises(gs.GeneratorAlreadyRegistered) as caught: + gs.register(second) + message = str(caught.value) + for expected in ("first-snapshot", "second-snapshot", "ONCE", + "unregister()"): + assert expected in message, f"the refusal no longer says {expected!r}" + assert gs.current() is first + + +def test_unregister_makes_a_deliberate_swap_explicit() -> None: + first, _ = _declared("first-snapshot") + second, _ = _declared("second-snapshot") + gs.register(first) + gs.unregister() + assert gs.is_registered() is False + assert gs.register(second) is second + + +# -------------------------------------------------------------------------- +# 6 — the entry points' default (R1Q10 (a), in R1Q3 (a)'s pattern) +# -------------------------------------------------------------------------- + +def test_openDoxs_own_generator_writes_the_neutral_contract() -> None: + """The conformance clause names T053's neutral kind for openDox's own.""" + declared = default_generator.GENERATOR + assert isinstance(declared, gs.SnapshotGenerator) + assert declared.contract == gs.NEUTRAL_SNAPSHOT_KIND + assert declared.inputs == () + assert declared.generate is default_generator.generate + assert list(inspect.signature(default_generator.generate).parameters) == \ + list(gs.OPERATION_ARGUMENTS) + + +def test_the_neutral_kind_carries_no_word_f5_3_forbids() -> None: + """The kind is a string value of every neutral snapshot, and F5.3 sweeps + every string value, lowercased, for its declared words.""" + pattern = re.compile(r"\b(" + "|".join(re.escape(w) for w in F5_3_WORDS) + r")\b") + assert not pattern.search(gs.NEUTRAL_SNAPSHOT_KIND.lower()), \ + gs.NEUTRAL_SNAPSHOT_KIND + + +def test_register_default_registers_only_where_nothing_is_registered() -> None: + host, _ = _declared("host-snapshot") + gs.register(host) + assert gs.register_default(default_generator.GENERATOR) is host + gs.unregister() + assert gs.register_default(default_generator.GENERATOR) is \ + default_generator.GENERATOR + other_default, _ = _declared(gs.NEUTRAL_SNAPSHOT_KIND) + assert gs.register_default(other_default) is default_generator.GENERATOR + + +def test_register_default_holds_the_default_to_the_neutral_contract() -> None: + """An entry point's default is openDox's own, so it writes the neutral kind. + Another contract is refused, whether or not anything is registered.""" + governed, _ = _declared("governed-stand-in") + with pytest.raises(gs.GeneratorNotConformant) as caught: + gs.register_default(governed) + assert gs.NEUTRAL_SNAPSHOT_KIND in str(caught.value) + assert gs.is_registered() is False + host, _ = _declared("host-snapshot") + gs.register(host) + with pytest.raises(gs.GeneratorNotConformant): + gs.register_default(governed) + assert gs.current() is host + + +def test_a_host_replaces_a_default_nothing_was_generated_from() -> None: + assert gs.register_default(default_generator.GENERATOR) is \ + default_generator.GENERATOR + host, _ = _declared("host-snapshot") + assert gs.register(host) is host + assert gs.current() is host + + +def test_a_host_after_a_generation_from_the_default_is_refused(tmp_path) -> None: + """R1Q3 (ii)'s reason, for the generator: one process would then write two + contracts. A stand-in writes the neutral kind here, because openDox's own + projection is T054's.""" + stand_in_default, _ = _declared(gs.NEUTRAL_SNAPSHOT_KIND) + gs.register_default(stand_in_default) + gs.generate(tmp_path, "fixture") + host, _ = _declared("host-snapshot") + with pytest.raises(gs.GeneratorAlreadyRegistered) as caught: + gs.register(host) + message = str(caught.value) + for expected in ("default generator", "generated", "BEFORE", "R1Q3 (ii)", + "RN-1 (a)", "5850003126", "unregister()"): + assert expected in message, f"the refusal no longer says {expected!r}" + assert gs.current() is stand_in_default + + +def test_asking_or_being_refused_is_not_a_generation(tmp_path) -> None: + """Only a generation closes the default's window. Asking which generator is + registered does not, and nor does an input refused before the call.""" + stand_in_default, calls = _declared(gs.NEUTRAL_SNAPSHOT_KIND) + gs.register_default(stand_in_default) + assert gs.is_registered() is True + assert gs.current() is stand_in_default + gs.name_of(gs.current()) + with pytest.raises(gs.GeneratorInputRefused): + gs.generate(tmp_path, "fixture", project_register_source=Path("r.yaml")) + assert calls == [] + host, _ = _declared("host-snapshot") + assert gs.register(host) is host + + +def test_unregister_clears_the_default_and_its_generation(tmp_path) -> None: + stand_in_default, _ = _declared(gs.NEUTRAL_SNAPSHOT_KIND) + gs.register_default(stand_in_default) + gs.generate(tmp_path, "fixture") + gs.unregister() + host, _ = _declared("host-snapshot") + assert gs.register(host) is host + + +def test_a_host_that_registers_the_default_itself_holds_a_hosts_registration() -> None: + """Which kind a registration is, is set by the call that MADE it.""" + gs.register(default_generator.GENERATOR) + assert gs.register_default(default_generator.GENERATOR) is \ + default_generator.GENERATOR + host, _ = _declared("host-snapshot") + with pytest.raises(gs.GeneratorAlreadyRegistered) as caught: + gs.register(host) + assert "a host's generator is already registered" in str(caught.value) + + +def test_openDoxs_own_generator_refuses_until_its_projection_lands(tmp_path) -> None: + """Plan 034 orders the seam (T052) before the projection (T054). So the + default refuses, naming itself and T054, and generates nothing. It never + answers an empty snapshot. T054 replaces this case with its own tests.""" + with pytest.raises(default_generator.NeutralProjectionNotBuilt) as caught: + default_generator.generate(tmp_path, "fixture") + message = str(caught.value) + for expected in (gs.NEUTRAL_SNAPSHOT_KIND, "T054", gs.REGISTRATION_CALL): + assert expected in message, f"the refusal no longer says {expected!r}" + assert isinstance(caught.value, gs.GeneratorSeamError) + gs.register_default(default_generator.GENERATOR) + with pytest.raises(default_generator.NeutralProjectionNotBuilt): + gs.generate(tmp_path, "fixture") + + +# -------------------------------------------------------------------------- +# 7 — each entry point registers openDox's own generator +# -------------------------------------------------------------------------- + +def _is_the_registration(stmt: ast.stmt) -> bool: + """`generator_seam.register_default(default_generator.GENERATOR)`, alone.""" + call = stmt.value if isinstance(stmt, ast.Expr) else None + return (isinstance(call, ast.Call) and isinstance(call.func, ast.Attribute) + and call.func.attr == "register_default" + and isinstance(call.func.value, ast.Name) + and call.func.value.id == "generator_seam" + and [ast.unparse(arg) for arg in call.args] + == ["default_generator.GENERATOR"] + and not call.keywords) + + +def _binds_the_seam(stmt: ast.stmt) -> bool: + """`from opendox import default_generator, generator_seam`.""" + return (isinstance(stmt, ast.ImportFrom) and stmt.level == 0 + and stmt.module == "opendox" + and {"default_generator", "generator_seam"} + <= {alias.asname or alias.name for alias in stmt.names}) + + +def _module_and_function(path: Path, function: str) -> tuple[list, list]: + """`path`'s module-level statements, and the body of its top-level `function`.""" + tree = ast.parse(path.read_text(encoding="utf-8")) + for node in tree.body: + if isinstance(node, ast.FunctionDef) and node.name == function: + return tree.body, node.body + raise AssertionError(f"{path.name} no longer defines {function}()") + + +@pytest.mark.parametrize("path,function", ( + (CLI, "build_parser"), (CLI, "main"), (SERVE, "build_server"), (SERVE, "main"))) +def test_each_entry_point_registers_openDoxs_own_generator_once(path: Path, + function: str) -> None: + module_body, body = _module_and_function(path, function) + registrations = [stmt for stmt in body if _is_the_registration(stmt)] + assert len(registrations) == 1, ( + f"{path.name}:{function}() makes {len(registrations)} registrations of " + "openDox's own generator, where R1Q10 (a) asks for exactly one " + "`generator_seam.register_default(default_generator.GENERATOR)`") + assert [stmt for stmt in module_body if _binds_the_seam(stmt)], ( + f"{path.name} registers openDox's own generator without importing it") + + +def test_build_parser_registers_openDoxs_own_generator_where_no_host_has() -> None: + from opendox import cli + + cli.build_parser() + assert gs.current() is default_generator.GENERATOR + + +def test_cli_main_registers_openDoxs_own_generator() -> None: + from opendox import cli + + with pytest.raises(SystemExit) as exited: + cli.main(["--help"]) + assert exited.value.code == 0 + assert gs.current() is default_generator.GENERATOR + + +def test_a_host_registered_first_is_kept_by_the_cli_entry_points() -> None: + from opendox import cli + + host, _ = _declared("host-snapshot") + gs.register(host) + cli.build_parser() + with pytest.raises(SystemExit): + cli.main(["--help"]) + assert gs.current() is host + + +@pytest.mark.parametrize("function", ("build_server", "main")) +def test_the_server_entry_points_register_openDoxs_own_generator(function: str) -> None: + """`serve.build_server()` and `serve.main()` cannot run in a lone checkout + until phase 2 routes their snapshot source (research R7). So the import + and the registration are lifted out of `serve.py` and executed: the tree's + own statements, not a paraphrase of them.""" + module_body, body = _module_and_function(SERVE, function) + lifted = [stmt for stmt in module_body if _binds_the_seam(stmt)] + \ + [stmt for stmt in body if _is_the_registration(stmt)] + module = ast.Module(body=lifted, type_ignores=[]) + exec(compile(ast.fix_missing_locations(module), str(SERVE), "exec"), {}) # noqa: S102 + assert gs.current() is default_generator.GENERATOR + host, _ = _declared("host-snapshot") + assert gs.register(host) is host, ( + "registering the default generated nothing, so a host still replaces it") + + +# -------------------------------------------------------------------------- +# 8 — `CorpusAdapter` stays closed at six members +# -------------------------------------------------------------------------- + +def test_corpus_adapter_stays_closed_at_six_members() -> None: + """The generator is handed over at its own seam, not as a seventh member.""" + assert corpus_adapter.OPERATIONS == ( + "resolve", "list_documents", "read", "classify", "check", "write_back") + assert set(corpus_adapter.CorpusAdapter.__protocol_attrs__) == \ + set(corpus_adapter.OPERATIONS) + assert not hasattr(corpus_adapter.CorpusAdapter, "generate") From 4ca45d27ac17a573cf0f5e5e304532b116b0b402 Mon Sep 17 00:00:00 2001 From: Brett Heap <1513478+brettheap@users.noreply.github.com> Date: Sun, 27 Sep 2026 17:35:47 +0000 Subject: [PATCH 03/34] T052: each declared input optional; a default's generation recorded once it answers Copilot's review of this PR at 834f8ead raised two findings. Both are addressed here. 1. A declared input the operation cannot do without was accepted when it was declared. SnapshotGenerator bound the seam's call with every declared input given. But generate() passes an input only when its caller has a value for it, so such an operation failed with a raw TypeError the first time the option was unset. The declaration now binds the call twice, once with every declared input given and once with none given, and every call the seam can make lies between the two. Conformance clause 2 now says that each declared input is optional. 2. The default's window closed before its operation ran. generate() recorded a generation from the entry point's default before the call. So a generation that failed shut every host out although it wrote nothing, and that included openDox's own refusal before T054. The record is now made once a conformant snapshot comes back. A generation that fails, or whose answer the seam refuses, records nothing. While a generation from the default is under way, a host's registration is refused, because that snapshot would come back after the swap. The early record gave that property. A count of the generations under way now holds it. The registry's bookkeeping moves under one threading.Lock, because serve.py's ThreadingHTTPServer answers each request on a thread of its own. generate() holds the lock only to resolve and mark, and to record the end, and never across the generator's call. So a generator may register, unregister or generate from inside its own call. Tests: 59 -> 66 cases. Against 834f8ead's seam logic, 8 of the 66 go red: the new declared-input case, the pre-T054 default case, the two wrote-nothing cases, the three under-way cases and the re-entrant case. The concurrency and re-entrancy cases run in a process of their own with a time limit, so a lock held across a call fails them rather than hanging the suite. 13 of 13 mutations are caught. Arc: neutral-product-standalone-operability Lane: openxfactory-4 (openXfactory-4-openDox_extraction) Co-Authored-By: Claude Opus 5.5 (1M context) --- src/opendox/generator_seam.py | 232 +++++++++++++++++++++---------- tests/test_generator_seam.py | 253 ++++++++++++++++++++++++++++++++-- 2 files changed, 394 insertions(+), 91 deletions(-) diff --git a/src/opendox/generator_seam.py b/src/opendox/generator_seam.py index 0e8edcb2..a2c8184b 100644 --- a/src/opendox/generator_seam.py +++ b/src/opendox/generator_seam.py @@ -64,8 +64,11 @@ contracts can never both claim to be the one conformant implementation, which is the failure 5.4 was raised against. 2. Its operation TAKES THE SEAM'S CALL: the four arguments, and every input it - declares. This is checked when the declaration is made, wherever the - callable's signature can be read. + declares, EACH ONE OPTIONAL, because the seam passes a declared input only + when its caller has a value for it. This is checked when the declaration is + made, wherever the callable's signature can be read. The call is bound once + with every declared input given and once with none given, and between them + the two binds cover every call the seam can make. 3. It DECLARES EVERY FURTHER INPUT it reads. An input that it does not declare is refused before the generator is called, and never dropped. 4. It ANSWERS A SNAPSHOT OF ITS DECLARED CONTRACT: a `dict` whose `kind` is @@ -96,13 +99,24 @@ is refused as `GeneratorAlreadyRegistered`. One process would otherwise write two contracts, and a reader could not tell which snapshot was which. This is the same reason `domain_profile` refuses a swap after a build - (R1Q3 (ii); RN-1 (a), comment `5850003126`). + (R1Q3 (ii); RN-1 (a), comment `5850003126`); +* WHILE a snapshot is being generated from the default, a host's registration + is refused too, because that snapshot would come back after the swap; +* a generation that fails, or whose answer the seam refuses, wrote nothing, + so it records nothing, and a host still replaces the default after it. ONE REGISTRATION. Registering the same declaration again is a no-op, so an idempotent host start is not punished. A different declaration over a host's is refused as `GeneratorAlreadyRegistered`. `unregister()` makes a deliberate swap explicit. +ONE LOCK. The registration and its records are kept under one lock. `serve.py` +answers each request on a thread of its own (a `ThreadingHTTPServer`), so a +generation can run beside a registration. `generate()` holds the lock only for +its bookkeeping, and never across the generator's own call. So a slow +generator holds up nobody else, and a generator that itself registers or +generates cannot deadlock the seam. + RESOLVE PER CALL. A caller generates through `generate()`, and does not hold `current()`'s answer across calls. So every caller in a process answers from the one registration that exists at the time. @@ -119,6 +133,7 @@ import inspect import keyword +import threading from dataclasses import dataclass from pathlib import Path from typing import Any, Callable @@ -190,7 +205,8 @@ class GeneratorAlreadyRegistered(GeneratorSeamError): ONE registration is the contract, as it is for the profile. A process whose snapshots came from two generators would write two contracts under one name. The entry point's default is replaceable only until a snapshot has - been generated from it. `unregister()` makes a deliberate swap explicit.""" + been generated from it, and not while one is being generated. + `unregister()` makes a deliberate swap explicit.""" class GeneratorInputRefused(GeneratorSeamError): @@ -244,8 +260,8 @@ class SnapshotGenerator: that is not a non-empty string, an operation that is not callable, an input name that is not an identifier (or is a keyword, a duplicate, or one of the operation's own four), and an operation whose signature cannot take the - seam's call. Frozen, and compared by identity, as `register()` compares - registrations.""" + seam's call, whether its declared inputs are given or not. Frozen, and + compared by identity, as `register()` compares registrations.""" contract: str generate: Callable[..., dict] @@ -291,17 +307,27 @@ def _refuse_an_operation_that_cannot_take_the_call(self) -> None: # The signature cannot be read (some builtins). The seam's call is # then checked when it is first made, and not here. return - try: - signature.bind(Path("."), "repository", source_revision=None, - generated_at=None, - **{name: None for name in self.inputs}) - except TypeError as exc: - raise TypeError( - f"{_qualified(self.generate)} cannot take the generator seam's " - "call, generate(repo_root, repository, *, source_revision=, " - "generated_at=" + "".join(f", {name}=" for name in self.inputs) - + f"): {exc}. A contributed generator takes the operation's own " - "four arguments and every input it declares") from None + # `generate()` passes a declared input only when its caller has a value + # for it. So the call is bound with every declared input given, which + # proves that each is taken, and with none given, which proves that + # each is optional. Every call the seam can make lies between the two. + calls = [("", dict.fromkeys(self.inputs))] + if self.inputs: + calls.append((" with none of its declared inputs given", {})) + for when, given in calls: + try: + signature.bind(Path("."), "repository", source_revision=None, + generated_at=None, **given) + except TypeError as exc: + raise TypeError( + f"{_qualified(self.generate)} cannot take the generator " + f"seam's call{when}, generate(repo_root, repository, *, " + "source_revision=, generated_at=" + + "".join(f", {name}=" for name in given) + + f"): {exc}. A contributed generator takes the operation's " + "own four arguments and every input it declares, and each " + "declared input is optional, because the seam passes one " + "only when its caller has a value for it") from None #: THE one registration, a host's or the entry point's default, or `None`. @@ -311,10 +337,23 @@ def _refuse_an_operation_that_cannot_take_the_call(self) -> None: #: (`register_default()`), rather than a host's own `register()`. _is_default: bool = False -#: Whether a snapshot has been generated from that default. Set by -#: `generate()`, before the default's operation is called. +#: Whether a snapshot has been generated from that default, meaning that its +#: operation has answered a snapshot the seam handed back. `generate()` sets it +#: once that snapshot has come back. A generation that failed, or whose answer +#: the seam refused, wrote nothing, and so records nothing. _generated_from_default: bool = False +#: How many generations from that default are under way: begun, and not yet +#: answered or failed. While one is, a host's registration is refused, as it is +#: after one, because the snapshot being generated would come back after the +#: swap. `unregister()` leaves it alone: it counts calls that are still +#: running, and each one takes itself off when it ends. +_default_generations_under_way: int = 0 + +#: Guards the four above. It is held only for bookkeeping, and never across a +#: generator's own call. +_lock = threading.Lock() + def name_of(generator: Any) -> str: """A generator's most nameable name, with its contract, for a refusal. @@ -346,43 +385,53 @@ def register(generator: SnapshotGenerator) -> SnapshotGenerator: Registering the SAME declaration again is a no-op. A DIFFERENT one over a host's registration raises `GeneratorAlreadyRegistered`. Over the entry - point's default it REPLACES the default while nothing has been generated - from it, and is refused once something has. Either way the host ends up - holding the one registration, or knows why it does not.""" + point's default it REPLACES the default while no snapshot has been + generated from it and none is being generated. It is refused once one has + been, or while one is. Either way the host ends up holding the one + registration, or knows why it does not.""" global _registered, _is_default, _generated_from_default _require_a_declaration(generator, "register()") - if _registered is generator: - # The same object again: a no-op, before any bookkeeping is touched. - return generator - if _registered is not None: - if not _is_default: - raise GeneratorAlreadyRegistered( - f"a host's generator is already registered at openDox's " - f"generator seam ({name_of(_registered)}), and " - f"{name_of(generator)} would replace it. Registration happens " - "ONCE, at process start: one process generating through two " - "generators would write two contracts, and a reader could not " - "tell which snapshot was which. Call " - "opendox.generator_seam.unregister() first if the swap is " - "deliberate.") - if _generated_from_default: - raise GeneratorAlreadyRegistered( - f"openDox's own default generator ({name_of(_registered)}) is " - "registered, because an entry point registered it where no host " - "had, and a snapshot has already been generated from it, so " - f"{name_of(generator)} cannot replace it now. A swap would leave " - "one process writing two contracts. A host's generator replaces " - "the default only BEFORE anything is generated from it, as a " - "host's profile replaces the default profile only before a build " - "(R1Q3 (ii), openxFactory#656 comment 5817152735; RN-1 (a), " - "comment 5850003126). So register the host's generator at " - "process start, ahead of the first generation. Call " - "opendox.generator_seam.unregister() first if the swap is " - "deliberate.") - _registered = generator - _is_default = False - _generated_from_default = False - return generator + with _lock: + held = _registered + if held is generator: + # The same object again: a no-op, before any bookkeeping is touched. + return generator + over_a_host = held is not None and not _is_default + why = "" + if held is not None and _is_default: + if _default_generations_under_way: + why = "a snapshot is being generated from it now" + elif _generated_from_default: + why = "a snapshot has already been generated from it" + if not over_a_host and not why: + _registered = generator + _is_default = False + _generated_from_default = False + return generator + # Named outside the lock: naming a generator can run its own code. + if over_a_host: + raise GeneratorAlreadyRegistered( + f"a host's generator is already registered at openDox's " + f"generator seam ({name_of(held)}), and " + f"{name_of(generator)} would replace it. Registration happens " + "ONCE, at process start: one process generating through two " + "generators would write two contracts, and a reader could not " + "tell which snapshot was which. Call " + "opendox.generator_seam.unregister() first if the swap is " + "deliberate.") + raise GeneratorAlreadyRegistered( + f"openDox's own default generator ({name_of(held)}) is " + "registered, because an entry point registered it where no host " + f"had, and {why}, so " + f"{name_of(generator)} cannot replace it now. A swap would leave " + "one process writing two contracts. A host's generator replaces " + "the default only BEFORE anything is generated from it, as a " + "host's profile replaces the default profile only before a build " + "(R1Q3 (ii), openxFactory#656 comment 5817152735; RN-1 (a), " + "comment 5850003126). So register the host's generator at " + "process start, ahead of the first generation. Call " + "opendox.generator_seam.unregister() first if the swap is " + "deliberate.") def register_default(generator: SnapshotGenerator) -> SnapshotGenerator: @@ -406,22 +455,26 @@ def register_default(generator: SnapshotGenerator) -> SnapshotGenerator: f"the neutral snapshot contract {NEUTRAL_SNAPSHOT_KIND!r}, and " f"{name_of(generator)} declares another. A host's generator is " f"registered with {REGISTRATION_CALL}.") - if _registered is None: - _registered = generator - _is_default = True - _generated_from_default = False - return _registered + with _lock: + if _registered is None: + _registered = generator + _is_default = True + _generated_from_default = False + return _registered def unregister() -> None: """Drop the registration, a host's or the entry point's default. For test isolation and for a host tearing down. The record of a generation - from the default goes with it.""" + from the default goes with it. A generation still under way is not stopped. + When it answers, it records its snapshot only if the same default is + registered at that moment.""" global _registered, _is_default, _generated_from_default - _registered = None - _is_default = False - _generated_from_default = False + with _lock: + _registered = None + _is_default = False + _generated_from_default = False def is_registered() -> bool: @@ -434,7 +487,8 @@ def current() -> SnapshotGenerator: It answers what is registered: a host's generator, or the default an entry point registered. It never falls back to the default itself, and asking - records nothing: only a generation closes the default's window.""" + records nothing. Only a generation closes the default's window, while it + runs and once it has answered.""" if _registered is None: raise GeneratorNotRegistered( "no snapshot generator is registered at openDox's generator seam " @@ -466,11 +520,22 @@ def generate(repo_root: Path | str, repository: str, *, called (`GeneratorInputRefused`). What the generator answers is handed back only if it is a snapshot of the contract the generator declared (`GeneratorNotConformant`). With nothing registered this refuses as - `current()` does.""" - global _generated_from_default - generator = current() + `current()` does. + + A generation from the entry point's default holds the default's window shut + while it runs. Once its snapshot has come back, it closes the window for + good (see `register()`). A generation that fails, or whose answer is + refused, reopens it.""" + global _default_generations_under_way given = {name: value for name, value in inputs.items() if value is not None} - undeclared = sorted(set(given) - set(generator.inputs)) + with _lock: + # One hold for the resolve and the mark, so that no registration can + # land between the generator this call resolves and its call. + generator = current() + undeclared = sorted(set(given) - set(generator.inputs)) + from_default = _is_default and not undeclared + if from_default: + _default_generations_under_way += 1 if undeclared: declared = ", ".join(generator.inputs) or "none" raise GeneratorInputRefused( @@ -479,17 +544,32 @@ def generate(repo_root: Path | str, repository: str, *, "input a generator does not declare is refused rather than dropped: " "a generator that silently ignored it would write a snapshot that " "looks as though the input had been read.") - if _is_default: - # Before the call, so a registration racing a first generation cannot - # slip in between them. - _generated_from_default = True - snapshot = generator.generate( - Path(repo_root), repository, source_revision=source_revision, - generated_at=generated_at, **given) - _refuse_a_snapshot_that_does_not_conform(generator, snapshot) + answered = False + try: + snapshot = generator.generate( + Path(repo_root), repository, source_revision=source_revision, + generated_at=generated_at, **given) + _refuse_a_snapshot_that_does_not_conform(generator, snapshot) + answered = True + finally: + if from_default: + _end_a_generation_from_the_default(generator, answered) return snapshot +def _end_a_generation_from_the_default(generator: SnapshotGenerator, + answered: bool) -> None: + """Take a generation from the default off the count of those under way. If + it answered a snapshot the seam handed back, record that one was generated. + The record is made only while the default it came from is still the one + registered, so after `unregister()` it records nothing against a host.""" + global _default_generations_under_way, _generated_from_default + with _lock: + _default_generations_under_way -= 1 + if answered and _registered is generator and _is_default: + _generated_from_default = True + + def _refuse_a_snapshot_that_does_not_conform(generator: SnapshotGenerator, snapshot: Any) -> None: if not isinstance(snapshot, dict): diff --git a/tests/test_generator_seam.py b/tests/test_generator_seam.py index d8e1bab6..9f9ca4ea 100644 --- a/tests/test_generator_seam.py +++ b/tests/test_generator_seam.py @@ -20,9 +20,10 @@ 2. NOTHING REGISTERED REFUSES, NAMING THE SEAM AND THE CALL (4.2's discipline). It never falls back to openDox's own generator. 3. THE DECLARATION IS CHECKED WHEN IT IS MADE. A contract that is not a name, - an operation that is not callable or cannot take the seam's call, and an - input that is not a name beyond the operation's own four are refused before - anything is registered. + an operation that is not callable or cannot take the seam's call (with its + declared inputs given, or with none of them given), and an input that is not + a name beyond the operation's own four are refused before anything is + registered. 4. THE OPERATION IS HANDED OVER AND ITS ANSWER CHECKED. The registered generator receives the four arguments and its declared inputs. An undeclared input is refused before the call. A `None` input is not passed. @@ -32,8 +33,10 @@ 6. THE ENTRY POINTS' DEFAULT, in R1Q3 (a)'s pattern. openDox's own generator writes the neutral kind and takes no input. `register_default()` registers it only where nothing is, and holds it to the neutral contract. A host - replaces it before a generation, and is refused after one. Until T054 lands, - the default refuses, naming itself. + replaces it before a generation, and after one that wrote nothing. A host is + refused while a generation runs, and after one that wrote a snapshot. A + generator may register or generate from inside its own call without + deadlocking the seam. Until T054 lands, the default refuses, naming itself. 7. EACH ENTRY POINT REGISTERS IT. `cli.build_parser()` and `cli.main()` run for real. `serve.build_server()` and `serve.main()` still cannot run in a lone checkout (research R7), so their registration is executed from their own @@ -90,17 +93,19 @@ def _isolated_registries(): The three registries an entry point writes are process-global by design: the generator seam, the profile and the home corpus. Each is saved whole, because each keeps more than its registration (whether it is the entry - point's default, and whether anything was generated or built from it), and - a restore through `register()` would hand an entry point's default back as - a host's. + point's default, and whether anything was generated or built from it, or + is being generated), and a restore through `register()` would hand an entry + point's default back as a host's. """ - seam = (gs._registered, gs._is_default, gs._generated_from_default) + seam = (gs._registered, gs._is_default, gs._generated_from_default, + gs._default_generations_under_way) profile = (domain_profile._registered, domain_profile._is_default, domain_profile._built_from_default) home = corpus_adapter._home_factory gs.unregister() yield - (gs._registered, gs._is_default, gs._generated_from_default) = seam + (gs._registered, gs._is_default, gs._generated_from_default, + gs._default_generations_under_way) = seam (domain_profile._registered, domain_profile._is_default, domain_profile._built_from_default) = profile corpus_adapter._home_factory = home @@ -130,11 +135,12 @@ def operation(repo_root, repository, *, source_revision=None, def _fresh_process(program: str) -> subprocess.CompletedProcess: """`program` in a fresh interpreter, with this checkout's `src` first. The registry is process-global, so a case about what a PROCESS meets runs in - one of its own.""" + one of its own. So does a case that could deadlock the seam's lock: the + time limit then fails it, where in this process it would hang the suite.""" return subprocess.run( [sys.executable, "-c", f"import sys; sys.path.insert(0, {str(SRC)!r})\n" + textwrap.dedent(program)], - capture_output=True, text=True, cwd=str(ROOT)) + capture_output=True, text=True, cwd=str(ROOT), timeout=120) # -------------------------------------------------------------------------- @@ -281,23 +287,66 @@ def takes_the_four(repo_root, repository, *, source_revision=None, inputs=("project_register_source",)) -def test_a_governed_generators_shape_is_declarable() -> None: +def test_a_declared_input_the_operation_cannot_do_without_is_refused() -> None: + """Conformance clause 2: each declared input is optional. The seam passes a + declared input only when its caller has a value for it. So an operation + that REQUIRES one would take the call while the option is set, and fail + with a raw `TypeError` the first time it is not. Such an operation is + refused when it is declared, whether it requires the input by keyword or by + position.""" + def requires_it_by_keyword(repo_root, repository, *, project_register_source, + source_revision=None, generated_at=None): + return {} + + def requires_it_by_position(repo_root, repository, project_register_source, + *, source_revision=None, generated_at=None): + return {} + + for operation in (requires_it_by_keyword, requires_it_by_position): + with pytest.raises(TypeError) as caught: + gs.SnapshotGenerator(contract="stand-in-snapshot", generate=operation, + inputs=("project_register_source",)) + message = str(caught.value) + for expected in ("with none of its declared inputs given", + "project_register_source", "optional"): + assert expected in message, ( + f"{operation.__name__}: the refusal no longer says {expected!r}") + + def takes_it_optionally(repo_root, repository, project_register_source=None, + *, source_revision=None, generated_at=None): + return {} + + gs.SnapshotGenerator(contract="stand-in-snapshot", generate=takes_it_optionally, + inputs=("project_register_source",)) + + +def test_a_governed_generators_shape_is_declarable(tmp_path) -> None: """A consumer's generator keeps its own signature and declares its extras. This is the call shape of openXdox's `generator.generate_snapshot`, as T059 will declare it, restated here because openDox may not import it. Its keyword-only test hooks stay undeclared, and so the seam never passes them. + It generates with its declared inputs unset and with one of them set. """ def generate_snapshot(repo_root, repository, *, source_revision=None, generated_at=None, git=None, generator_version="v", project_register_source=None, possibles_source=None, excluded_documents=None): - return {} + return {"schema_version": 1, "kind": "governed-stand-in", + "register": project_register_source, + "possibles": possibles_source, "git": git} declared = gs.SnapshotGenerator( contract="governed-stand-in", generate=generate_snapshot, inputs=("project_register_source", "possibles_source")) assert declared.inputs == ("project_register_source", "possibles_source") + gs.register(declared) + unset = gs.generate(tmp_path, "fixture", project_register_source=None, + possibles_source=None) + assert (unset["register"], unset["possibles"], unset["git"]) == (None, None, None) + one_set = gs.generate(tmp_path, "fixture", + project_register_source=Path("register.yaml")) + assert (one_set["register"], one_set["possibles"]) == (Path("register.yaml"), None) @pytest.mark.parametrize("candidate", (None, "a generator", default_generator.generate)) @@ -502,6 +551,174 @@ def test_unregister_clears_the_default_and_its_generation(tmp_path) -> None: assert gs.register(host) is host +@pytest.mark.parametrize("failure", ("raises", "answers another kind")) +def test_a_generation_from_the_default_that_wrote_nothing_is_not_a_generation( + failure: str, tmp_path) -> None: + """A generation that fails, or whose answer the seam refuses, wrote no + snapshot, so it leaves the default's window open. A host still replaces the + default after it, and nothing is left counted as under way.""" + if failure == "raises": + def operation(repo_root, repository, *, source_revision=None, + generated_at=None): + raise OSError("the checkout is not there") + + stand_in_default = gs.SnapshotGenerator( + contract=gs.NEUTRAL_SNAPSHOT_KIND, generate=operation) + expected = OSError + else: + stand_in_default, _ = _declared( + gs.NEUTRAL_SNAPSHOT_KIND, + answer={"schema_version": 1, "kind": "another-snapshot"}) + expected = gs.GeneratorNotConformant + gs.register_default(stand_in_default) + with pytest.raises(expected): + gs.generate(tmp_path, "fixture") + assert gs._default_generations_under_way == 0 + host, _ = _declared("host-snapshot") + assert gs.register(host) is host + + +#: A program for a fresh process. A generation from the default runs on a +#: thread of its own, as a request does in `serve.py`'s `ThreadingHTTPServer`. +#: A host's registration is tried while it runs and again once it has ended. +#: `OUTCOME` is how the generation ends. +_UNDER_WAY = """ + import threading + from opendox import generator_seam as gs + OUTCOME = OUTCOME_VALUE + started, release = threading.Event(), threading.Event() + + def operation(repo_root, repository, *, source_revision=None, generated_at=None): + started.set() + release.wait(30) + if OUTCOME == "raises": + raise RuntimeError("the checkout went away mid-generation") + kind = gs.NEUTRAL_SNAPSHOT_KIND if OUTCOME == "answers" else "another-snapshot" + return {"schema_version": 1, "kind": kind} + + def host_operation(repo_root, repository, *, source_revision=None, generated_at=None): + return {"schema_version": 1, "kind": "host-snapshot"} + + host = gs.SnapshotGenerator(contract="host-snapshot", generate=host_operation) + gs.register_default(gs.SnapshotGenerator(contract=gs.NEUTRAL_SNAPSHOT_KIND, + generate=operation)) + ended = [] + + def request(): + try: + gs.generate(".", "fixture") + ended.append("answered") + except Exception as exc: + ended.append(type(exc).__name__) + + def try_the_host(): + try: + gs.register(host) + return "registered" + except gs.GeneratorAlreadyRegistered as exc: + if "a snapshot is being generated from it now" in str(exc): + return "refused-under-way" + if "a snapshot has already been generated from it" in str(exc): + return "refused-generated" + return "refused-otherwise" + + worker = threading.Thread(target=request) + worker.start() + assert started.wait(30), "the generation never started" + during = try_the_host() + release.set() + worker.join(30) + assert not worker.is_alive(), "the generation never ended" + assert gs._default_generations_under_way == 0, gs._default_generations_under_way + print(during, ended[0], try_the_host()) +""" + + +@pytest.mark.parametrize("outcome,ends,host_after", ( + ("answers", "answered", "refused-generated"), + ("raises", "RuntimeError", "registered"), + ("answers another kind", "GeneratorNotConformant", "registered"))) +def test_a_host_is_refused_while_a_generation_from_the_default_is_under_way( + outcome: str, ends: str, host_after: str) -> None: + """While a snapshot is being generated from the default, a host's + registration is refused, because that snapshot would come back after the + swap. Whether the window stays shut afterwards depends on whether the + generation WROTE a snapshot. A conformant answer shuts it for good. A + failure, or an answer the seam refuses, reopens it. The case runs in a + process of its own (see `_fresh_process`).""" + done = _fresh_process(_UNDER_WAY.replace("OUTCOME_VALUE", repr(outcome))) + assert done.returncode == 0, done.stderr + assert done.stdout.split() == ["refused-under-way", ends, host_after] + + +def test_a_generator_may_register_or_generate_from_inside_its_own_call() -> None: + """The seam's lock is held only for its bookkeeping, and never across a + generator's call. So a generator that registers, unregisters or generates + from inside its own call cannot deadlock the seam, and the seam's records + stay true. A default swapped out from inside its own call records nothing + against the default that replaced it. The case runs in a process of its own + (see `_fresh_process`).""" + done = _fresh_process(""" + from opendox import generator_seam as gs + KIND = gs.NEUTRAL_SNAPSHOT_KIND + seen = [] + + def snapshot(repository): + return {"schema_version": 1, "kind": KIND, "repository": repository} + + def host_operation(repo_root, repository, *, source_revision=None, + generated_at=None): + return {"schema_version": 1, "kind": "host-snapshot"} + + host = gs.SnapshotGenerator(contract="host-snapshot", generate=host_operation) + + def nesting(repo_root, repository, *, source_revision=None, generated_at=None): + if repository == "outer": + try: + gs.register(host) + seen.append("host-registered-inside") + except gs.GeneratorAlreadyRegistered as exc: + seen.append("refused-inside" + if "is being generated from it now" in str(exc) + else "refused-otherwise") + seen.append(gs.generate(repo_root, "inner")["repository"]) + return snapshot(repository) + + gs.register_default(gs.SnapshotGenerator(contract=KIND, generate=nesting)) + seen.append(gs.generate(".", "outer")["repository"]) + try: + gs.register(host) + seen.append("host-registered-after") + except gs.GeneratorAlreadyRegistered: + seen.append("refused-after") + + def replacing(repo_root, repository, *, source_revision=None, + generated_at=None): + return snapshot(repository) + + replacement = gs.SnapshotGenerator(contract=KIND, generate=replacing) + + def swapping(repo_root, repository, *, source_revision=None, + generated_at=None): + gs.unregister() + gs.register_default(replacement) + return snapshot(repository) + + gs.unregister() + gs.register_default(gs.SnapshotGenerator(contract=KIND, generate=swapping)) + gs.generate(".", "swapped") + assert gs.current() is replacement + seen.append("host-registered-over-the-replacement" + if gs.register(host) is host else "not-registered") + assert gs._default_generations_under_way == 0 + print(" ".join(seen)) + """) + assert done.returncode == 0, done.stderr + assert done.stdout.split() == ["refused-inside", "inner", "outer", + "refused-after", + "host-registered-over-the-replacement"] + + def test_a_host_that_registers_the_default_itself_holds_a_hosts_registration() -> None: """Which kind a registration is, is set by the call that MADE it.""" gs.register(default_generator.GENERATOR) @@ -516,7 +733,9 @@ def test_a_host_that_registers_the_default_itself_holds_a_hosts_registration() - def test_openDoxs_own_generator_refuses_until_its_projection_lands(tmp_path) -> None: """Plan 034 orders the seam (T052) before the projection (T054). So the default refuses, naming itself and T054, and generates nothing. It never - answers an empty snapshot. T054 replaces this case with its own tests.""" + answers an empty snapshot. Because it wrote nothing, its refusal shuts no + host out: a host's generator still replaces it afterwards. T054 replaces + this case with its own tests.""" with pytest.raises(default_generator.NeutralProjectionNotBuilt) as caught: default_generator.generate(tmp_path, "fixture") message = str(caught.value) @@ -526,6 +745,10 @@ def test_openDoxs_own_generator_refuses_until_its_projection_lands(tmp_path) -> gs.register_default(default_generator.GENERATOR) with pytest.raises(default_generator.NeutralProjectionNotBuilt): gs.generate(tmp_path, "fixture") + host, _ = _declared("host-snapshot") + assert gs.register(host) is host, ( + "a refused generation from openDox's own generator wrote nothing, so " + "a host still replaces it") # -------------------------------------------------------------------------- From f52b11178c129f84508d7fb493496d953dda5abc Mon Sep 17 00:00:00 2001 From: Brett Heap <1513478+brettheap@users.noreply.github.com> Date: Sun, 27 Sep 2026 18:20:50 +0000 Subject: [PATCH 04/34] T051: the malformed fixture, one title-and-summary-are-text violation (plan 034) tests/fixtures/malformed/ carries two .md documents: notes-bee-boxes.md is an ordinary valid source (non-empty title and summary), and notes-empty-title.md declares `title:` with nothing after the colon -- present in the header, but an empty string once parsed -- while its `summary:` stays ordinary. That is exactly one violation of the neutral snapshot schema T053 adds (opensoft/openDox-spec#16 at cd49eb25): `documents[].title` and `.summary` are each `type: ["string", "null"], minLength: 1` (x-rule `title-and-summary-are-text`, "each non-empty text, or null"). A document that declares no title/summary line at all yields null, which is valid (openDox-spec's own no-front-matter example); an empty string is the one value that is neither null nor non-empty text. Holder decision: T054 copies title/summary verbatim, without coercing an empty declared value to null or excluding the document, so the violation survives unchanged into the generated snapshot. tests/fixtures/malformed/EXPECTED_RULE holds the violated rule's identifier verbatim, `title-and-summary-are-text`, for the future `opendox generate --strict` to name (spec.md AT-R1 scenario 2). T051 carries no falsifier of its own (tasks.md: "used by F7.2"); this PR adds only the fixture and the sentinel file. Arc: neutral-product-standalone-operability Lane: openxfactory-4 (openXfactory-4-openDox_extraction) Co-Authored-By: Claude Opus 5.5 (1M context) --- tests/fixtures/malformed/EXPECTED_RULE | 1 + tests/fixtures/malformed/notes-bee-boxes.md | 8 ++++++++ tests/fixtures/malformed/notes-empty-title.md | 7 +++++++ 3 files changed, 16 insertions(+) create mode 100644 tests/fixtures/malformed/EXPECTED_RULE create mode 100644 tests/fixtures/malformed/notes-bee-boxes.md create mode 100644 tests/fixtures/malformed/notes-empty-title.md diff --git a/tests/fixtures/malformed/EXPECTED_RULE b/tests/fixtures/malformed/EXPECTED_RULE new file mode 100644 index 00000000..b9af1223 --- /dev/null +++ b/tests/fixtures/malformed/EXPECTED_RULE @@ -0,0 +1 @@ +title-and-summary-are-text diff --git a/tests/fixtures/malformed/notes-bee-boxes.md b/tests/fixtures/malformed/notes-bee-boxes.md new file mode 100644 index 00000000..c530fb76 --- /dev/null +++ b/tests/fixtures/malformed/notes-bee-boxes.md @@ -0,0 +1,8 @@ +title: Setting up two new bee boxes +summary: Where the new hives will go, and how far from the path. + +# Setting up two new bee boxes + +Two new bee boxes go in along the back fence, angled away from the main +path so foragers do not cross where people walk. Each box gets a paving +slab underneath to keep it level and dry through the winter. diff --git a/tests/fixtures/malformed/notes-empty-title.md b/tests/fixtures/malformed/notes-empty-title.md new file mode 100644 index 00000000..4cf64104 --- /dev/null +++ b/tests/fixtures/malformed/notes-empty-title.md @@ -0,0 +1,7 @@ +title: +summary: A short summary is here; only the title line is declared and left empty. + +# A note with no title + +The title line above is present but carries no text after the colon. Its +summary is ordinary. Nothing else about this note is unusual. From 25fe7526f94342a022a1d36a1044bf3542284113 Mon Sep 17 00:00:00 2001 From: Brett Heap <1513478+brettheap@users.noreply.github.com> Date: Sun, 27 Sep 2026 21:19:08 +0000 Subject: [PATCH 05/34] T054 (work in progress): openDox's small neutral projection over CorpusAdapter (plan 034) A checkpoint commit, pushed on the coordinator's usage stop. It is not the finished task: four node-harness cases in tests/test_display_facet.py still carry the governed snapshot values in their fixtures and fail against the new defaults, and T054's own test file is not written yet. What this commit holds: - src/opendox/neutral_projection.py (new): the projection. It writes T053's opendox-snapshot, places a document by its neutral stage: key (reading a value outside the six role keys as a source and reporting it), copies title and summary verbatim, and applies the topic rule and the group rule its docstring names. - src/opendox/default_generator.py: generate() projects the home corpus instead of refusing. NeutralProjectionNotBuilt is retired, and so is its refusal case in tests/test_generator_seam.py. - src/opendox/display_profile.py and web/views/display.js: SNAPSHOT_VALUES' defaults become the neutral snapshot's values. display.js keeps its line count, so the web census row is unchanged. - src/opendox/runtime/local_git_adapter.py: leading_header() is public, NEUTRAL_FIELDS is declared, WorkingTreeCorpus defaults required_fields to it, and the suffix branch of classify reports missing fields when fields are required. Arc: neutral-product-standalone-operability Lane: openxfactory-4 (openXfactory-4-openDox_extraction) Co-Authored-By: Claude Opus 5.5 (1M context) --- src/opendox/default_generator.py | 102 +++-- src/opendox/display_profile.py | 19 +- src/opendox/neutral_projection.py | 490 +++++++++++++++++++++++ src/opendox/runtime/local_git_adapter.py | 98 ++++- src/opendox/web/views/display.js | 42 +- tests/test_generator_seam.py | 24 +- 6 files changed, 678 insertions(+), 97 deletions(-) create mode 100644 src/opendox/neutral_projection.py diff --git a/src/opendox/default_generator.py b/src/opendox/default_generator.py index f2b9cb69..3d516cab 100644 --- a/src/opendox/default_generator.py +++ b/src/opendox/default_generator.py @@ -8,20 +8,33 @@ the neutral snapshot contract, `generator_seam.NEUTRAL_SNAPSHOT_KIND` (plan 034's T053; R1Q11 (a)), and takes no input beyond the operation's own four. -ITS PROJECTION IS NOT BUILT YET, AND IT SAYS SO. Plan 034 orders the seam -before the projection. T054, openDox's small neutral projection over -`CorpusAdapter` (#1144's 5.1-5.3), comes after this task (T052). The entry -points' registration calls are this task's, because T054 edits neither -`cli.py` nor `serve.py`, whose single-writer order runs T052, then T055. So -`generate()` below refuses, naming itself and the task that builds it, and it -generates nothing. It never answers an empty snapshot, which would read exactly -like an honest one. +WHAT IT GENERATES. `generate()` reads the home corpus through the home-corpus +seam, `corpus_adapter.home()`, and projects it with openDox's own small +neutral projection, `opendox.neutral_projection` (plan 034's T054; #1144's +5.1-5.3). Standalone, the home corpus is the default the entry points register, +`local_git_adapter.WorkingTreeCorpus`, which is how the projection is bound to +`LocalGitCorpus` (5.2). With no home corpus registered, the seam's own refusal +names the call that is missing, and nothing is generated. + +ITS TWO ANCHORS. `source_revision` is recorded as the caller pins it, and +otherwise it is the revision the corpus resolved at, its git HEAD. The +projection refuses where there is neither. `generated_at` is recorded as the +caller gives it, and otherwise it is the committer date of that revision, +which `git` reads from the checkout (`git show -s --format=%cI`, the stamp the +generate verbs have always used). It is looked up only for a revision spelled +as a hexadecimal object id, so a revision is never handed to `git` where it +could read as an option. It is left out where the lookup fails. Nothing reads +the clock, so the same tree at the same anchors answers the same snapshot. + +WHAT IT REPORTS. The projection's notices go to standard error, one line each, +`notice: : `. A `stage:` value outside the +six role keys is one (R1Q13 (a)): the line names the document, the value and +the six keys, and the snapshot reads that document as a source. NO VERB REACHES IT YET. The generate verbs still call the consumer's generator, -through `consumer_reach`, until T055 routes them through the seam, and T055 -comes after T054. The refusal is therefore reachable only by a library caller -that generates through the seam directly. T054 replaces it with the -projection, and the declaration below keeps its contract and its inputs. +through `consumer_reach`, until T055 routes them through the seam (T055 comes +after T054, in `cli.py`'s and `serve.py`'s single-writer order). Until then a +library caller reaches it through the seam, `generator_seam.generate()`. HOW IT IS REGISTERED: by the entry points, and never at import (R1Q3 (a)'s pattern). `cli.build_parser()`, `cli.main()`, `serve.build_server()` and @@ -29,8 +42,11 @@ registers it only where nothing is registered. Importing this module registers nothing. -IMPORT WEIGHT. `opendox.generator_seam` and the standard library. So this -module imports with no extra installed and no sibling present. +IMPORT WEIGHT. `opendox.generator_seam`, `opendox.corpus_adapter`, +`opendox.neutral_projection` (which adds `opendox.display_profile`, +`opendox.path_slug` and `opendox.runtime.local_git_adapter`), and the standard +library. So this module imports with no extra installed and no sibling +present. A CREATED FILE: it has no row in openxFactory's `docs/opendox-carve-manifest.yaml`, because the manifest declares what LEAVES @@ -39,36 +55,58 @@ from __future__ import annotations +import re +import sys from pathlib import Path from typing import Any -from opendox import generator_seam +from opendox import corpus_adapter, generator_seam, neutral_projection +from opendox.runtime.local_git_adapter import GitCommandFailed, GitRunner -__all__ = ["GENERATOR", "NeutralProjectionNotBuilt", "generate"] +__all__ = ["GENERATOR", "generate"] +#: A revision spelled as a hexadecimal object id, abbreviated or whole. Only +#: such a revision is handed to `git` to read its date. +_OBJECT_ID = re.compile(r"[0-9a-fA-F]{4,64}") -class NeutralProjectionNotBuilt(generator_seam.GeneratorSeamError): - """openDox's own generator is declared, and its projection is not built. +#: `git show -s --format=%cI`'s shape: a strict ISO 8601 date-time with an +#: explicit offset, as the neutral schema's `generated-at-is-rfc3339` rule +#: admits it. +_COMMIT_DATE = re.compile( + r"(?!0000)[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-5][0-9]" + r"(?:Z|[+-][0-9]{2}:[0-9]{2})") - A subclass of the seam's own refusal, so a verb that reports the seam's - refusals reports this one too. It goes when plan 034's T054 lands the - projection.""" + +def _commit_date(location: str, revision: str) -> str | None: + """The committer date of `revision` in the checkout at `location`, or + None where `git` cannot answer one.""" + if not _OBJECT_ID.fullmatch(revision): + return None + try: + stamp = GitRunner(Path(location)).out( + "show", "-s", "--format=%cI", f"{revision}^{{commit}}", "--") + except (GitCommandFailed, OSError): + return None + text = stamp.decode("utf-8", "replace").strip() + return text if _COMMIT_DATE.fullmatch(text) else None def generate(repo_root: Path, repository: str, *, source_revision: str | None = None, generated_at: str | None = None) -> dict[str, Any]: - """The operation the seam hands over, for openDox's own generator. - - It REFUSES until plan 034's T054 builds the neutral projection. See the - module docstring for why the declaration lands first.""" - raise NeutralProjectionNotBuilt( - f"openDox's own generator is declared at the generator seam, writing " - f"{generator_seam.NEUTRAL_SNAPSHOT_KIND!r}, but its projection is not " - f"built at this commit (plan 034's T054 builds it), so nothing was " - f"generated for {repository!r} at {str(repo_root)!r}. A host that has " - f"a generator of its own registers it at process start with " - f"{generator_seam.REGISTRATION_CALL}.") + """The operation the seam hands over, for openDox's own generator: the + neutral snapshot of the home corpus at `repo_root`.""" + adapter, ref = corpus_adapter.home()(str(repo_root)) + corpus = adapter.resolve(ref) + anchor = source_revision if source_revision is not None else corpus.revision + if generated_at is None and anchor is not None: + generated_at = _commit_date(corpus.location, anchor) + projection = neutral_projection.project( + adapter, corpus, repository, + source_revision=anchor, generated_at=generated_at) + for notice in projection.notices: + print(f"notice: {notice}", file=sys.stderr) + return projection.snapshot #: openDox's OWN generator, as the entry points register it where no host has. diff --git a/src/opendox/display_profile.py b/src/opendox/display_profile.py index 18b2b2fc..821a2c7b 100644 --- a/src/opendox/display_profile.py +++ b/src/opendox/display_profile.py @@ -224,12 +224,25 @@ class C — *"replace every governance word in it with a placeholder and the fil #: A HOST MAY STILL OVERRIDE THEM, on the `values` block of the facet, for the #: descendant whose generator writes different words into the same schema. The #: default is openDox's declaration, not a domain's silence. +#: +#: THE DEFAULTS ARE openDox's OWN NEUTRAL SNAPSHOT'S VALUES (RULED R1Q11 (a), +#: openxFactory#656 comment `5850003126`; plan 034 T054, as T007's batch G +#: amends #1144's 5.3). openDox generates that snapshot itself +#: (`neutral_projection`), and T053's schema closes both enums: a document's +#: stage is one of the six station role keys, and a candidate's state is one +#: of `unselected`, `selected`, `declined` and `replaced`. So the product's own +#: views match its own snapshot with no facet declared. A source document's +#: stage is `source`, and a document that gathers others is at `grouping`, so +#: those two are the captured and the organized stage. The candidate's four +#: are NEUTRAL_DISPLAY's own candidate words below, role for role. No word is +#: re-authored. The governed snapshot's own values move to openXdox's facet, +#: on its `values` block (T060). SNAPSHOT_VALUES: dict[str, dict[str, str]] = { # `documents[].stage` — which pipeline column a source document sits in. - "document_stage": {"captured": "brainstorm", "organized": "staged"}, + "document_stage": {"captured": "source", "organized": "grouping"}, # `possibles[].state` — the candidate register's own four-state enum. - "register_state": {"captured": "latent", "proposed": "picked", - "retired": "rejected", "superseded": "superseded"}, + "register_state": {"captured": "unselected", "proposed": "selected", + "retired": "declined", "superseded": "replaced"}, } #: THE STAGING TEMPLATE'S CANONICAL HEADING ORDER — the last member of § 2.2 diff --git a/src/opendox/neutral_projection.py b/src/opendox/neutral_projection.py new file mode 100644 index 00000000..edb75a80 --- /dev/null +++ b/src/opendox/neutral_projection.py @@ -0,0 +1,490 @@ +"""openDox's OWN small neutral projection: a plain corpus, read through +`CorpusAdapter`, written as the neutral snapshot (plan 034 T054). + +WHAT IT REALIZES. #1144's boxes 5.1, 5.2 and 5.3, as plan 034's T054 reads +them. + +* 5.1. This is new code over the `CorpusAdapter` protocol, written to openDox's + own needs. It is not openXdox's generator re-expressed, and it imports + nothing of openXdox's or openxFactory's. +* 5.2. It reads whatever adapter the home-corpus seam hands it. Standalone, + that is openDox's own default, `local_git_adapter.WorkingTreeCorpus`, a + `LocalGitCorpus` (T022). `default_generator` makes that call. +* 5.3. It writes T053's neutral snapshot contract, `opendox-snapshot` + (openDox-spec's `contracts/schemas/opendox-snapshot.schema.yaml`; R1Q11 + (a), openxFactory#656 comment `5850003126`). Its values are the six station + role keys and the four candidate states, and `display_profile + .SNAPSHOT_VALUES` defaults to those same values, so openDox's views place + every card by them and render the six words. + +WHERE A DOCUMENT LANDS (RULED R1Q13 (a) with (c), same comment). A document +names its station with the neutral `stage:` key of its leading `Name: value` +header, which is `local_git_adapter.leading_header`'s block. That is the same +reader the default adapter's `classify` uses. The value must be one of the six +station role keys, `display_profile.STAGE_ROLES`. A document that declares no +`stage:` is a source. A `stage:` value outside the six, the empty value +included, is not a declaration either. The document is read as a source, and +the projection reports it, naming the document, the value and the six keys. So +no other value can reach the snapshot, whose schema admits only the six. + +THE TOPIC RULE. Every document carries `topics`. A document that declares a +`topics:` header carries the ones it lists, split on commas, with whitespace +collapsed and letters case-folded. A document that declares none carries the +words of its NAME, plus the words of the name of every other document it +MENTIONS. Its name is its `title:`, else its first `#` heading, else its file +name without the suffix. It mentions another document where its text holds +that document's name, or that document's file name, as a run of whole words. +A word is a run of three or more letters, case-folded, that is not one of +`STOPWORDS`. So the rule needs no front matter at all. AT-R1's repository (b) +(quickstart.md § 2) is three notes named by their headings, and a note that +mentions another shares that note's name as a topic. An entry the adapter +cannot classify is listed but never read, so it carries no topic and nothing +mentions it. + +GROUPS (the grouping station). A group forms wherever two or more sources +share a topic. Topics that the same sources share form one group, named by +those topics. A document that declares `stage: grouping` is a group of its +own: it gathers itself and every source that shares one of its topics. Each +group holds one edge per member document, naming the topics that matched. + +THE OTHER STATIONS. A declared candidate is a candidate. It is `unselected`, +because only an act selects, declines or replaces one, and it is claimed by +every group it shares a topic with. A declared selection is a selection, and +a declared submission or completion is a changes entry, `active` or +`archived` as `display_profile.STAGE_FIELDS` declares them. Each of the three +lists the document as its file. So a plain repository fills every station its +documents name, and no other. + +TITLE AND SUMMARY are copied from the header EXACTLY as the header gives them, +an empty value included, and `null` only where the document does not give the +key. They are never coerced, and no document is left out for lacking them +(the holder's ruling on T051: the malformed fixture's empty `title:` reaches +the snapshot and fails `title-and-summary-are-text`). + +DETERMINISTIC. The same tree at the same anchors answers the same snapshot. +Every list is ordered by path or sorted, and nothing reads the clock. +`generation.source_revision` is the revision the caller pins, else the one the +corpus resolved. `generation.generated_at` is recorded only as the caller +hands it, and `default_generator` hands the source revision's own commit date. + +A CREATED FILE: it has no row in openxFactory's +`docs/opendox-carve-manifest.yaml`, because the manifest declares what LEAVES +openxFactory and never what a destination assembles (RULED OQ-C). +""" + +from __future__ import annotations + +import re +from collections import Counter +from dataclasses import dataclass +from pathlib import PurePosixPath +from typing import Any + +from opendox import generator_seam +from opendox.corpus_adapter import CorpusAdapter, ResolvedCorpus +from opendox.display_profile import STAGE_FIELDS, STAGE_ROLES +from opendox.path_slug import slug +from opendox.runtime.local_git_adapter import leading_header + +__all__ = [ + "GENERATOR_VERSION", + "Notice", + "Projection", + "ProjectionRefused", + "SCHEMA_VERSION", + "STOPWORDS", + "project", +] + +#: The neutral contract's version, which its schema holds at `const: 1`. +SCHEMA_VERSION = 1 + +#: This projection's own name and version, written to `generation`. +GENERATOR_VERSION = "opendox-neutral-1" + +#: The neutral header keys this projection reads. `title` and `summary` are the +#: default adapter's small field set (`local_git_adapter.NEUTRAL_FIELDS`). +STAGE_KEY = "stage" +TOPICS_KEY = "topics" +TITLE_KEY = "title" +SUMMARY_KEY = "summary" + +#: The station a document lands in when it declares none (R1Q13 (c)). +SOURCE = "source" +GROUPING = "grouping" +CANDIDATE = "candidate" +SELECTION = "selection" + +#: A candidate's state until an act selects, declines or replaces it. +UNSELECTED = "unselected" + +#: `changes[].status` for the two stations that share the `changes` section, +#: read off `display_profile.STAGE_FIELDS` rather than restated. +CHANGE_STATUS: dict[str, str] = { + role: status for role, _section, status in STAGE_FIELDS if status} + +#: Common English words the topic rule never takes as a topic. Words shorter +#: than three letters are never topics anyway, so none of those is listed. +STOPWORDS: frozenset[str] = frozenset(""" + about above across actually after again against all along already also + always among and another any are around back been before behind being + below beside between beyond both but can cannot could did does doing done + down during each either else even ever every few for from further get + gets got had has have having her here hers herself him himself his how + inside into its itself just last less let like made make many may maybe + might more most much must myself near neither never new next none nor + not nothing now off often once one only onto other others ought our ours + ourselves out outside over own per perhaps quite rather really same say + see she should since some still such than that the their theirs them + themselves then there these they this those though three through thus + till too toward towards two under until upon use very via want was way + well were what whatever when where whether which while who whom whose why + will with within without would yet you your yours yourself yourselves +""".split()) + +#: A run of letters or digits: what the topic rule reads as one token. +_TOKEN = re.compile(r"[^\W_]+") + +#: An ATX heading (`# Roadmap`, `## Budget ##`), its text in group 1. +_ATX_HEADING = re.compile(r"^ {0,3}#{1,6}(?:[ \t]+(.*?))?(?:[ \t]+#+)?[ \t]*$") + +#: A fenced code block's opening or closing line. A `#` line inside one is code. +_FENCE = re.compile(r"^ {0,3}(`{3,}|~{3,})") + +#: The characters the neutral schema refuses in a path or a topic +#: (`path-is-repo-relative`, `topic-is-trimmed-text`). +_CONTROL = re.compile("[\u0000-\u001f\u007f-\u009f]") + + +class ProjectionRefused(generator_seam.GeneratorSeamError): + """The projection cannot write a snapshot the neutral contract admits. + + A subclass of the generator seam's own refusal, so a verb that reports the + seam's refusals reports this one too.""" + + +@dataclass(frozen=True) +class Notice: + """One thing the projection read otherwise than the document declared it. + + `document` is the document's path as the adapter lists it. The generate + verbs report each notice, and the snapshot never carries one.""" + + document: str + message: str + + def __str__(self) -> str: + return f"{self.document}: {self.message}" + + +@dataclass(frozen=True) +class Projection: + """The neutral snapshot, and what the projection had to report making it.""" + + snapshot: dict[str, Any] + notices: tuple[Notice, ...] + + +@dataclass +class _Document: + """One listed document, as the projection reads it.""" + + path: str + stage: str = SOURCE + title: str | None = None + summary: str | None = None + topics: tuple[str, ...] = () + read: bool = False + name: str = "" + tokens: tuple[str, ...] = () + declared_topics: tuple[str, ...] | None = None + + @property + def stem(self) -> str: + return PurePosixPath(self.path).stem or self.path + + +def _tokens(text: str) -> tuple[str, ...]: + return tuple(token.casefold() for token in _TOKEN.findall(text)) + + +def _is_word(token: str) -> bool: + return len(token) >= 3 and token.isalpha() and token not in STOPWORDS + + +def _words(text: str) -> set[str]: + return {token for token in _tokens(text) if _is_word(token)} + + +def _first_heading(text: str) -> str | None: + """The text of the first ATX heading outside a fenced code block.""" + fence: str | None = None + for line in text.splitlines(): + marker = _FENCE.match(line) + if fence is not None: + if (marker and marker.group(1)[0] == fence[0] + and len(marker.group(1)) >= len(fence)): + fence = None + continue + if marker: + fence = marker.group(1) + continue + heading = _ATX_HEADING.match(line) + if heading and (heading.group(1) or "").strip(): + return heading.group(1).strip() + return None + + +def _declared_topics(value: str) -> tuple[str, ...]: + """The topics a `topics:` header lists: comma-separated, each with its + whitespace collapsed and its letters case-folded, each once, sorted. A + topic holding a control character is dropped, since no snapshot can carry + it (`topic-is-trimmed-text`).""" + topics: set[str] = set() + for part in value.split(","): + topic = " ".join(part.replace("", " ").split()).casefold() + if topic and not _CONTROL.search(topic): + topics.add(topic) + return tuple(sorted(topics)) + + +def _unwritable(path: str) -> str | None: + """Why the neutral schema could not carry `path`, or None where it can. + + The schema's `path-is-repo-relative` rule, stated as reasons: git can list + a path that holds any of these, and the snapshot must not.""" + if not path: + return "it is empty" + if path.startswith("/"): + return "it starts with a slash" + if re.match(r"[A-Za-z]:", path): + return "it starts with a drive letter and a colon" + if "\\" in path: + return "it holds a backslash" + if _CONTROL.search(path): + return "it holds a control character" + if ".." in path.split("/"): + return "it has a .. segment" + if any("\ud800" <= char <= "\udfff" for char in path): + return "its name is not valid UTF-8" + return None + + +def _unique(base: str, taken: set[str]) -> str: + """`base`, or `base-2`, `base-3`, ... where an earlier entry holds it.""" + candidate, number = base, 2 + while candidate in taken: + candidate, number = f"{base}-{number}", number + 1 + taken.add(candidate) + return candidate + + +def _read_documents(adapter: CorpusAdapter, corpus: ResolvedCorpus, + notices: list[Notice]) -> list[_Document]: + """Every listed document the snapshot can carry, in path order, with its + station, its two copied fields, its name and its tokens.""" + stations = set(STAGE_ROLES) + documents: list[_Document] = [] + listed = sorted(adapter.list_documents(corpus), key=lambda d: d.key) + for identity in listed: + reason = _unwritable(identity.key) + if reason is not None: + # NAMED BY ITS repr: the path is one no snapshot can carry, and a + # control character or a lone surrogate in it must not reach a + # terminal raw either. + notices.append(Notice( + repr(identity.key), + f"is listed, but the neutral snapshot cannot carry its path " + f"({reason}), so it is left out")) + continue + document = _Document(path=identity.key) + documents.append(document) + if adapter.classify(corpus, identity).kind is None: + # UNCLASSIFIABLE: listed, never read. It is a source with nothing + # to copy, no topic, and no name another document can mention. + continue + text = adapter.read(corpus, identity).content.decode("utf-8", "replace") + header = leading_header(text) + document.read = True + document.title = header.get(TITLE_KEY) + document.summary = header.get(SUMMARY_KEY) + declared = header.get(STAGE_KEY) + if declared is not None: + if declared in stations: + document.stage = declared + else: + notices.append(Notice( + identity.key, + f"its stage: value {declared!r} is not one of the six " + f"station role keys ({', '.join(STAGE_ROLES)}), so it is " + "not a declaration; the document is read as a source")) + listed_topics = _declared_topics(header.get(TOPICS_KEY) or "") + document.declared_topics = listed_topics or None + document.name = (document.title or _first_heading(text) + or document.stem) + document.tokens = _tokens(text) + return documents + + +def _assign_topics(documents: list[_Document]) -> None: + """The topic rule (this module's docstring), over the documents read.""" + read = [document for document in documents if document.read] + own = [_words(document.name) for document in read] + # Each name is looked for by its first WORD, at that word's offset, so a + # text is scanned once and a common short token ("the") costs nothing. + by_first_word: dict[str, list[tuple[tuple[str, ...], int, int]]] = {} + for index, document in enumerate(read): + for alias in {_tokens(document.name), _tokens(document.stem)}: + # A name with no word in it contributes no topic, so it is not + # looked for: `a.md` would otherwise be mentioned by every "a". + offset = next((at for at, token in enumerate(alias) + if _is_word(token)), None) + if offset is not None: + by_first_word.setdefault(alias[offset], []).append( + (alias, offset, index)) + for index, document in enumerate(read): + if document.declared_topics is not None: + document.topics = document.declared_topics + continue + mentioned: set[int] = set() + tokens = document.tokens + for position, token in enumerate(tokens): + for alias, offset, other in by_first_word.get(token, ()): + start = position - offset + if (other != index and other not in mentioned and start >= 0 + and tokens[start:start + len(alias)] == alias): + mentioned.add(other) + topics = set(own[index]) + for other in mentioned: + topics |= own[other] + document.topics = tuple(sorted(topics)) + + +def _groups(documents: list[_Document], notices: list[Notice]) -> list[dict]: + """The grouping station: each declared group, then each derived one.""" + sources = [document for document in documents if document.stage == SOURCE] + taken: set[str] = set() + clusters: list[dict] = [] + for document in documents: + if document.stage != GROUPING: + continue + if not document.topics: + notices.append(Notice( + document.path, + "declares stage grouping but carries no topic, so no group " + "forms around it")) + continue + mine = set(document.topics) + members = [document] + [source for source in sources + if mine & set(source.topics)] + members.sort(key=lambda member: member.path) + clusters.append({ + "id": _unique(slug(document.stem), taken), + "name": document.name, + "topics": list(document.topics), + "document_edges": [ + {"document": member.path, + "matched_topics": sorted(mine & set(member.topics))} + for member in members], + }) + carriers: dict[str, list[str]] = {} + for source in sources: + for topic in source.topics: + carriers.setdefault(topic, []).append(source.path) + shared: dict[tuple[str, ...], list[str]] = {} + for topic, paths in carriers.items(): + if len(paths) >= 2: + shared.setdefault(tuple(paths), []).append(topic) + for topics, paths in sorted((tuple(sorted(topics)), paths) + for paths, topics in shared.items()): + name = " · ".join(topics) + clusters.append({ + "id": _unique(slug(name), taken), + "name": name, + "topics": list(topics), + "document_edges": [ + {"document": path, "matched_topics": list(topics)} + for path in sorted(paths)], + }) + return clusters + + +def project(adapter: CorpusAdapter, corpus: ResolvedCorpus, repository: str, + *, source_revision: str | None = None, + generated_at: str | None = None) -> Projection: + """The neutral snapshot of `corpus`, read through `adapter`. + + `source_revision`, where given, is recorded verbatim as the determinism + anchor; otherwise the revision `corpus` was resolved at is. With neither, + the projection refuses, since the contract requires an anchor. + `generated_at`, where given, is recorded verbatim, and it is otherwise + absent. `repository` is recorded as given.""" + revision = source_revision if source_revision is not None else corpus.revision + if revision is None: + raise ProjectionRefused( + f"the neutral snapshot of {repository!r} needs a source revision, " + f"and none was given, and the corpus at {corpus.location!r} " + "resolved to none (a repository with no commit yet). Commit the " + "documents, or pass the revision to anchor them to.") + notices: list[Notice] = [] + documents = _read_documents(adapter, corpus, notices) + _assign_topics(documents) + clusters = _groups(documents, notices) + + possibles: list[dict] = [] + staged_topics: list[dict] = [] + changes: list[dict] = [] + taken: dict[str, set[str]] = {"possibles": set(), "staged_topics": set(), + "changes": set()} + for document in documents: + if document.stage == CANDIDATE: + mine = set(document.topics) + candidate: dict[str, Any] = { + "id": _unique(slug(document.stem), taken["possibles"]), + "title": document.name, + } + if document.summary: + candidate["claim"] = document.summary + candidate["state"] = UNSELECTED + candidate["claiming_clusters"] = [ + cluster["id"] for cluster in clusters + if mine & set(cluster["topics"])] + possibles.append(candidate) + elif document.stage == SELECTION: + staged_topics.append({ + "staging_id": _unique(slug(document.stem), + taken["staged_topics"]), + "files": [document.path], + }) + elif document.stage in CHANGE_STATUS: + changes.append({ + "id": _unique(slug(document.stem), taken["changes"]), + "status": CHANGE_STATUS[document.stage], + "files": [document.path], + }) + + counts = Counter(topic for document in documents + for topic in document.topics) + generation: dict[str, Any] = {"source_revision": revision} + if generated_at is not None: + generation["generated_at"] = generated_at + generation["generator_version"] = GENERATOR_VERSION + snapshot: dict[str, Any] = { + "schema_version": SCHEMA_VERSION, + "kind": generator_seam.NEUTRAL_SNAPSHOT_KIND, + "repository": repository, + "generation": generation, + "documents": [ + {"id": document.path, "path": document.path, + "stage": document.stage, "title": document.title, + "summary": document.summary, "topics": list(document.topics)} + for document in documents], + "clusters": clusters, + "possibles": possibles, + "staged_topics": staged_topics, + "changes": changes, + "keyword_index": [ + {"keyword": keyword, "declared_doc_count": counts[keyword]} + for keyword in sorted(counts)], + } + return Projection(snapshot=snapshot, notices=tuple(notices)) diff --git a/src/opendox/runtime/local_git_adapter.py b/src/opendox/runtime/local_git_adapter.py index 2afdd9f8..de37dd3b 100644 --- a/src/opendox/runtime/local_git_adapter.py +++ b/src/opendox/runtime/local_git_adapter.py @@ -140,9 +140,11 @@ DEFAULT_BRANCH = "main" #: What this corpus can say about a document's SHAPE, which is all a plain git -#: repository knows. Every kind obliges NO fields: obligations are governance, -#: and a pre-governed repository has none — that is the whole of what makes -#: this the trivial implementation rather than a small governed one. +#: repository knows. By default every kind obliges NO fields: obligations are +#: governance, and a pre-governed repository has none — that is the whole of +#: what makes this the trivial implementation rather than a small governed one. +#: A corpus constructed with `required_fields` obliges those; openDox's own +#: standalone default obliges `NEUTRAL_FIELDS` below, and nothing more. KINDS_BY_SUFFIX: dict[str, str] = { ".md": "text", ".markdown": "text", @@ -160,6 +162,16 @@ #: near its top, so classifying one is never a scan of its whole body. MAX_HEADER_LINES = 64 +#: THE SMALL NEUTRAL FIELD SET (RULED R1Q13 (a), openxFactory#656 comment +#: `5850003126`, in the answer's own example; plan 034 T054): what openDox's own +#: standalone default adapter, `WorkingTreeCorpus`, obliges of a document it +#: recognizes, so `authoring.required_header_fields()` answers it. They are +#: the `title` and `summary` of T053's neutral snapshot, which openDox's own +#: projection copies from the same header (`leading_header`). A document +#: without them is still listed and read, as a source: a missing field is +#: reported, never a reason to refuse or drop the document. +NEUTRAL_FIELDS: tuple[str, ...] = ("title", "summary") + #: The corpus's ONE verdict of its own, and it is a fact about git rather than #: a rule about documents: RULING C3 says "documents are always git-backed", so #: a tracked file whose checkout differs from the resolved commit is a document @@ -202,6 +214,30 @@ def _leading_lines(text: str, limit: int) -> Iterator[str]: start = boundary.end() +def leading_header(text: str) -> dict[str, str]: + """The leading `Name: value` block of `text`: this corpus's one header + convention, as a mapping from each name to its stripped value. + + The block is the run of lines up to the first blank one, and at most + `MAX_HEADER_LINES` lines are looked at. A line with no colon is passed + over, a later line repeating a name wins, and a name given with nothing + after its colon maps to the empty string. So a key that is present with + no value is still told apart from one that is absent. + + PUBLIC because openDox's own neutral projection reads the same block + (`opendox.neutral_projection`, plan 034 T054). The fields it copies are + the ones `classify` below reports as missing, so the two cannot disagree + about what a document gives: both call this function.""" + header: dict[str, str] = {} + for line in _leading_lines(text, MAX_HEADER_LINES): + if not line.strip(): + break + name, separator, value = line.partition(":") + if separator: + header[name.strip()] = value.strip() + return header + + #: Environment variables that SELECT A REPOSITORY or inject configuration, and #: which are therefore stripped from every `git` this package runs. #: @@ -1906,10 +1942,17 @@ def classify(self, corpus: ResolvedCorpus, (RULED openxFactory#656 comment 5714365086, Q-F2 (a) — see `__init__`). WITHOUT `kind_field` — the default, and what this class has always done - — the shape is the file SUFFIX and `required_fields` is empty for every - kind. That is the honest answer for a plain local git repository rather - than an unfinished one: obligations are governance, and a pre-governed - repository has none. + — the shape is the file SUFFIX, and with the default `required_fields` + of `()` no kind obliges any field. That is the honest answer for a + plain local git repository rather than an unfinished one: obligations + are governance, and a pre-governed repository has none. A corpus built + WITH `required_fields` obliges those of every kind it recognizes, and + reports as missing each one the document's header does not give, or + gives with nothing after its colon (an empty value is no field, as + `authoring.missing_required_headers` has it). openDox's own standalone + default, `WorkingTreeCorpus`, is built with `NEUTRAL_FIELDS` (plan 034 + T054, R1Q13 (a)). A missing field is reported and nothing more: the + document is still listed, still readable, and still classified. WITH `kind_field` the corpus declares its own shape in a document HEADER, which is how the neutral conformance corpus is written @@ -1923,7 +1966,6 @@ def classify(self, corpus: ResolvedCorpus, """ if self._kind_field is not None: return self._classify_by_header(corpus, document) - del corpus suffix = Path(document.key).suffix.lower() kind = KINDS_BY_SUFFIX.get(suffix) if kind is None: @@ -1933,8 +1975,16 @@ def classify(self, corpus: ResolvedCorpus, f"{document.key!r} has no shape this corpus recognizes " f"(suffix {suffix or '(none)'!r}); it is still listed and " "still readable")) - return Classification(id=document, kind=kind, required_fields=(), - missing_fields=()) + if not self._required_fields: + # NOTHING OBLIGED, NOTHING READ: the bare default answers from the + # suffix alone, exactly as it always has. + return Classification(id=document, kind=kind, required_fields=(), + missing_fields=()) + header = self._header_of(corpus, document) + return Classification( + id=document, kind=kind, required_fields=self._required_fields, + missing_fields=tuple(field for field in self._required_fields + if not header.get(field))) def _classify_by_header(self, corpus: ResolvedCorpus, document: DocumentId) -> Classification: @@ -1975,14 +2025,7 @@ def _header_of(self, corpus: ResolvedCorpus, yields the same lines and stops. """ text = self.read(corpus, document).content.decode("utf-8", "replace") - header: dict[str, str] = {} - for line in _leading_lines(text, MAX_HEADER_LINES): - if not line.strip(): - break - name, separator, value = line.partition(":") - if separator: - header[name.strip()] = value.strip() - return header + return leading_header(text) # -- check ------------------------------------------------------------ @@ -3010,7 +3053,24 @@ class WorkingTreeCorpus(LocalGitCorpus): visible on the very next one. The factory that registers this default (`cli.py`'s and `serve.py`'s `_default_home_factory`) also constructs a fresh instance on every call, so no state survives across registrations - either (plan 034, T022).""" + either (plan 034, T022). + + THE SMALL NEUTRAL FIELD SET IS THIS CLASS'S DEFAULT (plan 034 T054; RULED + R1Q13 (a), openxFactory#656 comment `5850003126`). `required_fields` + defaults to `NEUTRAL_FIELDS`, `title` and `summary`, where + `LocalGitCorpus` defaults to `()`. So the factory above, which builds + `WorkingTreeCorpus()`, hands every standalone caller an adapter whose + `classify` obliges them, and `authoring.required_header_fields()` answers + them. Nothing else about the class changes: a document without them is + still listed and read, and `classify` reports what is missing.""" + + def __init__(self, *, executable: str = "git", + write_path: str | None = WRITE_PATH, + kind_field: str | None = None, + required_fields: tuple[str, ...] = NEUTRAL_FIELDS) -> None: + super().__init__(executable=executable, write_path=write_path, + kind_field=kind_field, + required_fields=required_fields) def _list_documents_bound(self, git: GitRunner, corpus: ResolvedCorpus, scope: str) -> tuple[DocumentId, ...]: diff --git a/src/opendox/web/views/display.js b/src/opendox/web/views/display.js index 309ab246..7b4576f6 100644 --- a/src/opendox/web/views/display.js +++ b/src/opendox/web/views/display.js @@ -220,29 +220,29 @@ export const SESSION_BRANCH_NAMESPACES = { [SCOPE_KINDS.candidate]: "possible", }; -// REGISTER_STATES — the candidate register's own four-state enum, by status -// role. The fifth SEAM KEY table on § 2.2 rule 3's footing: these are VALUES -// the snapshot carries on a candidate (`possibles[].state`), written by the -// generator and compared against here, never rendered — what a human reads is -// `display.status(VOCABULARY.CANDIDATE, role)`. openxFactory's own profile -// declares the same four against the same roles -// (`contracts/domain-profiles/openxfactory-engineering.yaml`:294-297), which is -// what makes the role the right key and the word the wrong one. -// THE SNAPSHOT'S CLOSED ENUM VALUES, by role — the rest of § 2.2 rule 3's -// schema half, mirrored from `display_profile.SNAPSHOT_VALUES` and carried on -// the payload (`values`) so a host may override them. -// -// `documents[].stage` and `possibles[].state` are values the renderer MATCHES — +// THE SNAPSHOT'S CLOSED ENUM VALUES, by role — the fifth SEAM KEY table on +// § 2.2 rule 3's footing and the rest of its schema half, mirrored from +// `display_profile.SNAPSHOT_VALUES` and carried on the payload (`values`) so a +// host may override them. `documents[].stage` and `possibles[].state`, the +// candidate register's four-state enum, are values the renderer MATCHES — // which column a card belongs in, which dot a candidate wears — and never -// values it RENDERS: every word a human reads comes from `display.status(...)`. -// That is § 2.2 rule 3 read over a closed enum rather than over a field name, -// and it is what keeps the product working: every profile in the estate today -// registers WITHOUT a `DISPLAY` facet, so a board that filtered on the facet's -// neutral word would show an empty column against the very snapshot it renders. +// values it RENDERS: every word a human reads comes from `display.status(...)`, +// by role. That is § 2.2 rule 3 read over a closed enum rather than over a +// field name, and it is what keeps the product working: a board that filtered +// on a facet's word would show an empty column against the snapshot it renders. +// +// The defaults are openDox's own neutral snapshot's values (R1Q11 (a); plan +// 034 T054), exactly as `display_profile.SNAPSHOT_VALUES` declares them: a +// source document's stage is `source` and a gathering one's is `grouping`, and +// a candidate is `unselected`, `selected`, `declined` or `replaced`, which are +// the candidate words NEUTRAL_DISPLAY renders, role for role. A host whose +// generator writes other values into the same schema declares them on its +// facet's `values` block, as openXdox's facet does for the governed snapshot +// (T060). export const SNAPSHOT_VALUES = { - document_stage: { captured: "brainstorm", organized: "staged" }, - register_state: { captured: "latent", proposed: "picked", - retired: "rejected", superseded: "superseded" }, + document_stage: { captured: "source", organized: "grouping" }, + register_state: { captured: "unselected", proposed: "selected", + retired: "declined", superseded: "replaced" }, }; // DRILL_KINDS — the explorer's own tile-kind keys, by stage role. The sixth and diff --git a/tests/test_generator_seam.py b/tests/test_generator_seam.py index 9f9ca4ea..40face7b 100644 --- a/tests/test_generator_seam.py +++ b/tests/test_generator_seam.py @@ -36,7 +36,8 @@ replaces it before a generation, and after one that wrote nothing. A host is refused while a generation runs, and after one that wrote a snapshot. A generator may register or generate from inside its own call without - deadlocking the seam. Until T054 lands, the default refuses, naming itself. + deadlocking the seam. What the default generates is T054's neutral + projection, and `tests/test_neutral_projection.py` holds it. 7. EACH ENTRY POINT REGISTERS IT. `cli.build_parser()` and `cli.main()` run for real. `serve.build_server()` and `serve.main()` still cannot run in a lone checkout (research R7), so their registration is executed from their own @@ -730,27 +731,6 @@ def test_a_host_that_registers_the_default_itself_holds_a_hosts_registration() - assert "a host's generator is already registered" in str(caught.value) -def test_openDoxs_own_generator_refuses_until_its_projection_lands(tmp_path) -> None: - """Plan 034 orders the seam (T052) before the projection (T054). So the - default refuses, naming itself and T054, and generates nothing. It never - answers an empty snapshot. Because it wrote nothing, its refusal shuts no - host out: a host's generator still replaces it afterwards. T054 replaces - this case with its own tests.""" - with pytest.raises(default_generator.NeutralProjectionNotBuilt) as caught: - default_generator.generate(tmp_path, "fixture") - message = str(caught.value) - for expected in (gs.NEUTRAL_SNAPSHOT_KIND, "T054", gs.REGISTRATION_CALL): - assert expected in message, f"the refusal no longer says {expected!r}" - assert isinstance(caught.value, gs.GeneratorSeamError) - gs.register_default(default_generator.GENERATOR) - with pytest.raises(default_generator.NeutralProjectionNotBuilt): - gs.generate(tmp_path, "fixture") - host, _ = _declared("host-snapshot") - assert gs.register(host) is host, ( - "a refused generation from openDox's own generator wrote nothing, so " - "a host still replaces it") - - # -------------------------------------------------------------------------- # 7 — each entry point registers openDox's own generator # -------------------------------------------------------------------------- From 0f42f678ec2de9bacf3eec5fbf385e826a29d76a Mon Sep 17 00:00:00 2001 From: Brett Heap <1513478+brettheap@users.noreply.github.com> Date: Sun, 27 Sep 2026 22:17:32 +0000 Subject: [PATCH 06/34] T054: the projection's tests, the holder's two rulings, and the display-facet cases (plan 034) This finishes the work 25fe7526 checkpointed. The holder ruled on two of T054's open questions, relayed by the coordinator: - An entry the adapter cannot classify (a Makefile, an image) is not a document. The projection now leaves it out unread, and nothing mentions it. - The wheel's grouping tile counts a group's edges, and the neutral schema is not widened with a tally. views/wheel-model.js now counts document_edges where a group carries no tallies.document_links, and uses the tally where one exists, as the governed snapshot's groups do. The file keeps its 1358 lines, so its census row is unchanged. tests/test_neutral_projection.py (new) holds T054's falsifier: - in a fresh process with every sibling blocked, the CLI entry point's defaults generate over T050's fixture through the seam, and the result validates against T053's schema with no F5.3 word; - the topic rule groups a copy of AT-R1's repository (b), which has no front matter; - a stage: value outside the six is reported, naming the document, the value and the six keys, and is read as a source. Around them it tests: - the holder's T051 rule (the malformed fixture breaks only its EXPECTED_RULE); - what a document is, the stations, and the anchors; - the default adapter's field set through the entry point; - the neutral display values in Python and in display.js, and the wheel's edge count; - that the projection makes no reach. The schema is a byte-identical copy of openDox-spec#16's at cd49eb25, held to its sha256, until T057 ships the packaged one. tests/test_display_facet.py: four node-harness cases carried the governed snapshot values in their fixtures. Each now reads the value openDox ships (SNAPSHOT_VALUES), and the property each one tests is unchanged. The canvas case's neutral assertion no longer compares against a value the neutral install can never write. It now asserts openDox's own word. Arc: neutral-product-standalone-operability Lane: openxfactory-4 (openXfactory-4-openDox_extraction) Co-Authored-By: Claude Opus 5.5 (1M context) --- src/opendox/neutral_projection.py | 33 +- src/opendox/web/views/wheel-model.js | 10 +- tests/fixtures/opendox-snapshot.schema.yaml | 443 ++++++++++ tests/test_display_facet.py | 50 +- tests/test_neutral_projection.py | 908 ++++++++++++++++++++ 5 files changed, 1403 insertions(+), 41 deletions(-) create mode 100644 tests/fixtures/opendox-snapshot.schema.yaml create mode 100644 tests/test_neutral_projection.py diff --git a/src/opendox/neutral_projection.py b/src/opendox/neutral_projection.py index edb75a80..b4e5cc10 100644 --- a/src/opendox/neutral_projection.py +++ b/src/opendox/neutral_projection.py @@ -17,6 +17,12 @@ .SNAPSHOT_VALUES` defaults to those same values, so openDox's views place every card by them and render the six words. +WHAT IS A DOCUMENT. An entry the adapter lists AND classifies. An entry it +cannot classify (a `Makefile`, an image) is not a document: it is left out +unread, and nothing mentions it (the holder's ruling on T054). So is an entry +whose path the neutral schema cannot carry (`path-is-repo-relative`), and the +projection reports that one, since the adapter did classify it. + WHERE A DOCUMENT LANDS (RULED R1Q13 (a) with (c), same comment). A document names its station with the neutral `stage:` key of its leading `Name: value` header, which is `local_git_adapter.leading_header`'s block. That is the same @@ -37,9 +43,7 @@ A word is a run of three or more letters, case-folded, that is not one of `STOPWORDS`. So the rule needs no front matter at all. AT-R1's repository (b) (quickstart.md § 2) is three notes named by their headings, and a note that -mentions another shares that note's name as a topic. An entry the adapter -cannot classify is listed but never read, so it carries no topic and nothing -mentions it. +mentions another shares that note's name as a topic. GROUPS (the grouping station). A group forms wherever two or more sources share a topic. Topics that the same sources share form one group, named by @@ -194,7 +198,6 @@ class _Document: title: str | None = None summary: str | None = None topics: tuple[str, ...] = () - read: bool = False name: str = "" tokens: tuple[str, ...] = () declared_topics: tuple[str, ...] | None = None @@ -281,12 +284,14 @@ def _unique(base: str, taken: set[str]) -> str: def _read_documents(adapter: CorpusAdapter, corpus: ResolvedCorpus, notices: list[Notice]) -> list[_Document]: - """Every listed document the snapshot can carry, in path order, with its - station, its two copied fields, its name and its tokens.""" + """Every document the snapshot can carry, in path order, with its station, + its two copied fields, its name and its tokens.""" stations = set(STAGE_ROLES) documents: list[_Document] = [] listed = sorted(adapter.list_documents(corpus), key=lambda d: d.key) for identity in listed: + if adapter.classify(corpus, identity).kind is None: + continue # not a document (this module's docstring) reason = _unwritable(identity.key) if reason is not None: # NAMED BY ITS repr: the path is one no snapshot can carry, and a @@ -299,13 +304,8 @@ def _read_documents(adapter: CorpusAdapter, corpus: ResolvedCorpus, continue document = _Document(path=identity.key) documents.append(document) - if adapter.classify(corpus, identity).kind is None: - # UNCLASSIFIABLE: listed, never read. It is a source with nothing - # to copy, no topic, and no name another document can mention. - continue text = adapter.read(corpus, identity).content.decode("utf-8", "replace") header = leading_header(text) - document.read = True document.title = header.get(TITLE_KEY) document.summary = header.get(SUMMARY_KEY) declared = header.get(STAGE_KEY) @@ -327,14 +327,13 @@ def _read_documents(adapter: CorpusAdapter, corpus: ResolvedCorpus, def _assign_topics(documents: list[_Document]) -> None: - """The topic rule (this module's docstring), over the documents read.""" - read = [document for document in documents if document.read] - own = [_words(document.name) for document in read] + """The topic rule (this module's docstring), over every document.""" + own = [_words(document.name) for document in documents] # Each name is looked for by its first WORD, at that word's offset, so a # text is scanned once and a common short token ("the") costs nothing. by_first_word: dict[str, list[tuple[tuple[str, ...], int, int]]] = {} - for index, document in enumerate(read): - for alias in {_tokens(document.name), _tokens(document.stem)}: + for index, document in enumerate(documents): + for alias in sorted({_tokens(document.name), _tokens(document.stem)}): # A name with no word in it contributes no topic, so it is not # looked for: `a.md` would otherwise be mentioned by every "a". offset = next((at for at, token in enumerate(alias) @@ -342,7 +341,7 @@ def _assign_topics(documents: list[_Document]) -> None: if offset is not None: by_first_word.setdefault(alias[offset], []).append( (alias, offset, index)) - for index, document in enumerate(read): + for index, document in enumerate(documents): if document.declared_topics is not None: document.topics = document.declared_topics continue diff --git a/src/opendox/web/views/wheel-model.js b/src/opendox/web/views/wheel-model.js index 0a403140..99a68c51 100644 --- a/src/opendox/web/views/wheel-model.js +++ b/src/opendox/web/views/wheel-model.js @@ -47,6 +47,9 @@ const CANDIDATE_ROLE = STAGE_ROLES[2]; const cap = (s) => (s ? s.charAt(0).toUpperCase() + s.slice(1) : s); const MAX_DEMO_POSSIBLES = 6; +// A group's document count: its tally where the snapshot carries one, and its +// edges otherwise (openDox's neutral snapshot, whose groups carry no tallies). +const docLinks = (c) => c?.tallies?.document_links ?? (c?.document_edges || []).length; function basename(path) { return String(path).split("/").at(-1) || String(path); @@ -68,9 +71,7 @@ export function isUndisposedDerived(possible) { export function synthesizeDemoPossibles(clusters, display) { const d = display || neutralDisplay(); const ranked = [...(clusters || [])].sort((a, b) => { - const ta = a.tallies?.document_links || 0; - const tb = b.tallies?.document_links || 0; - return tb - ta || String(a.id).localeCompare(String(b.id)); + return docLinks(b) - docLinks(a) || String(a.id).localeCompare(String(b.id)); }); return ranked.slice(0, MAX_DEMO_POSSIBLES).map((c) => ({ id: "demo-" + c.id, @@ -117,8 +118,7 @@ function buildItems({ display, sources, groups, candidates, selections, })), [GROUPING]: groups.map((c) => ({ id: c.id, label: c.name || c.id, - sub: (c.tallies?.document_links || 0) + " " - + display.count(SOURCE, c.tallies?.document_links || 0), ref: c, + sub: docLinks(c) + " " + display.count(SOURCE, docLinks(c)), ref: c, })), [CANDIDATE]: candidates.map((p) => ({ id: p.id, label: p.title || p.id, diff --git a/tests/fixtures/opendox-snapshot.schema.yaml b/tests/fixtures/opendox-snapshot.schema.yaml new file mode 100644 index 00000000..2be5018f --- /dev/null +++ b/tests/fixtures/opendox-snapshot.schema.yaml @@ -0,0 +1,443 @@ +# openDox's own neutral snapshot contract (plan 034, T053). +# +# WHY THIS FILE IS JSON. It is YAML whose body is one JSON object, and that is +# deliberate. The leg's required `validate` check installs pytest and nothing +# else, so `tests/test_opendox_snapshot_contract.py` reads this file with +# Python's built-in `json` module once these comment lines are set aside. +# JSON is YAML, so every YAML loader in the family reads the same object, and +# the same test proves that PyYAML agrees wherever PyYAML is installed. This +# leg's negative chat-turn examples already take this form. +# +# SECTIONS. openDox's views render six stations from five sections, as +# openDox's own STAGE_FIELDS declares them: source reads documents, grouping +# reads clusters, candidate reads possibles, selection reads staged_topics, +# and the submission and completion stations read the changes entries whose +# status is active and archived. Every station section is required, and it +# is an empty list when its station holds nothing. keyword_index is optional: +# without it, a reader derives the keyword rail from the documents' topics. +# +# CLOSED VALUES, all neutral. A document's stage is one of the six station +# role keys, a candidate's state is one of unselected, selected, declined and +# replaced, and a changes entry's status is active or archived. openDox's +# views match them through SNAPSHOT_VALUES, whose defaults T054 moves to +# these values. +# +# DETERMINISTIC, which is the generator's to keep and no schema can check: the +# same tree yields a byte-identical snapshot, so nothing in it records when +# the generator ran. generated_at, when present, is fixed by the source +# revision (its commit date, or a date recorded with it), never read from the +# clock. FORWARD-COMPATIBLE. A reader ignores unknown properties, so no object +# sets additionalProperties false, and an additive field needs no +# schema_version bump. +# +# RULES. Every rule has an identifier. A subschema names, in x-rule, the rule +# its keywords enforce; x-rules lists every rule with its class; and a +# validator's refusal names the rule it broke. A shape rule is enforced by +# this schema's own keywords. A reference rule is a cross-reference that no +# JSON Schema keyword can state, so a validator enforces it. +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "opendox-snapshot.schema.yaml", + "title": "openDox neutral snapshot", + "contract_schema_version": 1, + "description": "openDox's own snapshot: what its neutral generator writes over a plain repository, and what its views read, with no consumer installed (plan 034 T053, on R1Q11 (a) and R1Q12 (a): opensoft/openxFactory issue 656, comment 5850003126). openXdox's governed generator keeps its own contract, openXdox-spec's ideation-dashboard-snapshot, which this schema leaves unchanged; a contributed generator declares which of the two kinds it writes (T052). The file's comment header, the section descriptions and the x-rules catalog say the rest.", + "x-rule": "envelope-keys", + "type": "object", + "required": [ + "schema_version", + "kind", + "repository", + "generation", + "documents", + "clusters", + "possibles", + "staged_topics", + "changes" + ], + "properties": { + "schema_version": {"x-rule": "schema-version-is-1", "const": 1}, + "kind": {"x-rule": "kind-is-opendox-snapshot", "const": "opendox-snapshot"}, + "repository": {"x-rule": "repository-is-text", "type": "string", "minLength": 1}, + "generation": {"$ref": "#/$defs/generation"}, + "documents": { + "description": "The source station, and the product's whole document list: every document the corpus adapter lists, in the station its stage names. Every entry carries its stage, and no reader supplies one: the generator writes source for a document that declares no stage.", + "x-rule": "section-is-a-list", + "type": "array", + "items": {"$ref": "#/$defs/document"} + }, + "clusters": { + "description": "The grouping station: the groups that form around topics documents share, each with one edge per member document.", + "x-rule": "section-is-a-list", + "type": "array", + "items": {"$ref": "#/$defs/group"} + }, + "possibles": { + "description": "The candidate station.", + "x-rule": "section-is-a-list", + "type": "array", + "items": {"$ref": "#/$defs/candidate"} + }, + "staged_topics": { + "description": "The selection station.", + "x-rule": "section-is-a-list", + "type": "array", + "items": {"$ref": "#/$defs/selection"} + }, + "changes": { + "description": "The submission station (status active) and the completion station (status archived), which share this one section.", + "x-rule": "section-is-a-list", + "type": "array", + "items": {"$ref": "#/$defs/submission"} + }, + "keyword_index": { + "description": "Optional. The keyword rail's seed. When present, every topic the documents carry has one entry, and each entry counts the documents that carry its keyword.", + "x-rule": "section-is-a-list", + "type": "array", + "items": {"$ref": "#/$defs/keyword_entry"} + } + }, + "$defs": { + "id": {"x-rule": "id-is-text", "type": "string", "minLength": 1}, + "path": { + "description": "A path relative to the repository root, as the corpus adapter lists it.", + "x-rule": "path-is-repo-relative", + "type": "string", + "pattern": "^(?!/)(?![A-Za-z]:)(?![\\s\\S]*[\\\\\\u0000-\\u001f\\u007f-\\u009f])(?!(?:[\\s\\S]*/)?\\.\\.(?:/|$))[\\s\\S]+$" + }, + "topic": { + "x-rule": "topic-is-trimmed-text", + "type": "string", + "pattern": "^(?![\\s\\ufeff])(?![\\s\\S]*[\\s\\ufeff]$)[^\\u0000-\\u001f\\u007f-\\u009f]+$" + }, + "stage_role": { + "description": "The station a document sits in: one of the six station role keys, in spine order, exactly openDox's display_profile.STAGE_ROLES. The set is closed. A declared value outside it is not a declaration: the generator reads that document as a source and writes stage source, so the value never reaches a snapshot.", + "x-rule": "stage-is-a-station-role", + "enum": ["source", "grouping", "candidate", "selection", "submission", "completion"] + }, + "candidate_state": { + "description": "A candidate's state, in openDox's own words for the candidate station (NEUTRAL_DISPLAY's candidate vocabulary). A candidate is unselected until an act selects, declines or replaces it.", + "x-rule": "candidate-state-is-known", + "enum": ["unselected", "selected", "declined", "replaced"] + }, + "submission_status": { + "description": "The station a changes entry sits in: active for submission, archived for completion, as openDox's STAGE_FIELDS declares them.", + "x-rule": "submission-status-is-known", + "enum": ["active", "archived"] + }, + "generation": { + "description": "The generation stamp. source_revision is the determinism anchor: the tree revision the snapshot projects.", + "x-rule": "generation-anchored", + "type": "object", + "required": ["source_revision"], + "properties": { + "source_revision": {"x-rule": "generation-anchored", "type": "string", "minLength": 1}, + "generated_at": { + "x-rule": "generated-at-is-rfc3339", + "type": "string", + "format": "date-time", + "pattern": "^(?![\\s\\S]*[\\u0000-\\u001f\\u007f-\\u009f])(?!0000)(?:[0-9]{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12][0-9]|3[01])|(?:0[469]|11)-(?:0[1-9]|[12][0-9]|30)|02-(?:0[1-9]|1[0-9]|2[0-8]))|(?:[0-9]{2}(?:0[48]|[2468][048]|[13579][26])|(?:[02468][048]|[13579][26])00)-02-29)[Tt](?:[01][0-9]|2[0-3]):[0-5][0-9]:[0-5][0-9](?:\\.[0-9]+)?(?:[Zz]|[+-](?:[01][0-9]|2[0-3]):[0-5][0-9])$" + }, + "generator_version": {"x-rule": "generation-anchored", "type": "string", "minLength": 1} + } + }, + "document": { + "description": "One document the corpus adapter lists. title and summary are the default adapter's small neutral field set (R1Q13 (a)). Neither is required, and the generator writes null for one the document does not give. topics, which every entry carries, are the topics the generator assigns: the ones the document declares, or the ones its topic rule derives when it declares none, and an empty list when there are none.", + "x-rule": "document-keys", + "type": "object", + "required": ["id", "path", "stage", "topics"], + "properties": { + "id": {"$ref": "#/$defs/id"}, + "path": {"$ref": "#/$defs/path"}, + "stage": {"$ref": "#/$defs/stage_role"}, + "title": { + "x-rule": "title-and-summary-are-text", + "type": ["string", "null"], + "minLength": 1 + }, + "summary": { + "x-rule": "title-and-summary-are-text", + "type": ["string", "null"], + "minLength": 1 + }, + "topics": { + "x-rule": "topics-are-unique", + "type": "array", + "uniqueItems": true, + "items": {"$ref": "#/$defs/topic"} + } + } + }, + "group": { + "description": "One group in the grouping station. document_edges holds one edge per member document, naming the topics that matched; the funnel draws them.", + "x-rule": "group-keys", + "type": "object", + "required": ["id", "name", "topics", "document_edges"], + "properties": { + "id": {"$ref": "#/$defs/id"}, + "name": {"x-rule": "group-keys", "type": "string", "minLength": 1}, + "topics": { + "x-rule": "group-has-a-topic", + "type": "array", + "minItems": 1, + "uniqueItems": true, + "items": {"$ref": "#/$defs/topic"} + }, + "document_edges": { + "x-rule": "group-keys", + "type": "array", + "items": {"$ref": "#/$defs/edge"} + } + } + }, + "edge": { + "x-rule": "edge-keys", + "type": "object", + "required": ["document", "matched_topics"], + "properties": { + "document": {"$ref": "#/$defs/id"}, + "matched_topics": { + "x-rule": "edge-keys", + "type": "array", + "minItems": 1, + "uniqueItems": true, + "items": {"$ref": "#/$defs/topic"} + } + } + }, + "candidate": { + "description": "One candidate in the candidate station. claiming_clusters holds the groups that claim it; pick holds the selection a selected candidate went to.", + "x-rule": "candidate-keys", + "type": "object", + "required": ["id", "title", "state"], + "properties": { + "id": {"$ref": "#/$defs/id"}, + "title": {"x-rule": "candidate-keys", "type": "string", "minLength": 1}, + "claim": {"x-rule": "candidate-keys", "type": "string", "minLength": 1}, + "state": {"$ref": "#/$defs/candidate_state"}, + "claiming_clusters": { + "x-rule": "candidate-keys", + "type": "array", + "uniqueItems": true, + "items": {"$ref": "#/$defs/id"} + }, + "pick": { + "x-rule": "selected-candidate-has-pick", + "type": "object", + "required": ["staging_id"], + "properties": {"staging_id": {"$ref": "#/$defs/id"}} + }, + "reason": {"x-rule": "closed-candidate-has-reason", "type": "string", "minLength": 1} + }, + "allOf": [ + { + "if": {"required": ["state"], "properties": {"state": {"const": "selected"}}}, + "then": {"x-rule": "selected-candidate-has-pick", "required": ["pick"]} + }, + { + "if": {"required": ["state"], "properties": {"state": {"enum": ["declined", "replaced"]}}}, + "then": {"x-rule": "closed-candidate-has-reason", "required": ["reason"]} + } + ] + }, + "selection": { + "description": "One selection in the selection station. files lists what it is made of, and target_change names the changes entry it went on to.", + "x-rule": "selection-keys", + "type": "object", + "required": ["staging_id"], + "properties": { + "staging_id": {"$ref": "#/$defs/id"}, + "files": { + "x-rule": "selection-keys", + "type": "array", + "uniqueItems": true, + "items": {"$ref": "#/$defs/path"} + }, + "target_change": {"$ref": "#/$defs/id"} + } + }, + "submission": { + "description": "One entry of the submission or the completion station. files lists what it is made of, as a selection's files do, and the tile opens onto them.", + "x-rule": "submission-keys", + "type": "object", + "required": ["id", "status"], + "properties": { + "id": {"$ref": "#/$defs/id"}, + "status": {"$ref": "#/$defs/submission_status"}, + "files": { + "x-rule": "submission-keys", + "type": "array", + "uniqueItems": true, + "items": {"$ref": "#/$defs/path"} + } + } + }, + "keyword_entry": { + "description": "declared_doc_count counts the documents whose topics carry the keyword.", + "x-rule": "keyword-entry-keys", + "type": "object", + "required": ["keyword", "declared_doc_count"], + "properties": { + "keyword": {"$ref": "#/$defs/topic"}, + "declared_doc_count": {"x-rule": "keyword-entry-keys", "type": "integer", "minimum": 0} + } + } + }, + "x-rules": [ + { + "id": "envelope-keys", + "class": "shape", + "says": "A snapshot is an object carrying schema_version, kind, repository, generation and the five station sections: documents, clusters, possibles, staged_topics and changes." + }, + {"id": "schema-version-is-1", "class": "shape", "says": "schema_version is 1."}, + { + "id": "kind-is-opendox-snapshot", + "class": "shape", + "says": "kind is opendox-snapshot. The governed generator's ideation-dashboard-snapshot is a different contract, and this schema refuses it." + }, + { + "id": "repository-is-text", + "class": "shape", + "says": "repository, the canonical id of the repository the snapshot projects, is non-empty text." + }, + { + "id": "generation-anchored", + "class": "shape", + "says": "generation is an object carrying source_revision, the revision of the tree the snapshot projects, as non-empty text; generator_version, when present, is non-empty text too." + }, + { + "id": "generated-at-is-rfc3339", + "class": "shape", + "says": "generation.generated_at, when present, is an RFC 3339 date-time on a day the calendar has (the pattern knows each month's length and the leap years), with no control character. It is narrower than RFC 3339 in two places. Its year is never 0000, which Python's datetime cannot hold. Its seconds run from 00 to 59 and are never a leap second's 60, which a git commit date cannot hold and neither Python's datetime nor a browser's Date can read. jsonschema's date-time checker refuses both. It is fixed by the source revision, never read from the clock." + }, + { + "id": "section-is-a-list", + "class": "shape", + "says": "Each section is a list: documents, clusters, possibles, staged_topics, changes and, when present, keyword_index. A station with nothing in it is an empty list." + }, + { + "id": "id-is-text", + "class": "shape", + "says": "Every entry id, and every reference to one, is non-empty text." + }, + { + "id": "document-keys", + "class": "shape", + "says": "A document is an object carrying id, path, stage and topics." + }, + { + "id": "path-is-repo-relative", + "class": "shape", + "says": "A path is relative to the repository root: it does not start with a slash or with a drive letter and a colon (C:/x or C:x, which Windows joins onto a root as a path outside it), holds no backslash and no control character (U+0000 to U+001F, U+007F to U+009F), and has no .. segment." + }, + { + "id": "stage-is-a-station-role", + "class": "shape", + "says": "A document's stage is one of the six station role keys: source, grouping, candidate, selection, submission, completion." + }, + { + "id": "title-and-summary-are-text", + "class": "shape", + "says": "A document's title and summary are each non-empty text, or null." + }, + { + "id": "topics-are-unique", + "class": "shape", + "says": "A document's topics are a list that names each topic once." + }, + { + "id": "topic-is-trimmed-text", + "class": "shape", + "says": "A topic is non-empty text with no leading or trailing whitespace and no control character (U+0000 to U+001F, U+007F to U+009F). Whitespace is what both Python and a browser count as whitespace, U+FEFF included, so both refuse the same topics." + }, + { + "id": "group-keys", + "class": "shape", + "says": "A group (a clusters entry) is an object carrying id, name, topics and document_edges. Its name is non-empty text, and its edges are a list." + }, + { + "id": "group-has-a-topic", + "class": "shape", + "says": "A group's topics name at least one topic, each once: a group forms around topics that its documents share." + }, + { + "id": "edge-keys", + "class": "shape", + "says": "A group's document edge is an object that names the document and at least one matched topic, each once." + }, + { + "id": "candidate-keys", + "class": "shape", + "says": "A candidate (a possibles entry) is an object carrying id, title and state. Its title, and its claim when present, are non-empty text; claiming_clusters, when present, names each group once." + }, + { + "id": "candidate-state-is-known", + "class": "shape", + "says": "A candidate's state is one of unselected, selected, declined and replaced." + }, + { + "id": "selected-candidate-has-pick", + "class": "shape", + "says": "A selected candidate carries pick, an object whose staging_id names the selection it went to." + }, + { + "id": "closed-candidate-has-reason", + "class": "shape", + "says": "A declined or replaced candidate carries reason, non-empty text that says why." + }, + { + "id": "selection-keys", + "class": "shape", + "says": "A selection (a staged_topics entry) is an object carrying staging_id; files, when present, lists repository-relative paths, each once." + }, + { + "id": "submission-keys", + "class": "shape", + "says": "A changes entry, a submission or a completed item, is an object carrying id and status; files, when present, lists repository-relative paths, each once." + }, + { + "id": "submission-status-is-known", + "class": "shape", + "says": "A changes entry's status is active (the submission station) or archived (the completion station)." + }, + { + "id": "keyword-entry-keys", + "class": "shape", + "says": "A keyword_index entry is an object carrying keyword and declared_doc_count, a whole number no less than 0." + }, + { + "id": "ids-are-unique", + "class": "reference", + "says": "Within each section, entry ids are unique: documents, clusters, possibles and changes by id, and staged_topics by staging_id." + }, + { + "id": "edge-names-a-document", + "class": "reference", + "says": "Every group edge names a document in documents." + }, + { + "id": "one-edge-per-document", + "class": "reference", + "says": "Within one group, the edges name each document once: one edge per member document. A document may feed several groups." + }, + { + "id": "candidate-names-a-group", + "class": "reference", + "says": "Every group that a candidate's claiming_clusters names is in clusters." + }, + { + "id": "pick-names-a-selection", + "class": "reference", + "says": "A candidate's pick.staging_id names a selection in staged_topics." + }, + { + "id": "target-names-a-submission", + "class": "reference", + "says": "A selection's target_change names an entry in changes." + }, + { + "id": "keyword-index-matches-topics", + "class": "reference", + "says": "keyword_index, when present, agrees with the documents: every topic a document carries has an entry, no keyword has two, and each entry's declared_doc_count is the number of documents that carry its keyword, 0 for a keyword that none carries." + } + ] +} diff --git a/tests/test_display_facet.py b/tests/test_display_facet.py index d376ffd7..8270642a 100644 --- a/tests/test_display_facet.py +++ b/tests/test_display_facet.py @@ -298,8 +298,9 @@ def test_two_roles_may_not_share_one_snapshot_enum_value(): display_manifest({"values": {"register_state": { "captured": "same", "proposed": "same"}}}) # the partial-override case: one declared value colliding with a shipped one + shipped = SNAPSHOT_VALUES["register_state"]["captured"] with pytest.raises(DisplayFacetError, match="BOTH"): - display_manifest({"values": {"register_state": {"proposed": "latent"}}}) + display_manifest({"values": {"register_state": {"proposed": shipped}}}) # …and distinct values are taken whole ok = display_manifest({"values": {"register_state": { "captured": "new", "proposed": "chosen"}}}) @@ -653,7 +654,8 @@ def test_the_board_really_emits_a_stripe_class_the_stylesheet_matches(tmp_path): const {{ renderBoard }} = await import(base + "board.js"); const snap = {{ documents: [{{ id: "a.md", path: "ideation/brainstorm/a.md", kind: "document", - stage: "brainstorm", summary: "a note", dates: {{ captured: "2026-01-01" }} }}], + stage: D.SNAPSHOT_VALUES.document_stage.captured, summary: "a note", + dates: {{ captured: "2026-01-01" }} }}], clusters: [], possibles: [], staged_topics: [{{ staging_id: "topic-x", files: ["a.md"] }}], changes: [{{ id: "ch-1", status: "active" }}, @@ -688,8 +690,8 @@ def test_the_board_really_emits_a_stripe_class_the_stylesheet_matches(tmp_path): f"declares no `.card.{emitted}` rule — the card renders with no " "stripe, which no assertion over its words would notice") # THE CLASS DOES NOT MOVE WITH THE HOST'S WORDS. `declared` renames the - # source station and keeps openxFactory's `brainstorm` enum; the classes are - # identical because they were never that word. + # source station and keeps openDox's shipped enum values; the classes are + # identical because they were never a word. assert sorted(out["declared"]) == sorted(out["neutral"]) @@ -1075,24 +1077,27 @@ def test_the_explorer_file_rows_show_the_declared_stage_word(tmp_path): const base = {views} + "/"; const D = await import(base + "display.js"); const {{ mountExplorer }} = await import(base + "explorer.js"); -const snap = {{ +// each facet is shown a document at ITS OWN captured enum value: the host's +// declared `brainstorm`, and the value openDox ships +const snapAt = (stage) => ({{ staged_topics: [{{ staging_id: "topic-x", files: ["a.md"] }}], - documents: [{{ path: "a.md", stage: "brainstorm", kind: "note" }}], + documents: [{{ path: "a.md", stage, kind: "note" }}], changes: [], -}}; +}}); const declared = D.readDisplay({{ display: {{ schema_version: 1, kind: "opendox.display-facet", host_facet: "declared", values: {{ document_stage: {{ captured: "brainstorm" }} }}, statuses: {{ document: {{ captured: "jotted" }} }}, }} }}); -function rowTexts(display) {{ +function rowTexts(display, stage) {{ const host = new Node("div"); - const explorer = mountExplorer(host, snap, {{ display }}); + const explorer = mountExplorer(host, snapAt(stage), {{ display }}); explorer.openTile(D.DRILL_KINDS.selection, "topic-x"); return flatten(host).filter((n) => n.className === "where").map((n) => n.textContent); }} console.log(JSON.stringify({{ - neutral: rowTexts(D.neutralDisplay()), declared: rowTexts(declared), + neutral: rowTexts(D.neutralDisplay(), D.SNAPSHOT_VALUES.document_stage.captured), + declared: rowTexts(declared, "brainstorm"), }})); """, tmp_path) assert out["declared"], "the tile rendered no file row at all" @@ -1146,9 +1151,13 @@ def test_the_canvas_confirmation_shows_the_declared_seed_word(tmp_path): declared_state = [t for t in out["declared"] if t.startswith("state: ")] assert declared_state, out["declared"] assert declared_state[0] == "state: parked", declared_state + # The neutral install seeds openDox's own shipped value, and the line reads + # openDox's own word for that role (the two are spelled alike since T054, + # which made the neutral snapshot's values the defaults). neutral_state = [t for t in out["neutral"] if t.startswith("state: ")] - assert neutral_state[0] != "state: latent", ( - "the confirmation still spells the register's enum value") + assert neutral_state == [ + "state: " + NEUTRAL_DISPLAY["statuses"]["candidate"]["captured"]], ( + "the neutral confirmation does not read openDox's word for the state") @pytest.mark.skipif(NODE is None, reason="node is not installed") @@ -1168,27 +1177,30 @@ def test_the_funnel_search_matches_the_word_a_human_can_see(tmp_path): const base = {views} + "/"; const D = await import(base + "display.js"); const {{ renderFunnel }} = await import(base + "funnel.js"); -const snap = {{ - documents: [{{ id: "a.md", path: "a.md", stage: "brainstorm", topics: [] }}], +// each facet is shown a document at ITS OWN captured enum value +const snapAt = (stage) => ({{ + documents: [{{ id: "a.md", path: "a.md", stage, topics: [] }}], clusters: [], possibles: [], staged_topics: [], changes: [], -}}; +}}); const declared = D.readDisplay({{ display: {{ schema_version: 1, kind: "opendox.display-facet", host_facet: "declared", values: {{ document_stage: {{ captured: "brainstorm" }} }}, statuses: {{ document: {{ captured: "jotted" }} }}, }} }}); -function hays(display) {{ +function hays(display, stage) {{ const root = new Node("div"); - renderFunnel(root, snap, {{ display }}); + renderFunnel(root, snapAt(stage), {{ display }}); return flatten(root).filter((n) => n.dataset && n.dataset.hay) .map((n) => n.dataset.hay); }} console.log(JSON.stringify({{ - declared: hays(declared), neutral: hays(D.neutralDisplay()), + declared: hays(declared, "brainstorm"), + neutral: hays(D.neutralDisplay(), D.SNAPSHOT_VALUES.document_stage.captured), + shipped: D.SNAPSHOT_VALUES.document_stage.captured, }})); """, tmp_path) assert out["declared"] == ["a.md brainstorm jotted"] - assert out["neutral"] == ["a.md brainstorm captured"] + assert out["neutral"] == [f"a.md {out['shipped']} captured"] # --------------------------------------------------------------------------- diff --git a/tests/test_neutral_projection.py b/tests/test_neutral_projection.py new file mode 100644 index 00000000..2c616708 --- /dev/null +++ b/tests/test_neutral_projection.py @@ -0,0 +1,908 @@ +"""openDox's own small neutral projection, held to plan 034's T054. + +T054 realizes #1144's 5.1, 5.2 and 5.3: `opendox.neutral_projection`, new code +over `CorpusAdapter`, bound to `LocalGitCorpus` through the home-corpus seam, +writing T053's neutral snapshot. Its falsifier, from plan 034's tasks.md: + + an in-process test that the projection over T050's fixture, with neither + sibling importable, validates against T053's schema and carries none of + F5.3's declared words; the topic-rule test over a copy of repository (b); + and a test that a `stage:` value outside the six is reported and read as a + source. + +Those are the first three cases below, in that order. F5.3 itself runs +`python -m opendox.cli generate`, whose verb reaches the consumer's generator +until T055 routes it, so T056 and T063 quote it. + +THE SCHEMA. `tests/fixtures/opendox-snapshot.schema.yaml` is openDox-spec's +neutral snapshot contract, copied byte for byte from openDox-spec#16 at +`cd49eb25` (T053), and held here to that file's sha256. T057 ships the packaged +copy, and that copy replaces this one when it lands. The evaluator below is a +port of openDox-spec's own (`tests/test_opendox_snapshot_contract.py` there): +the JSON Schema keywords the contract uses, as draft 2020-12 defines them, and +its seven reference rules. The leg's test extra installs no `jsonschema`, so +none is imported. + +WHAT ELSE IT HOLDS. + +1. THE HOLDER'S RULE ON T051: the projection copies `title` and `summary` + without coercing or excluding them, so T051's malformed fixture reaches the + snapshot and breaks exactly its `EXPECTED_RULE`. +2. WHAT A DOCUMENT IS: an entry the adapter classifies. One it cannot classify + is left out (the holder's ruling on T054), and so is a path the schema + cannot carry, which is reported. +3. THE STATIONS: a declared group gathers the sources that share its topics, + one edge per document; candidates are claimed by the groups they share a + topic with; ids are unique within each section. +4. THE ANCHORS: `source_revision` is the corpus HEAD unless pinned, + `generated_at` is that commit's own date unless given, and the same tree + answers the same bytes. +5. THE FIELD SET: openDox's default adapter obliges `title` and `summary` + (R1Q13 (a)), so `authoring.required_header_fields()` answers them through + the entry point, and a document without them is still read. +6. THE VALUES: `SNAPSHOT_VALUES`' defaults, in Python and in `display.js`, are + the neutral schema's values, so openDox's views match openDox's snapshot. + And the wheel's grouping tile counts a group's edges where the group carries + no tally, as the neutral snapshot's groups do not (the holder's ruling: + the schema is not widened). +7. NO REACH: the projection imports with every sibling blocked and pulls in no + third-party module. + +`--noconftest` SAFE. The autouse fixture saves and restores the three +registries an entry point writes, so no case leaves a registration behind. + +A CREATED file: no carve-manifest row (RULED OQ-C). +""" + +from __future__ import annotations + +import hashlib +import json +import os +import re +import shutil +import subprocess +import sys +import textwrap +from pathlib import Path +from typing import Any, Callable, Iterator, NamedTuple + +import pytest + +from opendox import ( + authoring, + corpus_adapter, + default_generator, + display_profile, + domain_profile, +) +from opendox import generator_seam as gs +from opendox import neutral_projection as projection +from opendox.runtime import local_git_adapter as lga + +ROOT = Path(__file__).resolve().parents[1] +SRC = ROOT / "src" +FIXTURES = ROOT / "tests" / "fixtures" +PLAIN = FIXTURES / "plain-documents" # T050, openDox-code#53 +MALFORMED = FIXTURES / "malformed" # T051, openDox-code#56 +SCHEMA_PATH = FIXTURES / "opendox-snapshot.schema.yaml" +DISPLAY_JS = SRC / "opendox" / "web" / "views" / "display.js" +WHEEL_MODEL_JS = SRC / "opendox" / "web" / "views" / "wheel-model.js" +NODE = shutil.which("node") + +#: The sha256 of `contracts/schemas/opendox-snapshot.schema.yaml` at +#: openDox-spec#16's head, `cd49eb25` (T053; its PR records the same digest). +SCHEMA_SHA256 = "f9e3e111af1d4bd4c377c933027d81b582ae2b0a395b66f4e4621992454a584a" + +#: The four packages a neutral openDox must import without (#1144's F2.1). +SIBLINGS = ("openxdox", "ideation_dashboard", "doc_health", + "corpus_adapter_openxfactory") + +#: F5.3's declared vocabulary, verbatim from #1144's `tasks.md`. +F5_3_WORDS = ("brainstorm", "staged", "draft", "ratified", "standard", + "superseded", "retired", "record", "openspec", "proposal.md", + "tasks.md", "design.md", "added requirements", + "modified requirements") + +#: AT-R1's repository (b), byte for byte as plan 034's quickstart.md § 2 +#: writes it with `printf`: ordinary Markdown with no front matter at all. +REPOSITORY_B = { + "roadmap.md": "# Roadmap\n\nThe roadmap links to the [budget](budget.md) " + "and the [notes](notes.md).\n", + "budget.md": "# Budget\n\nBudget figures for the roadmap.\n", + "notes.md": "# Meeting notes\n\nWe discussed the roadmap and the budget.\n", +} + +#: Every fixture commit is made at this date, so `generated_at` is known. +ANCHOR_DATE = "2026-09-27T12:00:00+00:00" + + +# --------------------------------------------------------------------------- +# the registries, and plain git repositories to project +# --------------------------------------------------------------------------- + +def _home(root: str): + """The entry points' default home factory, in its own shape (T022).""" + return (lga.WorkingTreeCorpus(), + corpus_adapter.CorpusRef(name="home", location=str(root))) + + +@pytest.fixture(autouse=True) +def _isolated_registries(): + """Each case starts with no generator registered and openDox's default + home corpus registered, and PUTS BACK the three registries it found.""" + seam = (gs._registered, gs._is_default, gs._generated_from_default, + gs._default_generations_under_way) + profile = (domain_profile._registered, domain_profile._is_default, + domain_profile._built_from_default) + home = corpus_adapter._home_factory + gs.unregister() + corpus_adapter.register_home(_home) + yield + (gs._registered, gs._is_default, gs._generated_from_default, + gs._default_generations_under_way) = seam + (domain_profile._registered, domain_profile._is_default, + domain_profile._built_from_default) = profile + corpus_adapter._home_factory = home + + +def _git(root: Path, *args: str) -> str: + """`git` in `root`, with no inherited `GIT_*` variable and no user or + system configuration, as the fixture's own identity at a fixed date.""" + env = {k: v for k, v in os.environ.items() if not k.startswith("GIT_")} + env.update({ + "GIT_AUTHOR_NAME": "fixture", "GIT_AUTHOR_EMAIL": "fixture@example.invalid", + "GIT_COMMITTER_NAME": "fixture", + "GIT_COMMITTER_EMAIL": "fixture@example.invalid", + "GIT_AUTHOR_DATE": ANCHOR_DATE, "GIT_COMMITTER_DATE": ANCHOR_DATE, + "GIT_CONFIG_GLOBAL": os.devnull, "GIT_CONFIG_SYSTEM": os.devnull, + }) + return subprocess.run(["git", "-C", str(root), *args], check=True, + capture_output=True, env=env).stdout.decode() + + +def _repository(tmp_path: Path, *, copy: Path | None = None, + files: dict[str, str | bytes] | None = None, + commit: bool = True, name: str = "repository") -> Path: + """A FRESH plain git repository: `copy`'s files and then `files`, added, + and committed unless `commit` is false.""" + root = tmp_path / name + if copy is not None: + shutil.copytree(copy, root) + else: + root.mkdir(parents=True) + for relative, body in (files or {}).items(): + path = root / relative + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(body if isinstance(body, bytes) else body.encode()) + _git(root, "-c", "init.defaultBranch=main", "init", "-q") + _git(root, "add", "-A") + if commit: + _git(root, "commit", "-qm", "fixture") + return root + + +def _generate(root: Path, **anchors: Any) -> dict[str, Any]: + """openDox's own generator over `root`, exactly as the seam calls it.""" + return default_generator.generate(root, "fixture", **anchors) + + +def _by_path(snapshot: dict[str, Any]) -> dict[str, dict[str, Any]]: + return {document["path"]: document for document in snapshot["documents"]} + + +def _values(node: Any) -> Iterator[str]: + """Every string VALUE, as F5.3 reads them: keys are the product's own.""" + if isinstance(node, dict): + for value in node.values(): + yield from _values(value) + elif isinstance(node, list): + for value in node: + yield from _values(value) + elif isinstance(node, str): + yield node + + +_F5_3 = re.compile(r"\b(" + "|".join(re.escape(w) for w in F5_3_WORDS) + r")\b") + + +def _leaks(snapshot: dict[str, Any]) -> list[str]: + return sorted({m.group(1) for value in _values(snapshot) + for m in _F5_3.finditer(value.lower())}) + + +# --------------------------------------------------------------------------- +# the neutral contract, and an evaluator for it (openDox-spec's own, ported) +# --------------------------------------------------------------------------- + +def _no_duplicate_keys(pairs: list[tuple[str, Any]]) -> dict[str, Any]: + out: dict[str, Any] = {} + for key, value in pairs: + if key in out: + raise ValueError(f"the key {key!r} appears twice in one object") + out[key] = value + return out + + +def _no_constants(name: str) -> Any: + raise ValueError(f"{name} is not JSON") + + +def _read_schema(path: Path) -> Any: + """The schema file's body: one JSON object after its `#` comment lines.""" + lines = path.read_text(encoding="utf-8").splitlines(keepends=True) + while lines and (lines[0].startswith("#") or not lines[0].strip()): + lines.pop(0) + return json.loads("".join(lines), object_pairs_hook=_no_duplicate_keys, + parse_constant=_no_constants) + + +SCHEMA = _read_schema(SCHEMA_PATH) + + +class Violation(NamedTuple): + rule: str # the broken rule's id, from `x-rule` or a reference check + where: str # a JSON pointer into the snapshot ("" is the root) + keyword: str # the schema keyword that failed, or "reference" + detail: str + + +def _pointer(parts: Any) -> str: + return "".join("/" + str(p).replace("~", "~0").replace("/", "~1") + for p in parts) + + +def _is_type(value: Any, name: str) -> bool: + if name == "object": + return isinstance(value, dict) + if name == "array": + return isinstance(value, list) + if name == "string": + return isinstance(value, str) + if name == "null": + return value is None + if name == "boolean": + return isinstance(value, bool) + if isinstance(value, bool): # JSON true is not the number 1 + return False + if name == "integer": + return isinstance(value, int) or ( + isinstance(value, float) and value.is_integer()) + if name == "number": + return isinstance(value, (int, float)) + raise AssertionError(f"the contract names an unknown type: {name!r}") + + +def _canon(value: Any) -> Any: + """JSON equality: `true` is not `1`, `1` is `1.0`, and key order is noise.""" + if isinstance(value, bool): + return ("boolean", value) + if isinstance(value, (int, float)): + return ("number", value) + if isinstance(value, str): + return ("string", value) + if value is None: + return ("null",) + if isinstance(value, list): + return ("array", tuple(_canon(v) for v in value)) + return ("object", tuple(sorted((k, _canon(v)) for k, v in value.items()))) + + +def _resolve(ref: str) -> dict[str, Any]: + assert ref.startswith("#/"), f"only local references are in the contract: {ref!r}" + node: Any = SCHEMA + for part in ref[2:].split("/"): + node = node[part.replace("~1", "/").replace("~0", "~")] + return node + + +def _check(value: Any, schema: dict[str, Any], where: str) -> Iterator[Violation]: + if "$ref" in schema: + yield from _check(value, _resolve(schema["$ref"]), where) + rule = schema.get("x-rule", "") + + def broken(keyword: str, detail: str) -> Violation: + return Violation(rule, where, keyword, detail) + + if "type" in schema: + names = schema["type"] if isinstance(schema["type"], list) else [schema["type"]] + if not any(_is_type(value, n) for n in names): + yield broken("type", f"{value!r} is not of type {names}") + if "const" in schema and _canon(value) != _canon(schema["const"]): + yield broken("const", f"{value!r} is not {schema['const']!r}") + if "enum" in schema and _canon(value) not in {_canon(v) for v in schema["enum"]}: + yield broken("enum", f"{value!r} is not one of {schema['enum']}") + if isinstance(value, str): + if len(value) < schema.get("minLength", 0): + yield broken("minLength", f"{value!r} is shorter than {schema['minLength']}") + if "pattern" in schema and not re.search(schema["pattern"], value): + yield broken("pattern", f"{value!r} does not match the rule's pattern") + if _is_type(value, "number") and "minimum" in schema and value < schema["minimum"]: + yield broken("minimum", f"{value!r} is less than {schema['minimum']}") + if isinstance(value, list): + if len(value) < schema.get("minItems", 0): + yield broken("minItems", f"{len(value)} items, fewer than {schema['minItems']}") + if schema.get("uniqueItems") and len({_canon(v) for v in value}) != len(value): + yield broken("uniqueItems", f"{value!r} repeats an item") + if "items" in schema: + for i, item in enumerate(value): + yield from _check(item, schema["items"], f"{where}/{i}") + if isinstance(value, dict): + for key in schema.get("required", ()): + if key not in value: + yield broken("required", f"{key!r} is required") + for key, sub in schema.get("properties", {}).items(): + if key in value: + yield from _check(value[key], sub, where + _pointer([key])) + for sub in schema.get("allOf", ()): + yield from _check(value, sub, where) + if "if" in schema and "then" in schema: # `if` is a test, never a failure + if not any(True for _ in _check(value, schema["if"], where)): + yield from _check(value, schema["then"], where) + + +def _entries(snap: Any, section: str) -> list[tuple[int, dict[str, Any]]]: + items = snap.get(section) if isinstance(snap, dict) else None + if not isinstance(items, list): + return [] + return [(i, e) for i, e in enumerate(items) if isinstance(e, dict)] + + +def _ids(snap: Any, section: str, key: str = "id") -> set[str]: + return {e[key] for _i, e in _entries(snap, section) if isinstance(e.get(key), str)} + + +def _ids_are_unique(snap: Any) -> Iterator[Violation]: + for section, key in (("documents", "id"), ("clusters", "id"), + ("possibles", "id"), ("staged_topics", "staging_id"), + ("changes", "id")): + seen: set[str] = set() + for i, entry in _entries(snap, section): + value = entry.get(key) + if isinstance(value, str): + if value in seen: + yield Violation("ids-are-unique", f"/{section}/{i}/{key}", + "reference", f"{value!r} is already an id") + seen.add(value) + + +def _edges(group: dict[str, Any]) -> list[tuple[int, Any]]: + edges = group.get("document_edges") + return list(enumerate(edges if isinstance(edges, list) else [])) + + +def _edge_names_a_document(snap: Any) -> Iterator[Violation]: + known = _ids(snap, "documents") + for gi, group in _entries(snap, "clusters"): + for ei, edge in _edges(group): + ref = edge.get("document") if isinstance(edge, dict) else None + if isinstance(ref, str) and ref not in known: + yield Violation("edge-names-a-document", + f"/clusters/{gi}/document_edges/{ei}/document", + "reference", f"no document has the id {ref!r}") + + +def _one_edge_per_document(snap: Any) -> Iterator[Violation]: + for gi, group in _entries(snap, "clusters"): + seen: set[str] = set() + for ei, edge in _edges(group): + ref = edge.get("document") if isinstance(edge, dict) else None + if isinstance(ref, str): + if ref in seen: + yield Violation("one-edge-per-document", + f"/clusters/{gi}/document_edges/{ei}/document", + "reference", f"{ref!r} already has an edge") + seen.add(ref) + + +def _keyword_index_matches_topics(snap: Any) -> Iterator[Violation]: + index = snap.get("keyword_index") if isinstance(snap, dict) else None + if not isinstance(index, list): + return + carried: dict[str, int] = {} + for _i, document in _entries(snap, "documents"): + topics = document.get("topics") + for topic in ({t for t in topics if isinstance(t, str)} + if isinstance(topics, list) else ()): + carried[topic] = carried.get(topic, 0) + 1 + listed: set[str] = set() + rule = "keyword-index-matches-topics" + for ki, entry in enumerate(index): + keyword = entry.get("keyword") if isinstance(entry, dict) else None + if not isinstance(keyword, str): + continue + if keyword in listed: + yield Violation(rule, f"/keyword_index/{ki}/keyword", "reference", + f"{keyword!r} already has an entry") + continue + listed.add(keyword) + count = entry.get("declared_doc_count") + if _is_type(count, "integer") and count != carried.get(keyword, 0): + yield Violation(rule, f"/keyword_index/{ki}/declared_doc_count", + "reference", f"{count} documents, but " + f"{carried.get(keyword, 0)} carry {keyword!r}") + unlisted = sorted(set(carried) - listed) + if unlisted: + yield Violation(rule, "/keyword_index", "reference", + f"topics the documents carry and the index lacks: {unlisted}") + + +def _candidate_names_a_group(snap: Any) -> Iterator[Violation]: + known = _ids(snap, "clusters") + for pi, candidate in _entries(snap, "possibles"): + refs = candidate.get("claiming_clusters") + for ri, ref in enumerate(refs if isinstance(refs, list) else []): + if isinstance(ref, str) and ref not in known: + yield Violation("candidate-names-a-group", + f"/possibles/{pi}/claiming_clusters/{ri}", + "reference", f"no group has the id {ref!r}") + + +def _pick_names_a_selection(snap: Any) -> Iterator[Violation]: + known = _ids(snap, "staged_topics", "staging_id") + for pi, candidate in _entries(snap, "possibles"): + pick = candidate.get("pick") + ref = pick.get("staging_id") if isinstance(pick, dict) else None + if isinstance(ref, str) and ref not in known: + yield Violation("pick-names-a-selection", + f"/possibles/{pi}/pick/staging_id", "reference", + f"no selection has the staging_id {ref!r}") + + +def _target_names_a_submission(snap: Any) -> Iterator[Violation]: + known = _ids(snap, "changes") + for ti, selection in _entries(snap, "staged_topics"): + ref = selection.get("target_change") + if isinstance(ref, str) and ref not in known: + yield Violation("target-names-a-submission", + f"/staged_topics/{ti}/target_change", "reference", + f"no changes entry has the id {ref!r}") + + +REFERENCE_CHECKS: dict[str, Callable[[Any], Iterator[Violation]]] = { + "ids-are-unique": _ids_are_unique, + "edge-names-a-document": _edge_names_a_document, + "one-edge-per-document": _one_edge_per_document, + "candidate-names-a-group": _candidate_names_a_group, + "pick-names-a-selection": _pick_names_a_selection, + "target-names-a-submission": _target_names_a_submission, + "keyword-index-matches-topics": _keyword_index_matches_topics, +} + + +def violations(snapshot: Any) -> list[Violation]: + """Every rule of the neutral contract that `snapshot` breaks.""" + found = list(_check(snapshot, SCHEMA, "")) + for check in REFERENCE_CHECKS.values(): + found.extend(check(snapshot)) + return found + + +def test_the_schema_copy_is_openDox_specs_contract_and_matches_the_product() -> None: + """The copy is T053's file, and the product's own declarations are the + contract's closed values (T053's writer asked for this cross-check).""" + digest = hashlib.sha256(SCHEMA_PATH.read_bytes()).hexdigest() + assert digest == SCHEMA_SHA256, ( + f"tests/fixtures/opendox-snapshot.schema.yaml is {digest}, not " + "openDox-spec#16's contract at cd49eb25. Copy the spec leg's file " + "again and update SCHEMA_SHA256 with it, in one commit") + defs = SCHEMA["$defs"] + assert SCHEMA["properties"]["kind"]["const"] == gs.NEUTRAL_SNAPSHOT_KIND + assert SCHEMA["properties"]["schema_version"]["const"] == projection.SCHEMA_VERSION + assert defs["stage_role"]["enum"] == list(display_profile.STAGE_ROLES) + sections = {section for _role, section, _status in display_profile.STAGE_FIELDS} + assert sections <= set(SCHEMA["required"]) + assert set(defs["submission_status"]["enum"]) == set( + projection.CHANGE_STATUS.values()) + assert sorted(REFERENCE_CHECKS) == sorted( + rule["id"] for rule in SCHEMA["x-rules"] if rule["class"] == "reference") + + +# --------------------------------------------------------------------------- +# the falsifier, part 1: T050's fixture, with neither sibling importable +# --------------------------------------------------------------------------- + +_FALSIFIER = """ +import importlib, json, sys +from pathlib import Path +blocked = [] +for name in SIBLINGS: + sys.modules[name] = None + try: + importlib.import_module(name) + except ImportError: + blocked.append(name) +from opendox import cli, generator_seam +cli.build_parser() # the entry point registers openDox's own defaults +snapshot = generator_seam.generate(Path(ROOT_OF_REPOSITORY), "fixture") +print(json.dumps({"blocked": blocked, "snapshot": snapshot})) +""" + + +def test_the_projection_over_the_plain_documents_fixture_is_a_neutral_snapshot( + tmp_path: Path) -> None: + """T054's falsifier, first part. A fresh process blocks every sibling, + lets the CLI entry point register openDox's own defaults (the default + profile, `WorkingTreeCorpus` as the home corpus, openDox's own generator), + and generates through the seam over a fresh repository of T050's fixture. + The snapshot validates against T053's schema, carries none of F5.3's + declared words, and fills the station each document names.""" + root = _repository(tmp_path, copy=PLAIN) + program = (f"import sys; sys.path.insert(0, {str(SRC)!r})\n" + f"SIBLINGS = {SIBLINGS!r}\nROOT_OF_REPOSITORY = {str(root)!r}\n" + + textwrap.dedent(_FALSIFIER)) + done = subprocess.run([sys.executable, "-c", program], capture_output=True, + text=True, cwd=str(ROOT), timeout=300) + assert done.returncode == 0, done.stderr + assert "notice:" not in done.stderr, done.stderr + out = json.loads(done.stdout.strip().splitlines()[-1]) + assert out["blocked"] == list(SIBLINGS), out["blocked"] + snapshot = out["snapshot"] + + assert violations(snapshot) == [] + assert _leaks(snapshot) == [], ( + f"openxFactory's vocabulary leaked into the neutral projection: " + f"{_leaks(snapshot)}") + assert snapshot["kind"] == gs.NEUTRAL_SNAPSHOT_KIND + assert snapshot["documents"], "the snapshot is empty" + + # EVERY DOCUMENT IN THE STATION IT NAMES, read here off the fixture's own + # `stage:` lines rather than through the projection's reader. + expected = {} + for path in sorted(PLAIN.glob("*.md")): + declared = re.search(r"^stage: (\S+)$", + path.read_text(encoding="utf-8"), re.M) + expected[path.name] = declared.group(1) if declared else "source" + assert {p: d["stage"] for p, d in _by_path(snapshot).items()} == expected + assert len(set(expected.values())) == 6, "T050 spreads over six stations" + assert snapshot["possibles"] and snapshot["staged_topics"] + assert {c["status"] for c in snapshot["changes"]} == {"active", "archived"} + + # AT LEAST ONE GROUP FORMS FROM SOURCES ALONE, which is what lets AT-R1 + # open the chat pane: the two rain-barrel notes share their name's words. + sources = {p for p, stage in expected.items() if stage == "source"} + derived = [c for c in snapshot["clusters"] + if {e["document"] for e in c["document_edges"]} <= sources] + assert derived, snapshot["clusters"] + assert any(set(c["topics"]) >= {"rain", "barrel"} for c in derived), derived + + +# --------------------------------------------------------------------------- +# the falsifier, part 2: the topic rule over repository (b), no front matter +# --------------------------------------------------------------------------- + +def test_the_topic_rule_groups_repository_b_which_has_no_front_matter( + tmp_path: Path) -> None: + """T054's falsifier, second part. THE TOPIC RULE: a document that declares + no `topics:` carries the words of its name (its `title:`, else its first + `#` heading, else its file name) and the name words of every other + document it mentions by name or file name. Repository (b)'s three notes + have no header at all, and they name each other, so they group.""" + for body in REPOSITORY_B.values(): + assert lga.leading_header(body) == {}, "repository (b) has front matter" + root = _repository(tmp_path, files=REPOSITORY_B, name="plain-notes") + snapshot = _generate(root) + assert violations(snapshot) == [] + assert _leaks(snapshot) == [] + + documents = _by_path(snapshot) + assert set(documents) == set(REPOSITORY_B) + for document in documents.values(): + assert document["stage"] == "source" + assert document["title"] is None and document["summary"] is None + assert documents["budget.md"]["topics"] == ["budget", "roadmap"] + assert documents["notes.md"]["topics"] == ["budget", "meeting", "notes", "roadmap"] + + assert snapshot["clusters"], "repository (b) yields no grouping tile (AT-R1 FAILS)" + shared = {tuple(c["topics"]): sorted(e["document"] for e in c["document_edges"]) + for c in snapshot["clusters"]} + assert shared[("budget", "roadmap")] == sorted(REPOSITORY_B) + for cluster in snapshot["clusters"]: + assert len(cluster["document_edges"]) >= 2, cluster + + +# --------------------------------------------------------------------------- +# the falsifier, part 3: a `stage:` value outside the six +# --------------------------------------------------------------------------- + +@pytest.mark.parametrize("value", ["shelved", "Grouping", ""]) +def test_a_stage_value_outside_the_six_is_reported_and_read_as_a_source( + value: str, tmp_path: Path, capsys) -> None: + """T054's falsifier, third part (spec.md's edge case, the projection's + half; T056 holds the verb's). A value outside the six role keys, the empty + value and another casing included, is no declaration: the generator + reports it, naming the document, the value and the six keys, and the + snapshot reads the document as a source, so no other value reaches it.""" + target = "grouping-compost-corner.md" + root = _repository(tmp_path, copy=PLAIN) + path = root / target + path.write_text(path.read_text(encoding="utf-8").replace( + "stage: grouping\n", f"stage: {value}\n" if value else "stage:\n", 1), + encoding="utf-8") + _git(root, "commit", "-qam", "an undeclared stage") + + snapshot = _generate(root) + notices = [line for line in capsys.readouterr().err.splitlines() + if line.startswith("notice: ")] + assert len(notices) == 1, notices + assert notices[0].startswith(f"notice: {target}: "), notices[0] + assert repr(value) in notices[0] + for role in display_profile.STAGE_ROLES: + assert role in notices[0], f"the report does not name {role!r}" + + assert _by_path(snapshot)[target]["stage"] == "source" + assert "grouping-compost-corner" not in {c["id"] for c in snapshot["clusters"]} + assert violations(snapshot) == [] + + +# --------------------------------------------------------------------------- +# 1 — title and summary, exactly as the document gives them (the holder rule) +# --------------------------------------------------------------------------- + +def test_the_malformed_fixture_keeps_its_one_violation(tmp_path: Path) -> None: + """The holder's ruling on T051: the projection neither coerces nor drops + an empty `title:`, so T051's fixture breaks exactly its EXPECTED_RULE, at + that document's title. EXPECTED_RULE itself has no shape the adapter + recognizes, so it is not a document.""" + expected_rule = (MALFORMED / "EXPECTED_RULE").read_text(encoding="utf-8").strip() + snapshot = _generate(_repository(tmp_path, copy=MALFORMED)) + paths = [d["path"] for d in snapshot["documents"]] + assert paths == ["notes-bee-boxes.md", "notes-empty-title.md"] + found = violations(snapshot) + assert [(v.rule, v.where) for v in found] == [ + (expected_rule, f"/documents/{paths.index('notes-empty-title.md')}/title")] + assert _by_path(snapshot)["notes-empty-title.md"]["title"] == "" + + +def test_title_and_summary_are_copied_exactly_and_null_only_when_absent( + tmp_path: Path) -> None: + root = _repository(tmp_path, files={ + "both.md": "title: Both fields\nsummary: kept as written \n\n# Other\n", + "none.md": "# Only a heading\n\nNo header here.\n", + "empty.md": "title:\nsummary:\n\nBody.\n", + "quoted.md": 'title: "Quoted"\n\nBody.\n', + }) + documents = _by_path(_generate(root)) + assert (documents["both.md"]["title"], documents["both.md"]["summary"]) == ( + "Both fields", "kept as written") + assert (documents["none.md"]["title"], documents["none.md"]["summary"]) == (None, None) + assert (documents["empty.md"]["title"], documents["empty.md"]["summary"]) == ("", "") + assert (documents["quoted.md"]["title"], documents["quoted.md"]["summary"]) == ( + '"Quoted"', None) + + +# --------------------------------------------------------------------------- +# 2 — what a document is +# --------------------------------------------------------------------------- + +def test_an_entry_the_adapter_cannot_classify_is_not_a_document(tmp_path: Path) -> None: + """The holder's ruling: a `Makefile` or an image is left out, and is never + read, so nothing mentions it either.""" + root = _repository(tmp_path, files={ + "Makefile": "all:\n\techo build\n", + "garden-layout.png": b"\x89PNG\r\n\x1a\n\x00binary", + "notes.md": "# Notes\n\nSee the Makefile and the garden layout png.\n", + }) + snapshot = _generate(root) + assert [d["path"] for d in snapshot["documents"]] == ["notes.md"] + assert _by_path(snapshot)["notes.md"]["topics"] == ["notes"] + assert violations(snapshot) == [] + + +@pytest.mark.parametrize(("name", "reason"), [ + ("back\\slash.md", "backslash"), ("tab\tin-name.md", "control character")]) +def test_a_path_the_schema_cannot_carry_is_left_out_and_reported( + name: str, reason: str, tmp_path: Path, capsys) -> None: + root = _repository(tmp_path, files={name: "# Odd\n", "plain.md": "# Plain\n"}) + snapshot = _generate(root) + assert [d["path"] for d in snapshot["documents"]] == ["plain.md"] + notices = [line for line in capsys.readouterr().err.splitlines() + if line.startswith("notice: ")] + assert len(notices) == 1 and repr(name) in notices[0] and reason in notices[0], notices + assert violations(snapshot) == [] + + +# --------------------------------------------------------------------------- +# 3 — topics, groups and the other stations +# --------------------------------------------------------------------------- + +def test_declared_topics_replace_the_derived_ones(tmp_path: Path) -> None: + root = _repository(tmp_path, files={ + "a.md": "title: Rain notes\ntopics: Rain, rain barrel , COMPOST, , " + "tab\there, bell\x07\n\nText.\n", + }) + snapshot = _generate(root) + assert _by_path(snapshot)["a.md"]["topics"] == [ + "compost", "rain", "rain barrel", "tab here"] + assert violations(snapshot) == [] + + +def test_a_declared_group_gathers_the_sources_that_share_its_topics( + tmp_path: Path) -> None: + root = _repository(tmp_path, files={ + "g.md": "stage: grouping\ntitle: Water, gathered\ntopics: water\n\n.\n", + "a.md": "topics: water, soil\n\n.\n", + "b.md": "topics: soil\n\n.\n", + "c.md": "stage: candidate\ntitle: Harvest the rain\nsummary: Gutters " + "to barrels.\ntopics: water\n\n.\n", + "d.md": "stage: selection\ntopics: water\n\n.\n", + }) + snapshot = _generate(root) + assert violations(snapshot) == [] + clusters = {c["id"]: c for c in snapshot["clusters"]} + assert list(clusters) == ["g", "soil"], "declared groups first, then derived" + assert clusters["g"]["name"] == "Water, gathered" + assert clusters["g"]["document_edges"] == [ + {"document": "a.md", "matched_topics": ["water"]}, + {"document": "g.md", "matched_topics": ["water"]}] + assert [e["document"] for e in clusters["soil"]["document_edges"]] == ["a.md", "b.md"] + [candidate] = snapshot["possibles"] + assert candidate == {"id": "c", "title": "Harvest the rain", + "claim": "Gutters to barrels.", "state": "unselected", + "claiming_clusters": ["g"]} + assert snapshot["staged_topics"] == [{"staging_id": "d", "files": ["d.md"]}] + + +def test_a_declared_group_with_no_topic_forms_no_group_and_is_reported( + tmp_path: Path, capsys) -> None: + snapshot = _generate(_repository(tmp_path, files={"g.md": "stage: grouping\ntitle: 2026\n"})) + assert snapshot["clusters"] == [] + assert _by_path(snapshot)["g.md"]["stage"] == "grouping" + assert "notice: g.md: declares stage grouping but carries no topic" in \ + capsys.readouterr().err + + +def test_station_ids_are_unique_within_a_section(tmp_path: Path) -> None: + snapshot = _generate(_repository(tmp_path, files={ + "a/plan.md": "stage: selection\n", "b/plan.md": "stage: selection\n", + "x/ship.md": "stage: submission\n", "y/ship.md": "stage: completion\n", + })) + assert [t["staging_id"] for t in snapshot["staged_topics"]] == ["plan", "plan-2"] + assert [(c["id"], c["status"]) for c in snapshot["changes"]] == [ + ("ship", "active"), ("ship-2", "archived")] + assert violations(snapshot) == [] + + +# --------------------------------------------------------------------------- +# 4 — the anchors, and determinism +# --------------------------------------------------------------------------- + +def test_the_anchors_come_from_the_source_revision_and_the_bytes_repeat( + tmp_path: Path) -> None: + root = _repository(tmp_path, copy=PLAIN) + first, second = _generate(root), _generate(root) + assert json.dumps(first) == json.dumps(second) + head = _git(root, "rev-parse", "HEAD").strip() + assert first["generation"] == { + "source_revision": head, + "generated_at": _git(root, "show", "-s", "--format=%cI", "HEAD").strip(), + "generator_version": projection.GENERATOR_VERSION} + assert first["generation"]["generated_at"] == ANCHOR_DATE + + pinned = _generate(root, source_revision="release-candidate", + generated_at="2026-01-02T03:04:05Z")["generation"] + assert (pinned["source_revision"], pinned["generated_at"]) == ( + "release-candidate", "2026-01-02T03:04:05Z") + unknown = _generate(root, source_revision="0" * 40)["generation"] + assert "generated_at" not in unknown + + +def test_a_repository_with_no_commit_needs_a_pinned_revision(tmp_path: Path) -> None: + root = _repository(tmp_path, files={"a.md": "# A\n"}, commit=False) + with pytest.raises(projection.ProjectionRefused) as caught: + _generate(root) + assert isinstance(caught.value, gs.GeneratorSeamError) + assert "source revision" in str(caught.value) + snapshot = _generate(root, source_revision="pinned-by-hand") + assert snapshot["generation"]["source_revision"] == "pinned-by-hand" + assert "generated_at" not in snapshot["generation"] + assert violations(snapshot) == [] + + +# --------------------------------------------------------------------------- +# 5 — the small neutral field set on openDox's default adapter +# --------------------------------------------------------------------------- + +def test_the_default_adapter_obliges_the_small_neutral_field_set(tmp_path: Path) -> None: + assert lga.NEUTRAL_FIELDS == ("title", "summary") + assert lga.WorkingTreeCorpus()._required_fields == lga.NEUTRAL_FIELDS + assert lga.LocalGitCorpus()._required_fields == () + root = _repository(tmp_path, files={ + "both.md": "title: T\nsummary: S\n", "none.md": "# N\n", + "empty.md": "title:\nsummary: S\n", "Makefile": "all:\n"}) + adapter, ref = _home(str(root)) + corpus = adapter.resolve(ref) + missing = {d.key: adapter.classify(corpus, d).missing_fields + for d in adapter.list_documents(corpus)} + assert missing == {"both.md": (), "none.md": ("title", "summary"), + "empty.md": ("title",), "Makefile": ()} + # a document without the fields is still read, as a source + assert _by_path(_generate(root))["none.md"]["stage"] == "source" + + +def test_required_header_fields_answers_the_neutral_set_through_the_entry_point() -> None: + from opendox import cli + + corpus_adapter._home_factory = corpus_adapter._UNSET + cli.build_parser() + assert authoring.required_header_fields() == ("title", "summary") + assert authoring.missing_required_headers("title: A note\n") == ["summary"] + + +def test_the_header_reader_keeps_an_empty_value_and_stops_at_a_blank_line() -> None: + assert lga.leading_header("a: 1\nno colon here\nb:\na: 2\n\nc: 3\n") == { + "a": "2", "b": ""} + + +# --------------------------------------------------------------------------- +# 6 — the product's views match the product's snapshot +# --------------------------------------------------------------------------- + +def _js_snapshot_values() -> dict[str, dict[str, str]]: + text = DISPLAY_JS.read_text(encoding="utf-8") + body = re.search(r"export const SNAPSHOT_VALUES = \{(.*?)\n\};", text, re.S).group(1) + return {name: dict(re.findall(r'(\w+): "([^"]*)"', table)) + for name, table in re.findall(r"(\w+): \{([^}]*)\}", body)} + + +def test_the_display_defaults_are_the_neutral_snapshots_values() -> None: + """R1Q11 (a), batch G's 5.3: the defaults are what openDox's own generator + writes, role for role, on both sides of the process boundary. The + candidate's four are NEUTRAL_DISPLAY's own words: no word is re-authored.""" + values = display_profile.SNAPSHOT_VALUES + assert values == { + "document_stage": {"captured": "source", "organized": "grouping"}, + "register_state": display_profile.NEUTRAL_DISPLAY["statuses"]["candidate"], + } + assert _js_snapshot_values() == values + defs = SCHEMA["$defs"] + assert set(values["document_stage"].values()) <= set(defs["stage_role"]["enum"]) + assert set(values["register_state"].values()) == set(defs["candidate_state"]["enum"]) + assert projection.SOURCE == values["document_stage"]["captured"] + assert projection.GROUPING == values["document_stage"]["organized"] + assert projection.UNSELECTED == values["register_state"]["captured"] + + +@pytest.mark.skipif(NODE is None, reason="node is not installed") +def test_the_wheel_counts_a_groups_edges_where_it_carries_no_tally(tmp_path: Path) -> None: + """The holder's ruling: the neutral schema is not widened with a tally, so + the wheel's grouping tile counts the group's edges. A group that carries a + tally, as the governed snapshot's do, still shows the tally.""" + snapshot = _generate(_repository(tmp_path, files=REPOSITORY_B)) + snapshot["clusters"].append({"id": "tallied", "name": "tallied", "topics": ["t"], + "document_edges": [], "tallies": {"document_links": 7}}) + script = tmp_path / "wheel.mjs" + script.write_text(textwrap.dedent(f""" + const W = await import({json.dumps(str(WHEEL_MODEL_JS))}); + const snap = {json.dumps(snapshot)}; + const model = W.buildWheelModel(snap); + const reel = model.wheels.find((w) => w.key === "grouping"); + console.log(JSON.stringify(reel.items.map((i) => [i.id, i.sub]))); + """), encoding="utf-8") + done = subprocess.run([NODE, str(script)], capture_output=True, text=True, timeout=120) + assert done.returncode == 0, done.stderr + subs = dict(json.loads(done.stdout.strip().splitlines()[-1])) + for cluster in snapshot["clusters"][:-1]: + assert subs[cluster["id"]].startswith(f"{len(cluster['document_edges'])} "), subs + assert subs["tallied"].startswith("7 "), subs + + +# --------------------------------------------------------------------------- +# 7 — no reach +# --------------------------------------------------------------------------- + +def test_the_projection_imports_with_no_sibling_and_no_third_party_module() -> None: + block = "".join(f"sys.modules[{name!r}] = None\n" for name in SIBLINGS) + program = (f"import sys; sys.path.insert(0, {str(SRC)!r})\n" + block + + textwrap.dedent(""" + before = set(sys.modules) + import opendox.neutral_projection, opendox.default_generator + added = {name.split(".")[0] for name in set(sys.modules) - before} + print(sorted(added - set(sys.stdlib_module_names) - {"opendox"})) + """)) + done = subprocess.run([sys.executable, "-c", program], capture_output=True, + text=True, cwd=str(ROOT), timeout=120) + assert done.returncode == 0, done.stderr + assert done.stdout.strip() == "[]", done.stdout + assert not hasattr(default_generator, "NeutralProjectionNotBuilt"), ( + "T052's refusal outlived the projection that retires it") From b2f92226714aab7173b563df3c182010a35b3105 Mon Sep 17 00:00:00 2001 From: Brett Heap <1513478+brettheap@users.noreply.github.com> Date: Sun, 27 Sep 2026 22:34:23 +0000 Subject: [PATCH 07/34] T054: compare the commit date as a time, as git spells it either way (plan 034) validate failed at 0f42f678 in one case, test_the_anchors_come_from_the_source_revision_and_the_bytes_repeat. CI's git prints `%cI` for a zero offset as `2026-09-27T12:00:00Z`, and the git this was written against (2.43.0) prints `2026-09-27T12:00:00+00:00`. The projection was right: it records the stamp exactly as git gives it, and the neutral schema admits both spellings. The test compared that stamp against the fixture's date as a string. It now compares the two as instants. The case's first assertion still holds generated_at to the checkout's own `git show` output, so a date read from anywhere else is still caught. Arc: neutral-product-standalone-operability Lane: openxfactory-4 (openXfactory-4-openDox_extraction) Co-Authored-By: Claude Opus 5.5 (1M context) --- tests/test_neutral_projection.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tests/test_neutral_projection.py b/tests/test_neutral_projection.py index 2c616708..2ccad059 100644 --- a/tests/test_neutral_projection.py +++ b/tests/test_neutral_projection.py @@ -64,6 +64,7 @@ import subprocess import sys import textwrap +from datetime import datetime from pathlib import Path from typing import Any, Callable, Iterator, NamedTuple @@ -777,7 +778,11 @@ def test_the_anchors_come_from_the_source_revision_and_the_bytes_repeat( "source_revision": head, "generated_at": _git(root, "show", "-s", "--format=%cI", "HEAD").strip(), "generator_version": projection.GENERATOR_VERSION} - assert first["generation"]["generated_at"] == ANCHOR_DATE + # The same instant as the commit's date. git spells a zero offset `Z` or + # `+00:00` depending on its version, and the stamp is recorded as git + # spells it, so the two are compared as times, not as strings. + assert datetime.fromisoformat(first["generation"]["generated_at"]) == \ + datetime.fromisoformat(ANCHOR_DATE) pinned = _generate(root, source_revision="release-candidate", generated_at="2026-01-02T03:04:05Z")["generation"] From c0747e965af9887a0ebe537d4ee0a70f7b879764 Mon Sep 17 00:00:00 2001 From: Brett Heap <1513478+brettheap@users.noreply.github.com> Date: Sun, 27 Sep 2026 22:41:20 +0000 Subject: [PATCH 08/34] T054: an empty topics header is a declaration; T050's docstring says what is true now (plan 034) Copilot's review of 0f42f678 left three threads. - r4117298988 (neutral_projection.py): a `topics:` header that listed nothing fell back to the derived topics, which contradicted the rule the module's own docstring states. The header is now a declaration even when it is empty. The document carries no topic, nothing is derived for it, and it joins no group. That follows the holder's literal-copy principle for title and summary. New case: test_an_empty_topics_header_declares_no_topic. With the old rule restored, it fails. - r4117299013 and r4117299033 (tests/test_plain_documents_fixture.py, T050's file, merged here from #53): its docstring said T052 and T054 had not landed. It also said the suite was run by node id outside validate's explicit list. T054 makes the first false. The second has been false since T036 (#52), when validate began running the whole suite. Both paragraphs, and the two phrases beside them ("a future topic-based grouping pass", "will require"), now say what is true. Only the docstring changes. Arc: neutral-product-standalone-operability Lane: openxfactory-4 (openXfactory-4-openDox_extraction) Co-Authored-By: Claude Opus 5.5 (1M context) --- src/opendox/neutral_projection.py | 11 +++++----- tests/test_neutral_projection.py | 16 ++++++++++++++ tests/test_plain_documents_fixture.py | 30 +++++++++++++-------------- 3 files changed, 36 insertions(+), 21 deletions(-) diff --git a/src/opendox/neutral_projection.py b/src/opendox/neutral_projection.py index b4e5cc10..a4eab457 100644 --- a/src/opendox/neutral_projection.py +++ b/src/opendox/neutral_projection.py @@ -33,9 +33,10 @@ the projection reports it, naming the document, the value and the six keys. So no other value can reach the snapshot, whose schema admits only the six. -THE TOPIC RULE. Every document carries `topics`. A document that declares a -`topics:` header carries the ones it lists, split on commas, with whitespace -collapsed and letters case-folded. A document that declares none carries the +THE TOPIC RULE. Every document carries `topics`. A document with a `topics:` +header carries exactly the topics it lists, split on commas, with whitespace +collapsed and letters case-folded, and none where it lists none: the header is +a declaration even when it is empty. A document without the header carries the words of its NAME, plus the words of the name of every other document it MENTIONS. Its name is its `title:`, else its first `#` heading, else its file name without the suffix. It mentions another document where its text holds @@ -318,8 +319,8 @@ def _read_documents(adapter: CorpusAdapter, corpus: ResolvedCorpus, f"its stage: value {declared!r} is not one of the six " f"station role keys ({', '.join(STAGE_ROLES)}), so it is " "not a declaration; the document is read as a source")) - listed_topics = _declared_topics(header.get(TOPICS_KEY) or "") - document.declared_topics = listed_topics or None + if TOPICS_KEY in header: # a declaration, even an empty one + document.declared_topics = _declared_topics(header[TOPICS_KEY]) document.name = (document.title or _first_heading(text) or document.stem) document.tokens = _tokens(text) diff --git a/tests/test_neutral_projection.py b/tests/test_neutral_projection.py index 2ccad059..72c98664 100644 --- a/tests/test_neutral_projection.py +++ b/tests/test_neutral_projection.py @@ -718,6 +718,22 @@ def test_declared_topics_replace_the_derived_ones(tmp_path: Path) -> None: assert violations(snapshot) == [] +def test_an_empty_topics_header_declares_no_topic(tmp_path: Path) -> None: + """A `topics:` header is a declaration even when it lists nothing: the + document carries no topic, and none is derived for it, so it joins no + group although its name shares words with two sources that do group.""" + root = _repository(tmp_path, files={ + "a.md": "title: Rain barrel\ntopics:\n\n.\n", + "b.md": "title: Rain barrel notes\n\n.\n", + "c.md": "title: Rain barrel log\n\n.\n", + }) + snapshot = _generate(root) + assert _by_path(snapshot)["a.md"]["topics"] == [] + [group] = snapshot["clusters"] + assert [e["document"] for e in group["document_edges"]] == ["b.md", "c.md"] + assert violations(snapshot) == [] + + def test_a_declared_group_gathers_the_sources_that_share_its_topics( tmp_path: Path) -> None: root = _repository(tmp_path, files={ diff --git a/tests/test_plain_documents_fixture.py b/tests/test_plain_documents_fixture.py index 6a0f23de..5a513495 100644 --- a/tests/test_plain_documents_fixture.py +++ b/tests/test_plain_documents_fixture.py @@ -5,10 +5,10 @@ 2, slice P2-F): a handful of `.md` documents a plain, ungoverned git repository could contain, read by AT-R1 (spec.md § "AT-R1 — the release-1 acceptance test", step 3(a)) and by F5.3, F7.2, F10.1 and F13.1 once those -falsifiers exist. None of it is wired to any `opendox` code yet — T052 and -T054 (the generator seam and the neutral projection) have not landed — so -this suite tests the fixture's own two guarantees rather than a projection -over it. +falsifiers exist. T054's neutral projection reads it: +`tests/test_neutral_projection.py` projects it with every sibling blocked and +validates the snapshot against T053's schema. This suite tests the fixture's +own two guarantees, which hold whatever reads it. THE TWO GUARANTEES, AND WHY. @@ -26,22 +26,20 @@ document that declares no `stage:` line at all is a SOURCE. This fixture carries one document per explicit station (`grouping`, `candidate`, `selection`, `submission`, `completion`) and three sources, two of which - share the phrase "rain barrel" verbatim so a future topic-based grouping - pass (T054) has a pair to find — the fixture must yield at least one + share the phrase "rain barrel" verbatim so T054's topic rule has a pair + to find — the fixture must yield at least one group, so AT-R1 can open the chat pane from a grouping tile (spec.md § AT-R1 steps 6-7). Every document also carries the small neutral field set the default adapter -will require regardless of station — `title` and `summary` — per T050's task -line and the answer's own example. - -NOT YET WIRED INTO `.github/workflows/validate.yml`'s explicit pytest list: -the phase-2 draft-ahead scope keeps this PR out of that file (conftest.py, -pyproject.toml, validate.yml and README.md are the phase-1 chain's), so this -suite runs by node id today, exactly as other narrowed-out suites in this -tree have (`tests/test_display_facet_leaves.py`'s own S7-residue history). -It joins the enumerated list whichever later task next touches it — most -likely F5.3/T056, or T049's own close. +requires regardless of station — `title` and `summary` +(`local_git_adapter.NEUTRAL_FIELDS`, T054) — per T050's task line and the +answer's own example. + +COLLECTED BY THE REQUIRED CHECK. Since T036, `validate` runs the whole suite +(`python -m pytest -q` over the configured testpaths) instead of an explicit +list of files, so this suite is collected like every other and needs no entry +anywhere. A CREATED file: no row in openxFactory's `docs/opendox-carve-manifest.yaml` (RULED OQ-C: the manifest declares what LEAVES openxFactory, never what a From 5a6fe637b20123fe6a1190702874a7ff563f0591 Mon Sep 17 00:00:00 2001 From: Brett Heap <1513478+brettheap@users.noreply.github.com> Date: Sun, 27 Sep 2026 22:53:27 +0000 Subject: [PATCH 09/34] T054: a word is a run of letters beside digits too; a pin labels the snapshot (plan 034) Copilot's review of c0747e96 raised two points. - Its overview said digit-adjacent topic names were tokenized wrongly, and they were. The docstring defines a word as a run of three or more letters, but the tokenizer took letter-and-digit runs and then dropped any run holding a digit, so `Q3planning` lost `planning`. Letter runs and digit runs are now separate tokens. New case: test_a_word_is_a_run_of_letters_even_beside_digits. With the old tokenizer restored, it fails. - r4117331489 asks that a supplied source_revision be passed into the CorpusRef. That is not taken, and the answer is on the product's own contract: - the seam defines source_revision as the source anchor to pin, and the CLI's help for --source-revision says "pin the source_revision anchor"; - openXdox's governed generator records a supplied revision and scans the tree it is handed; - the holder ruled for T054 that content comes from the working tree, per Brett's T022 ruling. default_generator's docstring now says so in a paragraph of its own. Only the docstring changes for it. Arc: neutral-product-standalone-operability Lane: openxfactory-4 (openXfactory-4-openDox_extraction) Co-Authored-By: Claude Opus 5.5 (1M context) --- src/opendox/default_generator.py | 9 +++++++++ src/opendox/neutral_projection.py | 5 +++-- tests/test_neutral_projection.py | 13 +++++++++++++ 3 files changed, 25 insertions(+), 2 deletions(-) diff --git a/src/opendox/default_generator.py b/src/opendox/default_generator.py index 3d516cab..1c814763 100644 --- a/src/opendox/default_generator.py +++ b/src/opendox/default_generator.py @@ -26,6 +26,15 @@ could read as an option. It is left out where the lookup fails. Nothing reads the clock, so the same tree at the same anchors answers the same snapshot. +A PIN LABELS THE SNAPSHOT; IT DOES NOT CHOOSE THE BYTES. The content is the +working tree's, on Brett's T022 ruling ("Working tree (Recommended)"), which the +holder confirmed for T054. A supplied `source_revision` is recorded as the +anchor the caller asserts, exactly as openXdox's governed generator records one +and scans the tree it is handed (its sealed-artifact lane pins a revision for a +tree that is not a checkout at all). So the pin never reaches the `CorpusRef`, +and a pin that names no commit here is still recorded rather than refused. A +caller that wants the tree at an older commit checks that commit out first. + WHAT IT REPORTS. The projection's notices go to standard error, one line each, `notice: : `. A `stage:` value outside the six role keys is one (R1Q13 (a)): the line names the document, the value and diff --git a/src/opendox/neutral_projection.py b/src/opendox/neutral_projection.py index a4eab457..d0addd82 100644 --- a/src/opendox/neutral_projection.py +++ b/src/opendox/neutral_projection.py @@ -147,8 +147,9 @@ will with within without would yet you your yours yourself yourselves """.split()) -#: A run of letters or digits: what the topic rule reads as one token. -_TOKEN = re.compile(r"[^\W_]+") +#: A run of letters, or a run of digits: what the topic rule reads as one +#: token. The two are split apart, so `Q3planning` holds the word `planning`. +_TOKEN = re.compile(r"[^\W\d_]+|\d+") #: An ATX heading (`# Roadmap`, `## Budget ##`), its text in group 1. _ATX_HEADING = re.compile(r"^ {0,3}#{1,6}(?:[ \t]+(.*?))?(?:[ \t]+#+)?[ \t]*$") diff --git a/tests/test_neutral_projection.py b/tests/test_neutral_projection.py index 72c98664..fb74cc77 100644 --- a/tests/test_neutral_projection.py +++ b/tests/test_neutral_projection.py @@ -734,6 +734,19 @@ def test_an_empty_topics_header_declares_no_topic(tmp_path: Path) -> None: assert violations(snapshot) == [] +def test_a_word_is_a_run_of_letters_even_beside_digits(tmp_path: Path) -> None: + """A word is a run of three or more letters: `Q3planning` holds `planning`, + so it groups with a source whose name spells the word on its own.""" + snapshot = _generate(_repository(tmp_path, files={ + "a.md": "title: Q3planning review\n\n.\n", + "b.md": "title: planning review\n\n.\n", + })) + assert _by_path(snapshot)["a.md"]["topics"] == ["planning", "review"] + [group] = snapshot["clusters"] + assert group["topics"] == ["planning", "review"] + assert violations(snapshot) == [] + + def test_a_declared_group_gathers_the_sources_that_share_its_topics( tmp_path: Path) -> None: root = _repository(tmp_path, files={ From bce09c50911686197c6910149bb8bf05ce149fc7 Mon Sep 17 00:00:00 2001 From: Brett Heap <1513478+brettheap@users.noreply.github.com> Date: Sun, 27 Sep 2026 23:15:04 +0000 Subject: [PATCH 10/34] T052: each registration keeps its own records at the generator seam Copilot's overview of openDox-code#57 at c0747e96 said that the generator seam's "replacement-generator tracking can reject valid registrations". It opened no thread. The claim is real, and there are two cases, both reproduced at 4ca45d27: 1. The count of generations under way was global, not tied to the registration it counted. A generation from a default that unregister() had dropped kept counting against whatever default was registered next. So a host was refused over a fresh default with "a snapshot is being generated from it now", although nothing was. 2. A generation that outlived its registration recorded its late answer against a fresh registration of the same declaration. That shut the fresh window, against unregister()'s own promise that the record of a generation goes with the registration. Now every change of registration goes through one helper: register(), register_default() where it registers, and unregister(). The helper moves a registration serial on and starts the two records afresh. A generation carries the serial it began under. When it ends, it touches the records only if that registration is still current. The generation is not stopped, and its caller still gets its snapshot. Tests: 66 -> 70 cases. The new case test_a_generation_that_outlives_its_registration_touches_no_later_one runs in four variants, in a process of its own. The follower is another default or the same declaration, and the host registers either while the generation runs or after it answers. Three variants are red against 4ca45d27's seam logic, and the fourth guards the new scoping. test_unregister_clears_the_default_and_its_generation now registers the same default afresh before the host tries. 15 of 15 mutations are caught. Arc: neutral-product-standalone-operability Lane: openxfactory-4 (openXfactory-4-openDox_extraction) Co-Authored-By: Claude Opus 5.5 (1M context) --- src/opendox/generator_seam.py | 89 ++++++++++++++++++++++------------- tests/test_generator_seam.py | 87 ++++++++++++++++++++++++++++++++-- 2 files changed, 139 insertions(+), 37 deletions(-) diff --git a/src/opendox/generator_seam.py b/src/opendox/generator_seam.py index a2c8184b..a15aecfe 100644 --- a/src/opendox/generator_seam.py +++ b/src/opendox/generator_seam.py @@ -110,6 +110,12 @@ is refused as `GeneratorAlreadyRegistered`. `unregister()` makes a deliberate swap explicit. +EACH REGISTRATION KEEPS ITS OWN RECORDS. A generation belongs to the +registration it began under. So a generation still under way when +`unregister()` drops that registration neither holds shut, nor closes, the +window of the registration that follows. That holds even for a later +registration of the same declaration. + ONE LOCK. The registration and its records are kept under one lock. `serve.py` answers each request on a thread of its own (a `ThreadingHTTPServer`), so a generation can run beside a registration. `generate()` holds the lock only for @@ -337,24 +343,45 @@ def _refuse_an_operation_that_cannot_take_the_call(self) -> None: #: (`register_default()`), rather than a host's own `register()`. _is_default: bool = False -#: Whether a snapshot has been generated from that default, meaning that its -#: operation has answered a snapshot the seam handed back. `generate()` sets it -#: once that snapshot has come back. A generation that failed, or whose answer -#: the seam refused, wrote nothing, and so records nothing. +#: Whether a snapshot has been generated from THIS registration of that +#: default, meaning that its operation has answered a snapshot the seam handed +#: back. `generate()` sets it once that snapshot has come back. A generation +#: that failed, or whose answer the seam refused, wrote nothing, and so records +#: nothing. _generated_from_default: bool = False -#: How many generations from that default are under way: begun, and not yet -#: answered or failed. While one is, a host's registration is refused, as it is -#: after one, because the snapshot being generated would come back after the -#: swap. `unregister()` leaves it alone: it counts calls that are still -#: running, and each one takes itself off when it ends. +#: How many generations from THIS registration of that default are under way: +#: begun, and not yet answered or failed. While one is, a host's registration +#: is refused, as it is after one, because the snapshot being generated would +#: come back after the swap. _default_generations_under_way: int = 0 -#: Guards the four above. It is held only for bookkeeping, and never across a +#: Which registration is current. Every change of registration moves it on: +#: `register()`, `register_default()` where it registers, and `unregister()`. +#: A generation carries the serial it began under, and when it ends it touches +#: the two records above only if that registration is still the current one. +#: So a change of registration starts both records afresh, and a generation +#: that outlived its registration records nothing against the next. +_registration_serial: int = 0 + +#: Guards the five above. It is held only for bookkeeping, and never across a #: generator's own call. _lock = threading.Lock() +def _begin_a_registration(generator: SnapshotGenerator | None, + is_default: bool) -> None: + """Make `generator` the registration (or none), with records of its own. + The one place the registration changes. The caller holds `_lock`.""" + global _registered, _is_default, _generated_from_default + global _default_generations_under_way, _registration_serial + _registration_serial += 1 + _registered = generator + _is_default = is_default + _generated_from_default = False + _default_generations_under_way = 0 + + def name_of(generator: Any) -> str: """A generator's most nameable name, with its contract, for a refusal. @@ -389,7 +416,6 @@ def register(generator: SnapshotGenerator) -> SnapshotGenerator: generated from it and none is being generated. It is refused once one has been, or while one is. Either way the host ends up holding the one registration, or knows why it does not.""" - global _registered, _is_default, _generated_from_default _require_a_declaration(generator, "register()") with _lock: held = _registered @@ -404,9 +430,7 @@ def register(generator: SnapshotGenerator) -> SnapshotGenerator: elif _generated_from_default: why = "a snapshot has already been generated from it" if not over_a_host and not why: - _registered = generator - _is_default = False - _generated_from_default = False + _begin_a_registration(generator, is_default=False) return generator # Named outside the lock: naming a generator can run its own code. if over_a_host: @@ -447,7 +471,6 @@ def register_default(generator: SnapshotGenerator) -> SnapshotGenerator: openDox's own generator to the neutral contract, so a declaration that writes another contract is refused as `GeneratorNotConformant`, whether or not anything is registered.""" - global _registered, _is_default, _generated_from_default _require_a_declaration(generator, "register_default()") if generator.contract != NEUTRAL_SNAPSHOT_KIND: raise GeneratorNotConformant( @@ -457,24 +480,22 @@ def register_default(generator: SnapshotGenerator) -> SnapshotGenerator: f"registered with {REGISTRATION_CALL}.") with _lock: if _registered is None: - _registered = generator - _is_default = True - _generated_from_default = False + _begin_a_registration(generator, is_default=True) return _registered def unregister() -> None: """Drop the registration, a host's or the entry point's default. - For test isolation and for a host tearing down. The record of a generation - from the default goes with it. A generation still under way is not stopped. - When it answers, it records its snapshot only if the same default is - registered at that moment.""" - global _registered, _is_default, _generated_from_default + For test isolation and for a host tearing down. The registration's records + go with it: whether a snapshot was generated from the default, and how many + generations from it are under way. A generation still under way is not + stopped, and its caller still gets its snapshot. But it belongs to the + registration that was dropped, so when it ends it records nothing against + whatever is registered next, even a later registration of the same + declaration.""" with _lock: - _registered = None - _is_default = False - _generated_from_default = False + _begin_a_registration(None, is_default=False) def is_registered() -> bool: @@ -534,6 +555,7 @@ def generate(repo_root: Path | str, repository: str, *, generator = current() undeclared = sorted(set(given) - set(generator.inputs)) from_default = _is_default and not undeclared + serial = _registration_serial if from_default: _default_generations_under_way += 1 if undeclared: @@ -553,20 +575,23 @@ def generate(repo_root: Path | str, repository: str, *, answered = True finally: if from_default: - _end_a_generation_from_the_default(generator, answered) + _end_a_generation_from_the_default(serial, answered) return snapshot -def _end_a_generation_from_the_default(generator: SnapshotGenerator, - answered: bool) -> None: +def _end_a_generation_from_the_default(serial: int, answered: bool) -> None: """Take a generation from the default off the count of those under way. If it answered a snapshot the seam handed back, record that one was generated. - The record is made only while the default it came from is still the one - registered, so after `unregister()` it records nothing against a host.""" + Both happen only while the registration it began under (`serial`) is still + the current one. A registration dropped since took its records with it. So + a generation that outlived its registration touches nothing, and records + nothing against what was registered after it.""" global _default_generations_under_way, _generated_from_default with _lock: + if serial != _registration_serial: + return _default_generations_under_way -= 1 - if answered and _registered is generator and _is_default: + if answered: _generated_from_default = True diff --git a/tests/test_generator_seam.py b/tests/test_generator_seam.py index 9f9ca4ea..62c9e43c 100644 --- a/tests/test_generator_seam.py +++ b/tests/test_generator_seam.py @@ -34,9 +34,11 @@ writes the neutral kind and takes no input. `register_default()` registers it only where nothing is, and holds it to the neutral contract. A host replaces it before a generation, and after one that wrote nothing. A host is - refused while a generation runs, and after one that wrote a snapshot. A - generator may register or generate from inside its own call without - deadlocking the seam. Until T054 lands, the default refuses, naming itself. + refused while a generation runs, and after one that wrote a snapshot. Each + registration keeps its own records: a generation that outlives its + registration records nothing against the next. A generator may register or + generate from inside its own call without deadlocking the seam. Until T054 + lands, the default refuses, naming itself. 7. EACH ENTRY POINT REGISTERS IT. `cli.build_parser()` and `cli.main()` run for real. `serve.build_server()` and `serve.main()` still cannot run in a lone checkout (research R7), so their registration is executed from their own @@ -98,14 +100,14 @@ def _isolated_registries(): point's default back as a host's. """ seam = (gs._registered, gs._is_default, gs._generated_from_default, - gs._default_generations_under_way) + gs._default_generations_under_way, gs._registration_serial) profile = (domain_profile._registered, domain_profile._is_default, domain_profile._built_from_default) home = corpus_adapter._home_factory gs.unregister() yield (gs._registered, gs._is_default, gs._generated_from_default, - gs._default_generations_under_way) = seam + gs._default_generations_under_way, gs._registration_serial) = seam (domain_profile._registered, domain_profile._is_default, domain_profile._built_from_default) = profile corpus_adapter._home_factory = home @@ -543,10 +545,13 @@ def test_asking_or_being_refused_is_not_a_generation(tmp_path) -> None: def test_unregister_clears_the_default_and_its_generation(tmp_path) -> None: + """The record goes with the registration. So the same default, registered + afresh, has a window of its own, and a host still replaces it.""" stand_in_default, _ = _declared(gs.NEUTRAL_SNAPSHOT_KIND) gs.register_default(stand_in_default) gs.generate(tmp_path, "fixture") gs.unregister() + assert gs.register_default(stand_in_default) is stand_in_default host, _ = _declared("host-snapshot") assert gs.register(host) is host @@ -719,6 +724,78 @@ def swapping(repo_root, repository, *, source_revision=None, "host-registered-over-the-replacement"] +#: A program for a fresh process. A generation from one registration of the +#: default is still under way when `unregister()` drops that registration, and +#: a default is registered after it. `FOLLOWING` says which default: another, +#: or the same declaration again. `WHEN` says when a host then registers: while +#: that generation still runs, or after it has answered. +_OUTLIVED = """ + import threading + from opendox import generator_seam as gs + FOLLOWING, WHEN = FOLLOWING_VALUE, WHEN_VALUE + KIND = gs.NEUTRAL_SNAPSHOT_KIND + started, release = threading.Event(), threading.Event() + + def slow(repo_root, repository, *, source_revision=None, generated_at=None): + started.set() + release.wait(30) + return {"schema_version": 1, "kind": KIND, "repository": repository} + + def quick(repo_root, repository, *, source_revision=None, generated_at=None): + return {"schema_version": 1, "kind": KIND, "repository": repository} + + def host_operation(repo_root, repository, *, source_revision=None, + generated_at=None): + return {"schema_version": 1, "kind": "host-snapshot"} + + dropped = gs.SnapshotGenerator(contract=KIND, generate=slow) + host = gs.SnapshotGenerator(contract="host-snapshot", generate=host_operation) + answers = [] + gs.register_default(dropped) + worker = threading.Thread( + target=lambda: answers.append(gs.generate(".", "outlived")["repository"])) + worker.start() + assert started.wait(30), "the generation never started" + gs.unregister() + gs.register_default(dropped if FOLLOWING == "the same declaration" + else gs.SnapshotGenerator(contract=KIND, generate=quick)) + + def try_the_host(): + try: + return "registered" if gs.register(host) is host else "not-registered" + except gs.GeneratorAlreadyRegistered: + return "refused" + + if WHEN == "while it runs": + result = try_the_host() + release.set() + worker.join(30) + assert not worker.is_alive(), "the generation never ended" + if WHEN == "after it answers": + result = try_the_host() + print(result, answers[0] if answers else "no-answer", + gs._default_generations_under_way) +""" + + +@pytest.mark.parametrize("following", ("another default", "the same declaration")) +@pytest.mark.parametrize("when", ("while it runs", "after it answers")) +def test_a_generation_that_outlives_its_registration_touches_no_later_one( + following: str, when: str) -> None: + """Each registration keeps its own records. A generation still under way + when `unregister()` drops its registration neither holds shut, nor closes, + the window of the registration that follows. That holds whether the + follower is another default or the same declaration registered again, and + whether a host registers while that generation still runs or after it has + answered. The generation is not stopped, and its caller still gets its + snapshot. The case runs in a process of its own (see `_fresh_process`).""" + program = (_OUTLIVED.replace("FOLLOWING_VALUE", repr(following)) + .replace("WHEN_VALUE", repr(when))) + done = _fresh_process(program) + assert done.returncode == 0, done.stderr + assert done.stdout.split() == ["registered", "outlived", "0"] + + def test_a_host_that_registers_the_default_itself_holds_a_hosts_registration() -> None: """Which kind a registration is, is set by the call that MADE it.""" gs.register(default_generator.GENERATOR) From 97b5b01fb29ea18e1edb456724a34a3e62c0942d Mon Sep 17 00:00:00 2001 From: Brett Heap <1513478+brettheap@users.noreply.github.com> Date: Sun, 27 Sep 2026 23:16:50 +0000 Subject: [PATCH 11/34] T052: leave point 6 of the seam test's docstring as T054 edits it T054's branch (openDox-code#57) stacks on this one and rewrites the last sentence of point 6 of tests/test_generator_seam.py's docstring. The previous commit re-wrapped that same paragraph, so taking this branch would have conflicted there. Point 6 is restored to its 4ca45d27 text, and the new sentence moves to point 5, ONE REGISTRATION, where it belongs. A trial merge of this head into #57's head 5a6fe637 is clean, and 94 seam and projection cases pass in the merged tree. Arc: neutral-product-standalone-operability Lane: openxfactory-4 (openXfactory-4-openDox_extraction) Co-Authored-By: Claude Opus 5.5 (1M context) --- tests/test_generator_seam.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/tests/test_generator_seam.py b/tests/test_generator_seam.py index 62c9e43c..f3876e46 100644 --- a/tests/test_generator_seam.py +++ b/tests/test_generator_seam.py @@ -29,16 +29,15 @@ undeclared input is refused before the call. A `None` input is not passed. Only a snapshot of the declared contract comes back. 5. ONE REGISTRATION. The same declaration twice is a no-op, and a second host - is refused. + is refused. Each registration keeps its own records: a generation that + outlives its registration records nothing against the next. 6. THE ENTRY POINTS' DEFAULT, in R1Q3 (a)'s pattern. openDox's own generator writes the neutral kind and takes no input. `register_default()` registers it only where nothing is, and holds it to the neutral contract. A host replaces it before a generation, and after one that wrote nothing. A host is - refused while a generation runs, and after one that wrote a snapshot. Each - registration keeps its own records: a generation that outlives its - registration records nothing against the next. A generator may register or - generate from inside its own call without deadlocking the seam. Until T054 - lands, the default refuses, naming itself. + refused while a generation runs, and after one that wrote a snapshot. A + generator may register or generate from inside its own call without + deadlocking the seam. Until T054 lands, the default refuses, naming itself. 7. EACH ENTRY POINT REGISTERS IT. `cli.build_parser()` and `cli.main()` run for real. `serve.build_server()` and `serve.main()` still cannot run in a lone checkout (research R7), so their registration is executed from their own From 8d507c5be2bbe9cc37d15a1106c368afd64a6598 Mon Sep 17 00:00:00 2001 From: Brett Heap <1513478+brettheap@users.noreply.github.com> Date: Sun, 27 Sep 2026 23:33:34 +0000 Subject: [PATCH 12/34] T052: current() reads the registration once Copilot's review of this PR at bce09c50 (r4117421102) found that current() read _registered twice: once for its None check, and once for its answer. A thread that unregistered between the two reads made it answer None after passing the check, against its SnapshotGenerator-or- refusal contract. generate() calls it while holding the seam's lock, so that path was safe, but a bare caller was not. current() now reads the registration into a local once, and answers that local. It still takes no lock of its own, because generate() holds the seam's lock, which is not re-entrant, when it calls it. Test: test_current_answers_the_registration_it_checked forces the interleaving deterministically. A line tracer drops the registration before every line of current() after its first. The test is red against 97b5b01f's seam logic ("current() answered None after passing its check") and green here. 16 of 16 mutations are caught, the new M16 among them. Arc: neutral-product-standalone-operability Lane: openxfactory-4 (openXfactory-4-openDox_extraction) Co-Authored-By: Claude Opus 5.5 (1M context) --- src/opendox/generator_seam.py | 12 +++++++++--- tests/test_generator_seam.py | 36 +++++++++++++++++++++++++++++++++++ 2 files changed, 45 insertions(+), 3 deletions(-) diff --git a/src/opendox/generator_seam.py b/src/opendox/generator_seam.py index a15aecfe..a4d62b59 100644 --- a/src/opendox/generator_seam.py +++ b/src/opendox/generator_seam.py @@ -509,8 +509,14 @@ def current() -> SnapshotGenerator: It answers what is registered: a host's generator, or the default an entry point registered. It never falls back to the default itself, and asking records nothing. Only a generation closes the default's window, while it - runs and once it has answered.""" - if _registered is None: + runs and once it has answered. + + It reads the registration ONCE. So a registration another thread drops + while it answers never makes it answer `None`: it answers the registration + it checked. It takes no lock of its own, because `generate()` calls it + while holding the seam's lock.""" + registered = _registered + if registered is None: raise GeneratorNotRegistered( "no snapshot generator is registered at openDox's generator seam " "(opendox.generator_seam), so there is nothing to generate a " @@ -525,7 +531,7 @@ def current() -> SnapshotGenerator: "dropped the registration since. A host that contributes its own " "generator registers it at process start with\n\n " + REGISTRATION_CALL + "\n\nbefore the first generation.") - return _registered + return registered def generate(repo_root: Path | str, repository: str, *, diff --git a/tests/test_generator_seam.py b/tests/test_generator_seam.py index f3876e46..7b32ef17 100644 --- a/tests/test_generator_seam.py +++ b/tests/test_generator_seam.py @@ -236,6 +236,42 @@ def test_generate_with_nothing_registered_refuses_as_the_seam(tmp_path) -> None: assert gs.is_registered() is False +def test_current_answers_the_registration_it_checked() -> None: + """`current()` reads the registration once. `generate()` calls it holding + the seam's lock, but a host or a test may call it bare while another thread + unregisters. So a registration dropped between its check and its answer + never makes it answer `None`: it answers the registration it checked. + + The interleaving is forced, deterministically. A line tracer drops the + registration before every line of `current()` after its first, which is + where another thread's `unregister()` could land.""" + declared, _ = _declared() + gs.register(declared) + lines = [] + + def inside_current(frame, event, arg): + if event == "line": + lines.append(frame.f_lineno) + if len(lines) > 1: + gs.unregister() + return inside_current + + def tracer(frame, event, arg): + return inside_current if frame.f_code is gs.current.__code__ else None + + previous = sys.gettrace() + sys.settrace(tracer) + try: + answer = gs.current() + finally: + sys.settrace(previous) + assert len(lines) > 1, "the tracer never reached a second line of current()" + assert gs.is_registered() is False, "the interleaved unregister() never ran" + assert answer is declared, ( + f"current() answered {answer!r} after passing its check, not the " + "registration it checked") + + # -------------------------------------------------------------------------- # 3 — the declaration is checked when it is made # -------------------------------------------------------------------------- From d89f252a2944f7c9c2a04582b727ff7b4a14d8d0 Mon Sep 17 00:00:00 2001 From: Brett Heap <1513478+brettheap@users.noreply.github.com> Date: Mon, 28 Sep 2026 16:21:16 +0000 Subject: [PATCH 13/34] T057: openDox's own validator, over its spec leg's four schemas (plan 034) #1144's 7.1, 7.1a, 7.1b and 7.2, with 7.1 as T007's batch G amends it (R1Q11 (a) and R1Q12 (a), openxFactory#656 comment 5850003126). - src/opendox/contracts/ holds the four packaged copies: ideation-workbench, opendox-snapshot, xfactory-workbench-chat-turn and xfactory-workbench-model-catalog. Each is byte for byte openDox-spec's file at cd49eb25, the head of openDox-spec#16 (T053). copies.yaml is the record that pins each copy's sha256. A copy is proved against it before a byte of the copy is read, and a changed, absent or unpinned copy is refused. - src/opendox/validator.py is the validator, new surface at the code leg (7.2). It evaluates JSON Schema 2020-12, exactly the keywords the four use. A refusal names its rule as []. It implements the neutral snapshot's seven reference rules, and it refuses a copy that uses anything it does not evaluate. Its docstring records why the consumer's script cannot be reused (7.1a), measured at openXdox-code 4610bca5. - tests/test_validator_input_set.py is the falsifier: the packaged-copy digest test and the 7.1b test, with the identity checks. - tests/test_validator.py tests the evaluator over openDox-spec's own 47 examples, which cover all 32 rules, keyword by keyword, and over openDox's own projection of T050's and T051's fixtures. tests/fixtures/spec-examples/ is that corpus. Held, and reported to the holder: the package-data line in pyproject.toml for opendox.contracts. The phase-2 scope rule keeps drafts out of that file. Arc: neutral-product-standalone-operability Lane: openxfactory-4 (openXfactory-4-openDox_extraction) Co-Authored-By: Claude Opus 5.5 (1M context) --- src/opendox/contracts/__init__.py | 222 +++++ src/opendox/contracts/copies.yaml | 44 + .../schemas/ideation-workbench.schema.yaml | 227 +++++ .../schemas/opendox-snapshot.schema.yaml | 443 ++++++++++ .../xfactory-workbench-chat-turn.schema.yaml | 414 +++++++++ ...actory-workbench-model-catalog.schema.yaml | 230 +++++ src/opendox/validator.py | 808 ++++++++++++++++++ ...on-workbench-adhoc-human-seen.example.yaml | 29 + ...tion-workbench-cluster-seeded.example.yaml | 41 + ...ndox-snapshot-candidate-keys.negative.yaml | 50 ++ ...shot-candidate-names-a-group.negative.yaml | 57 ++ ...hot-candidate-state-is-known.negative.yaml | 50 ++ ...-closed-candidate-has-reason.negative.yaml | 50 ++ ...endox-snapshot-document-keys.negative.yaml | 43 + .../opendox-snapshot-edge-keys.negative.yaml | 50 ++ ...apshot-edge-names-a-document.negative.yaml | 50 ++ ...endox-snapshot-envelope-keys.negative.yaml | 49 ++ ...shot-generated-at-is-rfc3339.negative.yaml | 53 ++ ...snapshot-generation-anchored.negative.yaml | 50 ++ ...x-snapshot-group-has-a-topic.negative.yaml | 50 ++ .../opendox-snapshot-group-keys.negative.yaml | 49 ++ .../opendox-snapshot-id-is-text.negative.yaml | 50 ++ ...ndox-snapshot-ids-are-unique.negative.yaml | 50 ++ ...-snapshot-keyword-entry-keys.negative.yaml | 56 ++ ...keyword-index-matches-topics.negative.yaml | 56 ++ ...hot-kind-is-opendox-snapshot.negative.yaml | 50 ++ ...apshot-one-edge-per-document.negative.yaml | 51 ++ ...apshot-path-is-repo-relative.negative.yaml | 50 ++ ...pshot-pick-names-a-selection.negative.yaml | 57 ++ ...-snapshot-repository-is-text.negative.yaml | 50 ++ ...snapshot-schema-version-is-1.negative.yaml | 50 ++ ...x-snapshot-section-is-a-list.negative.yaml | 50 ++ ...-selected-candidate-has-pick.negative.yaml | 50 ++ ...ndox-snapshot-selection-keys.negative.yaml | 50 ++ ...shot-stage-is-a-station-role.negative.yaml | 50 ++ ...dox-snapshot-submission-keys.negative.yaml | 50 ++ ...t-submission-status-is-known.negative.yaml | 50 ++ ...ot-target-names-a-submission.negative.yaml | 50 ++ ...t-title-and-summary-are-text.negative.yaml | 50 ++ ...apshot-topic-is-trimmed-text.negative.yaml | 50 ++ ...x-snapshot-topics-are-unique.negative.yaml | 50 ++ ...ndox-snapshot-no-front-matter.example.yaml | 54 ++ ...opendox-snapshot-six-stations.example.yaml | 164 ++++ ...nch-chat-turn-v2-full-context.example.yaml | 33 + ...bench-chat-turn-v2-loaded-set.example.yaml | 58 ++ ...h-chat-turn-v2-provider-retry.example.yaml | 49 ++ ...-chat-turn-v2-reduced-context.example.yaml | 33 + ...orkbench-chat-turn-v2-success.example.yaml | 42 + ...workbench-model-catalog-empty.example.yaml | 6 + ...catalog-hosted-zero-retention.example.yaml | 16 + ...workbench-model-catalog-local.example.yaml | 25 + ...ench-model-catalog-multimodal.example.yaml | 27 + ...er-than-a-non-resolved-member.example.yaml | 52 ++ ...ch-model-catalog-routing-rule.example.yaml | 49 ++ tests/test_validator.py | 552 ++++++++++++ tests/test_validator_input_set.py | 293 +++++++ 56 files changed, 5532 insertions(+) create mode 100644 src/opendox/contracts/__init__.py create mode 100644 src/opendox/contracts/copies.yaml create mode 100644 src/opendox/contracts/schemas/ideation-workbench.schema.yaml create mode 100644 src/opendox/contracts/schemas/opendox-snapshot.schema.yaml create mode 100644 src/opendox/contracts/schemas/xfactory-workbench-chat-turn.schema.yaml create mode 100644 src/opendox/contracts/schemas/xfactory-workbench-model-catalog.schema.yaml create mode 100644 src/opendox/validator.py create mode 100644 tests/fixtures/spec-examples/ideation-workbench-adhoc-human-seen.example.yaml create mode 100644 tests/fixtures/spec-examples/ideation-workbench-cluster-seeded.example.yaml create mode 100644 tests/fixtures/spec-examples/negative/opendox-snapshot-candidate-keys.negative.yaml create mode 100644 tests/fixtures/spec-examples/negative/opendox-snapshot-candidate-names-a-group.negative.yaml create mode 100644 tests/fixtures/spec-examples/negative/opendox-snapshot-candidate-state-is-known.negative.yaml create mode 100644 tests/fixtures/spec-examples/negative/opendox-snapshot-closed-candidate-has-reason.negative.yaml create mode 100644 tests/fixtures/spec-examples/negative/opendox-snapshot-document-keys.negative.yaml create mode 100644 tests/fixtures/spec-examples/negative/opendox-snapshot-edge-keys.negative.yaml create mode 100644 tests/fixtures/spec-examples/negative/opendox-snapshot-edge-names-a-document.negative.yaml create mode 100644 tests/fixtures/spec-examples/negative/opendox-snapshot-envelope-keys.negative.yaml create mode 100644 tests/fixtures/spec-examples/negative/opendox-snapshot-generated-at-is-rfc3339.negative.yaml create mode 100644 tests/fixtures/spec-examples/negative/opendox-snapshot-generation-anchored.negative.yaml create mode 100644 tests/fixtures/spec-examples/negative/opendox-snapshot-group-has-a-topic.negative.yaml create mode 100644 tests/fixtures/spec-examples/negative/opendox-snapshot-group-keys.negative.yaml create mode 100644 tests/fixtures/spec-examples/negative/opendox-snapshot-id-is-text.negative.yaml create mode 100644 tests/fixtures/spec-examples/negative/opendox-snapshot-ids-are-unique.negative.yaml create mode 100644 tests/fixtures/spec-examples/negative/opendox-snapshot-keyword-entry-keys.negative.yaml create mode 100644 tests/fixtures/spec-examples/negative/opendox-snapshot-keyword-index-matches-topics.negative.yaml create mode 100644 tests/fixtures/spec-examples/negative/opendox-snapshot-kind-is-opendox-snapshot.negative.yaml create mode 100644 tests/fixtures/spec-examples/negative/opendox-snapshot-one-edge-per-document.negative.yaml create mode 100644 tests/fixtures/spec-examples/negative/opendox-snapshot-path-is-repo-relative.negative.yaml create mode 100644 tests/fixtures/spec-examples/negative/opendox-snapshot-pick-names-a-selection.negative.yaml create mode 100644 tests/fixtures/spec-examples/negative/opendox-snapshot-repository-is-text.negative.yaml create mode 100644 tests/fixtures/spec-examples/negative/opendox-snapshot-schema-version-is-1.negative.yaml create mode 100644 tests/fixtures/spec-examples/negative/opendox-snapshot-section-is-a-list.negative.yaml create mode 100644 tests/fixtures/spec-examples/negative/opendox-snapshot-selected-candidate-has-pick.negative.yaml create mode 100644 tests/fixtures/spec-examples/negative/opendox-snapshot-selection-keys.negative.yaml create mode 100644 tests/fixtures/spec-examples/negative/opendox-snapshot-stage-is-a-station-role.negative.yaml create mode 100644 tests/fixtures/spec-examples/negative/opendox-snapshot-submission-keys.negative.yaml create mode 100644 tests/fixtures/spec-examples/negative/opendox-snapshot-submission-status-is-known.negative.yaml create mode 100644 tests/fixtures/spec-examples/negative/opendox-snapshot-target-names-a-submission.negative.yaml create mode 100644 tests/fixtures/spec-examples/negative/opendox-snapshot-title-and-summary-are-text.negative.yaml create mode 100644 tests/fixtures/spec-examples/negative/opendox-snapshot-topic-is-trimmed-text.negative.yaml create mode 100644 tests/fixtures/spec-examples/negative/opendox-snapshot-topics-are-unique.negative.yaml create mode 100644 tests/fixtures/spec-examples/opendox-snapshot-no-front-matter.example.yaml create mode 100644 tests/fixtures/spec-examples/opendox-snapshot-six-stations.example.yaml create mode 100644 tests/fixtures/spec-examples/workbench-chat-turn-v2-full-context.example.yaml create mode 100644 tests/fixtures/spec-examples/workbench-chat-turn-v2-loaded-set.example.yaml create mode 100644 tests/fixtures/spec-examples/workbench-chat-turn-v2-provider-retry.example.yaml create mode 100644 tests/fixtures/spec-examples/workbench-chat-turn-v2-reduced-context.example.yaml create mode 100644 tests/fixtures/spec-examples/workbench-chat-turn-v2-success.example.yaml create mode 100644 tests/fixtures/spec-examples/workbench-model-catalog-empty.example.yaml create mode 100644 tests/fixtures/spec-examples/workbench-model-catalog-hosted-zero-retention.example.yaml create mode 100644 tests/fixtures/spec-examples/workbench-model-catalog-local.example.yaml create mode 100644 tests/fixtures/spec-examples/workbench-model-catalog-multimodal.example.yaml create mode 100644 tests/fixtures/spec-examples/workbench-model-catalog-routing-rule-wider-than-a-non-resolved-member.example.yaml create mode 100644 tests/fixtures/spec-examples/workbench-model-catalog-routing-rule.example.yaml create mode 100644 tests/test_validator.py create mode 100644 tests/test_validator_input_set.py diff --git a/src/opendox/contracts/__init__.py b/src/opendox/contracts/__init__.py new file mode 100644 index 00000000..d39ecbf6 --- /dev/null +++ b/src/opendox/contracts/__init__.py @@ -0,0 +1,222 @@ +"""THE PACKAGED COPIES of openDox's spec-leg contracts, and their identity. + +WHY THIS PACKAGE EXISTS. Plan 034's T057 realizes #1144's 7.1, which T007's +batch G amends on R1Q11 (a) and R1Q12 (a) (`openxFactory#656` comment +`5850003126`): *"openDox's validator validates its spec leg's FOUR kinds ... +The code leg carries digest-checked copies of the four, which a test holds to +the spec-leg commit the openDox root pins."* 7.1 also settles how the four +reach an install. They ship as PACKAGE DATA, *"so `pip install openDox-code` +puts them on disk beside the validator and the assembly root remains their +source of truth for editing"*. So a code-leg checkout with no assembly root +around it still has them, and so does an install. `opendox.validator` reads +them from here, and T085's doxBench validators will read the same copies. + +WHAT IS HERE. + +* `schemas/`: the four copies. Each one is byte for byte the spec leg's file + of the same name, `contracts/schemas/.schema.yaml` in + opensoft/openDox-spec. +* `copies.yaml`: the record. It names the spec-leg commit the copies were taken + at, and each copy's id, spec-leg path and sha256. + +PRESENCE IS NOT IDENTITY. A copy is read only through `verified_bytes()`. It +recomputes the copy's sha256 and compares it with the record BEFORE a byte of +the copy is parsed, and it refuses, with `CopyRefused`, a copy that differs +from its digest, a copy that is absent, and a copy whose digest the record +leaves empty. That is `neutral-product-pin`'s rule for a vendored contract +(*"A vendored foreign contract is digest-verified before it is read"*): a +recomputed digest can never equal an empty recorded one, so an empty digest is +drift, and a file that merely exists proves nothing. + +CONSUMED, NOT OWNED. openDox-spec owns these four schemas. The code leg +releases none of them, and a copy is changed in the spec leg and then copied +here again, never edited here. The copies, the record, and the `commit` it +names move together, in one commit. + +IMPORT WEIGHT. The standard library, and PyYAML (the package's one runtime +dependency) when the record is read. Importing this package reads no file and +names no sibling. + +A CREATED FILE: it has no row in openxFactory's +`docs/opendox-carve-manifest.yaml`, because the manifest declares what LEAVES +openxFactory and never what a destination assembles (RULED OQ-C). +""" + +from __future__ import annotations + +import hashlib +import re +from dataclasses import dataclass +from importlib import resources +from pathlib import PurePosixPath +from typing import Any + +__all__ = [ + "COPY_KIND", + "CopyRefused", + "PackagedCopy", + "Record", + "RECORD_NAME", + "SPEC_LEG", + "load", + "record", + "verified_bytes", +] + +#: The record's file name, beside this module. +RECORD_NAME = "copies.yaml" + +#: The record's `kind`. +COPY_KIND = "packaged-contract-copies" + +#: The repository every copy comes from: openDox's own spec leg. +SPEC_LEG = "opensoft/openDox-spec" + +#: Where a copy sits under this package: `schemas/`. +SCHEMA_DIR = "schemas" + +_ID = re.compile(r"[a-z0-9]+(?:-[a-z0-9]+)*") +_COMMIT = re.compile(r"[0-9a-f]{40}") +_SHA256 = re.compile(r"[0-9a-f]{64}") + + +class CopyRefused(RuntimeError): + """A packaged copy, or the record that pins the copies, cannot be trusted. + + Raised before any byte of the copy is parsed. The message names the copy, + what was found, and the one remedy: copy the spec leg's file again at the + recorded commit, and record its digest in the same commit.""" + + +@dataclass(frozen=True) +class PackagedCopy: + """One copy, as the record declares it.""" + + id: str + path: str # its path in the spec leg: contracts/schemas/.schema.yaml + sha256: str # its digest at the recorded commit + + @property + def resource(self) -> str: + """Where the copy sits under this package.""" + return f"{SCHEMA_DIR}/{PurePosixPath(self.path).name}" + + +@dataclass(frozen=True) +class Record: + """`copies.yaml`, read and checked.""" + + spec_leg: str + commit: str + copies: tuple[PackagedCopy, ...] + + @property + def ids(self) -> tuple[str, ...]: + return tuple(copy.id for copy in self.copies) + + def copy(self, copy_id: str) -> PackagedCopy: + for copy in self.copies: + if copy.id == copy_id: + return copy + raise CopyRefused( + f"{copy_id!r} is not one of the packaged copies {list(self.ids)} " + f"({RECORD_NAME} records no such copy, so none is read)") + + +def _refuse(detail: str) -> CopyRefused: + return CopyRefused( + f"{RECORD_NAME} cannot be trusted: {detail}. The record is written " + f"with the copies it pins, in one commit, from {SPEC_LEG} at the " + "commit the openDox root pins") + + +def _read_package_file(name: str) -> bytes: + try: + return resources.files(__name__).joinpath(name).read_bytes() + except (FileNotFoundError, IsADirectoryError, NotADirectoryError) as exc: + raise CopyRefused( + f"opendox.contracts has no {name}: the package was built or " + f"installed without it ({type(exc).__name__})") from exc + + +def record() -> Record: + """The record, read from the package and checked field by field. + + Refuses, with `CopyRefused`, a record it cannot hold every copy to: a + missing or malformed field, an unknown field, a repeated id, a path that + is not the id's schema path in the spec leg, and an empty or malformed + digest.""" + import yaml + + try: + data = yaml.safe_load(_read_package_file(RECORD_NAME)) + except yaml.YAMLError as exc: + raise _refuse(f"it is not YAML ({exc.__class__.__name__})") from exc + if not isinstance(data, dict): + raise _refuse(f"it is a {type(data).__name__}, not a mapping") + expected = {"schema_version", "kind", "spec_leg", "commit", "copies"} + if set(data) != expected: + raise _refuse(f"its keys are {sorted(data)}, not {sorted(expected)}") + if data["schema_version"] != 1 or isinstance(data["schema_version"], bool): + raise _refuse(f"schema_version is {data['schema_version']!r}, not 1") + if data["kind"] != COPY_KIND: + raise _refuse(f"kind is {data['kind']!r}, not {COPY_KIND!r}") + if data["spec_leg"] != SPEC_LEG: + raise _refuse(f"spec_leg is {data['spec_leg']!r}, not {SPEC_LEG!r}") + commit = data["commit"] + if not isinstance(commit, str) or not _COMMIT.fullmatch(commit): + raise _refuse(f"commit is {commit!r}, not a full 40-hex commit id") + entries = data["copies"] + if not isinstance(entries, list) or not entries: + raise _refuse("copies is not a non-empty list") + copies: list[PackagedCopy] = [] + for index, entry in enumerate(entries): + where = f"copies[{index}]" + if not isinstance(entry, dict) or set(entry) != {"id", "path", "sha256"}: + raise _refuse(f"{where} is not a mapping of exactly id, path and sha256") + copy_id, path, digest = entry["id"], entry["path"], entry["sha256"] + if not isinstance(copy_id, str) or not _ID.fullmatch(copy_id): + raise _refuse(f"{where}.id is {copy_id!r}, not a lowercase hyphenated id") + if path != f"contracts/schemas/{copy_id}.schema.yaml": + raise _refuse(f"{where}.path is {path!r}, not " + f"'contracts/schemas/{copy_id}.schema.yaml'") + if not isinstance(digest, str) or not _SHA256.fullmatch(digest): + # An EMPTY digest lands here too: it is drift, never a pass. + raise _refuse(f"{where}.sha256 is {digest!r}, not a 64-hex sha256") + if any(copy.id == copy_id for copy in copies): + raise _refuse(f"{where}.id {copy_id!r} is recorded twice") + copies.append(PackagedCopy(copy_id, path, digest)) + return Record(SPEC_LEG, commit, tuple(copies)) + + +def verified_bytes(copy_id: str) -> bytes: + """The copy's bytes, once they are proved to be the recorded ones. + + Reads the copy, recomputes its sha256, and refuses with `CopyRefused` + unless it equals the record's digest. Nothing is parsed before that + comparison, so a caller never reads a copy whose identity is unproved.""" + pins = record() + pinned = pins.copy(copy_id) + data = _read_package_file(pinned.resource) + actual = hashlib.sha256(data).hexdigest() + if actual != pinned.sha256: + raise CopyRefused( + f"the packaged copy of {copy_id} ({pinned.resource}) is not the " + f"file the record pins: its sha256 is {actual}, and {RECORD_NAME} " + f"records {pinned.sha256} for {pinned.path} in {SPEC_LEG} at " + f"{pins.commit}. A copy is never edited in place: copy the spec " + "leg's file again, and record its digest in the same commit") + return data + + +def load(copy_id: str) -> Any: + """The copy, parsed, after `verified_bytes()` has proved its identity.""" + import yaml + + data = verified_bytes(copy_id) + try: + return yaml.safe_load(data) + except yaml.YAMLError as exc: + raise CopyRefused( + f"the packaged copy of {copy_id} matches its digest but is not " + f"YAML ({exc.__class__.__name__})") from exc diff --git a/src/opendox/contracts/copies.yaml b/src/opendox/contracts/copies.yaml new file mode 100644 index 00000000..d84a9e79 --- /dev/null +++ b/src/opendox/contracts/copies.yaml @@ -0,0 +1,44 @@ +# THE RECORD of openDox's packaged contract copies (plan 034 T057; #1144 7.1, +# as T007's batch G amends it; R1Q12 (a), opensoft/openxFactory#656 comment +# 5850003126). +# +# WHAT IT SAYS. Each schema under `schemas/` beside this file is, byte for byte, +# the file its `path` names in `spec_leg` at `commit`, and `sha256` is that +# file's digest. `opendox.contracts.verified_bytes` recomputes the digest before +# a copy is read, and refuses a copy that differs, a copy that is absent, and a +# copy whose digest is left empty. +# +# WHERE THE VALUES CAME FROM, read with `git show : | sha256sum` +# in opensoft/openDox-spec. +# +# * `commit` is the head of openDox-spec#16 (T053), which is still open. It is +# the one commit that carries all four files. +# * Three of the four files are unchanged there from the blobs the openDox root +# pins today (its `contracts/spec-pin.yaml`, 8fe8c4c7). Their digests are the +# ones the root's `contracts/manifest.yaml` records for them. +# * The fourth is T053's `opendox-snapshot`. Its digest is the one T053's root +# step records. +# +# When T053 lands and the root's spec pin moves to its commit, `commit` moves +# here in lockstep. A digest moves only when its file changes. +# +# NEVER EDIT A COPY OR A DIGEST IN PLACE. A copy changes in the spec leg. It +# arrives here when the spec leg's file is copied at the pinned commit and its +# digest recorded, in one commit. +schema_version: 1 +kind: packaged-contract-copies +spec_leg: opensoft/openDox-spec +commit: "cd49eb253a431202b67de70b2c9ed941a72d0aa4" +copies: + - id: ideation-workbench + path: contracts/schemas/ideation-workbench.schema.yaml + sha256: "d30438491119c20928fbe4e85088fc33682829eeb6558d87dafce651000faafc" + - id: opendox-snapshot + path: contracts/schemas/opendox-snapshot.schema.yaml + sha256: "f9e3e111af1d4bd4c377c933027d81b582ae2b0a395b66f4e4621992454a584a" + - id: xfactory-workbench-chat-turn + path: contracts/schemas/xfactory-workbench-chat-turn.schema.yaml + sha256: "350bfedc02696e7281a42c0bdc9a25059bf7af14d16d89d9f07018d3e691dc1d" + - id: xfactory-workbench-model-catalog + path: contracts/schemas/xfactory-workbench-model-catalog.schema.yaml + sha256: "e563cc9fc6ede03dfd62537935d0ae0842617d7de46702aee6ad9026aa021635" diff --git a/src/opendox/contracts/schemas/ideation-workbench.schema.yaml b/src/opendox/contracts/schemas/ideation-workbench.schema.yaml new file mode 100644 index 00000000..219c760f --- /dev/null +++ b/src/opendox/contracts/schemas/ideation-workbench.schema.yaml @@ -0,0 +1,227 @@ +$schema: "https://json-schema.org/draft/2020-12/schema" +$id: "ideation-workbench.schema.yaml" +title: "Ideation-workbench user-assembled reference-set manifest" +contract_schema_version: 1 +description: >- + A user-assembled temporary reference set for the ideation-area dashboard + (`add-ideation-dashboard`; promoted spec requirements "Workbench reference + sets" and "Keyword lens set-builder"). A set is seeded from a cluster card, + ad-hoc from the doc list, or from a saved keyword-lens recipe, and supports + bounded actions — scratch NotebookLM notebook creation, on-demand readiness + scoring, scoped doc-health, and draft-organize — each recorded in + `action_history`. Saved sets persist as gitignored schema-versioned + manifests under `ideation/workbench/` in the owning repository; + `ideation-workbench` manifests found in TRACKED repository content are + invalid (spec scenario "A workbench manifest is committed") — this schema + cannot express that rule structurally (a manifest is well-formed regardless + of where it sits), so enforcement is validator-side, checking the manifest's + path against the repo's tracked-file list, not this document's shape. + + Not deterministic: unlike `ideation-dashboard-snapshot` (a generated + projection required to be byte-identical for an unchanged tree), a workbench + manifest is live human session state — `created`/`updated` are ordinary + wall-clock timestamps written by whatever workbench action last touched the + set, with no source-revision anchor and no byte-identity requirement. + + Manual membership overrides are evidence, never a silent set edit: a + `manual-include` member and every `excluded` entry MUST carry a recorded + `reason` (spec scenario "A manual override is recorded"); an override without + one is invalid (enforced below via `allOf` and in the array item shape). + + Forward-compatible / additive: consumers MUST ignore unknown properties. A + later delta may add new action kinds, seed kinds, or recipe fields without a + `schema_version` bump; only a breaking change (a removed or retyped field) + requires one. This is why no object here sets `additionalProperties: false`. +type: object +required: + - schema_version + - kind + - repository + - name + - created + - updated + - members + - seed +properties: + schema_version: {const: 1} + # Normative kind literal from the promoted spec ("Workbench reference sets"); + # hyphenated per that requirement, not the underscored `xfactory_*` in-file + # kind convention (same divergence as the sibling snapshot schema). + kind: {const: ideation-workbench} + repository: + type: string + minLength: 1 + # Owning repository (`ideation/workbench/` is per-repo); mirrors the + # snapshot's `repository` field for cross-reference. + name: + type: string + minLength: 1 + # The set's human name, chosen at creation; not a stable id. + created: + type: string + format: date-time + # Wall-clock creation stamp — session state, not a determinism anchor. + updated: + type: string + format: date-time + # Wall-clock stamp of the most recent workbench action against this set. + members: + type: array + items: {$ref: "#/$defs/member"} + excluded: + type: array + # Recipe-matching docs the human removed from the forming/saved set — each + # entry is negative evidence and REQUIRES a reason (spec scenario "A manual + # override is recorded"); absent/empty means no exclusions recorded. + items: {$ref: "#/$defs/excluded_entry"} + seed: {$ref: "#/$defs/seed"} + recipe: + # D13 keyword-lens query behind an intensional set (spec: "Keyword lens + # set-builder" — "cluster-as-recipe persistence"). Optional for sets seeded + # any other way (a manually built set may later have a recipe saved against + # it), but REQUIRED when `seed.kind` is `recipe` (allOf below) — a + # recipe-seeded set without its stored query would be neither re-runnable + # nor auditable (task 2.7). + $ref: "#/$defs/recipe" + action_history: + type: array + items: {$ref: "#/$defs/action_history_entry"} + notebook: {$ref: "#/$defs/notebook_binding"} +allOf: + # seed.kind = cluster-seeded names the seeding cluster (task 2.2). + - if: + properties: {seed: {properties: {kind: {const: cluster-seeded}}}} + required: [seed] + then: + properties: {seed: {required: [cluster_id]}} + # seed.kind = recipe stores the seeding query: the manifest MUST carry the + # recipe block so the set stays re-runnable and auditable (task 2.7; spec: + # "the workbench manifest stores the query ... so the set re-runs as the + # corpus grows"). Cross-property (seed ↔ recipe), so it lives here at the + # top level, not inside a $def. + - if: + properties: {seed: {properties: {kind: {const: recipe}}}} + required: [seed] + then: + required: [recipe] +$defs: + # --- membership ("Workbench reference sets" / "Keyword lens set-builder") --- + member: + type: object + required: [document, via] + properties: + document: + type: string + minLength: 1 + # Doc reference: the snapshot's `document.id` when the doc already has + # one, else a repo-relative path for docs not yet captured in a + # snapshot. Distinguishes from `via` below, which records HOW the + # member entered this set, not WHICH document it is. + via: + type: string + enum: [recipe-match, manual-include, cluster-seed] + # Per-member entry mechanism — distinct from the set-level `seed.kind` + # below (a `cluster-seed` set can still gain `manual-include` members). + reason: + type: string + minLength: 1 + # REQUIRED when via=manual-include (allOf below); the recorded "why" + # behind a human override — never a silent set edit. + allOf: + - if: {properties: {via: {const: manual-include}}} + then: {required: [reason]} + excluded_entry: + type: object + required: [document, reason] + properties: + document: {type: string, minLength: 1} # same reference convention as member.document + reason: + type: string + minLength: 1 + # ALWAYS required — negative evidence for a recipe-matching doc the + # human removed (spec: overrides are captured evidence, never silent). + # --- seed provenance ("Workbench reference sets") --- + seed: + type: object + required: [kind] + properties: + kind: + type: string + enum: [cluster-seeded, ad-hoc, recipe] + # How the WHOLE set originated: from a cluster card, ad-hoc from the + # doc list, or from a saved keyword-lens recipe ("add as cluster"). + cluster_id: + type: string + minLength: 1 + # REQUIRED when kind=cluster-seeded (allOf above); the seeding + # cluster's id from the dashboard snapshot. + # --- recipe (D13; "Keyword lens set-builder" — cluster-as-recipe persistence) --- + recipe: + type: object + required: [checked] + properties: + checked: + type: array + minItems: 1 + items: {type: string, minLength: 1} + # Checked keywords stratifying the forming set (check-to-stratify); + # at least one — a zero-keyword query is not an intensional definition. + pinned: + type: array + items: {type: string, minLength: 1} + # Pinned (required) keywords (pin-to-require). VALIDATOR RULE (task + # 2.4/2.7 examples), not expressible cleanly here: every pinned + # keyword MUST also appear in `checked` — this schema does not encode + # the subset constraint. + last_run: + type: object + required: [source_revision] + properties: + source_revision: + type: string + minLength: 1 + # Snapshot `generation.source_revision` this recipe was last + # evaluated against — the re-run anchor (spec scenario "A recipe + # re-runs after corpus growth"). + at: {type: string, format: date-time} # wall-clock stamp of that evaluation + new_candidates: + type: array + items: {type: string, minLength: 1} + # Docs newly matching the recipe since last_run, surfaced without + # altering recorded overrides; cleared (emptied) once reviewed. + # --- action history ("Workbench reference sets") --- + action_history_entry: + type: object + required: [action, at] + properties: + action: + type: string + enum: [notebook, readiness, doc-health, draft-organize, compose-possible, derive-possibles, add-as-cluster] + # Bounded workbench actions; `compose-possible`/`derive-possibles` are + # recorded here once their gating deltas land (tasks.md 3.9) — the + # vocabulary is reserved now so this schema does not need a bump then. + # `add-as-cluster` (added additively, codexFactory 002-ideation-dashboard + # T024/polish): the lens "add as cluster" action — saves the + # recipe-seeded workbench set and records this honest action name, + # with the human-seen cross-reference-queue submission itself still + # PENDING the sibling contract (change task 3.5's `pending_review` + # disposition, tracked as codexFactory T028, not yet landed). + at: {type: string, format: date-time} + reference: + type: string + minLength: 1 + # Optional job/artifact ref produced by the action (e.g. a doc-health + # run id, a readiness score ref, the draft-organize skeleton path). + # --- scratch-notebook binding ("Workbench reference sets") --- + notebook_binding: + type: object + required: [alias] + properties: + alias: + type: string + pattern: "^xf-wb-.+$" + # Scratch NotebookLM notebook bound to this set. A derived artifact: + # deleted by the sync orphan sweep when this manifest is deleted + # (spec scenario "A workbench manifest is deleted"), never the + # reverse — removing a source from the notebook only drops it from + # `members` and leaves the corpus document untouched. diff --git a/src/opendox/contracts/schemas/opendox-snapshot.schema.yaml b/src/opendox/contracts/schemas/opendox-snapshot.schema.yaml new file mode 100644 index 00000000..2be5018f --- /dev/null +++ b/src/opendox/contracts/schemas/opendox-snapshot.schema.yaml @@ -0,0 +1,443 @@ +# openDox's own neutral snapshot contract (plan 034, T053). +# +# WHY THIS FILE IS JSON. It is YAML whose body is one JSON object, and that is +# deliberate. The leg's required `validate` check installs pytest and nothing +# else, so `tests/test_opendox_snapshot_contract.py` reads this file with +# Python's built-in `json` module once these comment lines are set aside. +# JSON is YAML, so every YAML loader in the family reads the same object, and +# the same test proves that PyYAML agrees wherever PyYAML is installed. This +# leg's negative chat-turn examples already take this form. +# +# SECTIONS. openDox's views render six stations from five sections, as +# openDox's own STAGE_FIELDS declares them: source reads documents, grouping +# reads clusters, candidate reads possibles, selection reads staged_topics, +# and the submission and completion stations read the changes entries whose +# status is active and archived. Every station section is required, and it +# is an empty list when its station holds nothing. keyword_index is optional: +# without it, a reader derives the keyword rail from the documents' topics. +# +# CLOSED VALUES, all neutral. A document's stage is one of the six station +# role keys, a candidate's state is one of unselected, selected, declined and +# replaced, and a changes entry's status is active or archived. openDox's +# views match them through SNAPSHOT_VALUES, whose defaults T054 moves to +# these values. +# +# DETERMINISTIC, which is the generator's to keep and no schema can check: the +# same tree yields a byte-identical snapshot, so nothing in it records when +# the generator ran. generated_at, when present, is fixed by the source +# revision (its commit date, or a date recorded with it), never read from the +# clock. FORWARD-COMPATIBLE. A reader ignores unknown properties, so no object +# sets additionalProperties false, and an additive field needs no +# schema_version bump. +# +# RULES. Every rule has an identifier. A subschema names, in x-rule, the rule +# its keywords enforce; x-rules lists every rule with its class; and a +# validator's refusal names the rule it broke. A shape rule is enforced by +# this schema's own keywords. A reference rule is a cross-reference that no +# JSON Schema keyword can state, so a validator enforces it. +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "opendox-snapshot.schema.yaml", + "title": "openDox neutral snapshot", + "contract_schema_version": 1, + "description": "openDox's own snapshot: what its neutral generator writes over a plain repository, and what its views read, with no consumer installed (plan 034 T053, on R1Q11 (a) and R1Q12 (a): opensoft/openxFactory issue 656, comment 5850003126). openXdox's governed generator keeps its own contract, openXdox-spec's ideation-dashboard-snapshot, which this schema leaves unchanged; a contributed generator declares which of the two kinds it writes (T052). The file's comment header, the section descriptions and the x-rules catalog say the rest.", + "x-rule": "envelope-keys", + "type": "object", + "required": [ + "schema_version", + "kind", + "repository", + "generation", + "documents", + "clusters", + "possibles", + "staged_topics", + "changes" + ], + "properties": { + "schema_version": {"x-rule": "schema-version-is-1", "const": 1}, + "kind": {"x-rule": "kind-is-opendox-snapshot", "const": "opendox-snapshot"}, + "repository": {"x-rule": "repository-is-text", "type": "string", "minLength": 1}, + "generation": {"$ref": "#/$defs/generation"}, + "documents": { + "description": "The source station, and the product's whole document list: every document the corpus adapter lists, in the station its stage names. Every entry carries its stage, and no reader supplies one: the generator writes source for a document that declares no stage.", + "x-rule": "section-is-a-list", + "type": "array", + "items": {"$ref": "#/$defs/document"} + }, + "clusters": { + "description": "The grouping station: the groups that form around topics documents share, each with one edge per member document.", + "x-rule": "section-is-a-list", + "type": "array", + "items": {"$ref": "#/$defs/group"} + }, + "possibles": { + "description": "The candidate station.", + "x-rule": "section-is-a-list", + "type": "array", + "items": {"$ref": "#/$defs/candidate"} + }, + "staged_topics": { + "description": "The selection station.", + "x-rule": "section-is-a-list", + "type": "array", + "items": {"$ref": "#/$defs/selection"} + }, + "changes": { + "description": "The submission station (status active) and the completion station (status archived), which share this one section.", + "x-rule": "section-is-a-list", + "type": "array", + "items": {"$ref": "#/$defs/submission"} + }, + "keyword_index": { + "description": "Optional. The keyword rail's seed. When present, every topic the documents carry has one entry, and each entry counts the documents that carry its keyword.", + "x-rule": "section-is-a-list", + "type": "array", + "items": {"$ref": "#/$defs/keyword_entry"} + } + }, + "$defs": { + "id": {"x-rule": "id-is-text", "type": "string", "minLength": 1}, + "path": { + "description": "A path relative to the repository root, as the corpus adapter lists it.", + "x-rule": "path-is-repo-relative", + "type": "string", + "pattern": "^(?!/)(?![A-Za-z]:)(?![\\s\\S]*[\\\\\\u0000-\\u001f\\u007f-\\u009f])(?!(?:[\\s\\S]*/)?\\.\\.(?:/|$))[\\s\\S]+$" + }, + "topic": { + "x-rule": "topic-is-trimmed-text", + "type": "string", + "pattern": "^(?![\\s\\ufeff])(?![\\s\\S]*[\\s\\ufeff]$)[^\\u0000-\\u001f\\u007f-\\u009f]+$" + }, + "stage_role": { + "description": "The station a document sits in: one of the six station role keys, in spine order, exactly openDox's display_profile.STAGE_ROLES. The set is closed. A declared value outside it is not a declaration: the generator reads that document as a source and writes stage source, so the value never reaches a snapshot.", + "x-rule": "stage-is-a-station-role", + "enum": ["source", "grouping", "candidate", "selection", "submission", "completion"] + }, + "candidate_state": { + "description": "A candidate's state, in openDox's own words for the candidate station (NEUTRAL_DISPLAY's candidate vocabulary). A candidate is unselected until an act selects, declines or replaces it.", + "x-rule": "candidate-state-is-known", + "enum": ["unselected", "selected", "declined", "replaced"] + }, + "submission_status": { + "description": "The station a changes entry sits in: active for submission, archived for completion, as openDox's STAGE_FIELDS declares them.", + "x-rule": "submission-status-is-known", + "enum": ["active", "archived"] + }, + "generation": { + "description": "The generation stamp. source_revision is the determinism anchor: the tree revision the snapshot projects.", + "x-rule": "generation-anchored", + "type": "object", + "required": ["source_revision"], + "properties": { + "source_revision": {"x-rule": "generation-anchored", "type": "string", "minLength": 1}, + "generated_at": { + "x-rule": "generated-at-is-rfc3339", + "type": "string", + "format": "date-time", + "pattern": "^(?![\\s\\S]*[\\u0000-\\u001f\\u007f-\\u009f])(?!0000)(?:[0-9]{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12][0-9]|3[01])|(?:0[469]|11)-(?:0[1-9]|[12][0-9]|30)|02-(?:0[1-9]|1[0-9]|2[0-8]))|(?:[0-9]{2}(?:0[48]|[2468][048]|[13579][26])|(?:[02468][048]|[13579][26])00)-02-29)[Tt](?:[01][0-9]|2[0-3]):[0-5][0-9]:[0-5][0-9](?:\\.[0-9]+)?(?:[Zz]|[+-](?:[01][0-9]|2[0-3]):[0-5][0-9])$" + }, + "generator_version": {"x-rule": "generation-anchored", "type": "string", "minLength": 1} + } + }, + "document": { + "description": "One document the corpus adapter lists. title and summary are the default adapter's small neutral field set (R1Q13 (a)). Neither is required, and the generator writes null for one the document does not give. topics, which every entry carries, are the topics the generator assigns: the ones the document declares, or the ones its topic rule derives when it declares none, and an empty list when there are none.", + "x-rule": "document-keys", + "type": "object", + "required": ["id", "path", "stage", "topics"], + "properties": { + "id": {"$ref": "#/$defs/id"}, + "path": {"$ref": "#/$defs/path"}, + "stage": {"$ref": "#/$defs/stage_role"}, + "title": { + "x-rule": "title-and-summary-are-text", + "type": ["string", "null"], + "minLength": 1 + }, + "summary": { + "x-rule": "title-and-summary-are-text", + "type": ["string", "null"], + "minLength": 1 + }, + "topics": { + "x-rule": "topics-are-unique", + "type": "array", + "uniqueItems": true, + "items": {"$ref": "#/$defs/topic"} + } + } + }, + "group": { + "description": "One group in the grouping station. document_edges holds one edge per member document, naming the topics that matched; the funnel draws them.", + "x-rule": "group-keys", + "type": "object", + "required": ["id", "name", "topics", "document_edges"], + "properties": { + "id": {"$ref": "#/$defs/id"}, + "name": {"x-rule": "group-keys", "type": "string", "minLength": 1}, + "topics": { + "x-rule": "group-has-a-topic", + "type": "array", + "minItems": 1, + "uniqueItems": true, + "items": {"$ref": "#/$defs/topic"} + }, + "document_edges": { + "x-rule": "group-keys", + "type": "array", + "items": {"$ref": "#/$defs/edge"} + } + } + }, + "edge": { + "x-rule": "edge-keys", + "type": "object", + "required": ["document", "matched_topics"], + "properties": { + "document": {"$ref": "#/$defs/id"}, + "matched_topics": { + "x-rule": "edge-keys", + "type": "array", + "minItems": 1, + "uniqueItems": true, + "items": {"$ref": "#/$defs/topic"} + } + } + }, + "candidate": { + "description": "One candidate in the candidate station. claiming_clusters holds the groups that claim it; pick holds the selection a selected candidate went to.", + "x-rule": "candidate-keys", + "type": "object", + "required": ["id", "title", "state"], + "properties": { + "id": {"$ref": "#/$defs/id"}, + "title": {"x-rule": "candidate-keys", "type": "string", "minLength": 1}, + "claim": {"x-rule": "candidate-keys", "type": "string", "minLength": 1}, + "state": {"$ref": "#/$defs/candidate_state"}, + "claiming_clusters": { + "x-rule": "candidate-keys", + "type": "array", + "uniqueItems": true, + "items": {"$ref": "#/$defs/id"} + }, + "pick": { + "x-rule": "selected-candidate-has-pick", + "type": "object", + "required": ["staging_id"], + "properties": {"staging_id": {"$ref": "#/$defs/id"}} + }, + "reason": {"x-rule": "closed-candidate-has-reason", "type": "string", "minLength": 1} + }, + "allOf": [ + { + "if": {"required": ["state"], "properties": {"state": {"const": "selected"}}}, + "then": {"x-rule": "selected-candidate-has-pick", "required": ["pick"]} + }, + { + "if": {"required": ["state"], "properties": {"state": {"enum": ["declined", "replaced"]}}}, + "then": {"x-rule": "closed-candidate-has-reason", "required": ["reason"]} + } + ] + }, + "selection": { + "description": "One selection in the selection station. files lists what it is made of, and target_change names the changes entry it went on to.", + "x-rule": "selection-keys", + "type": "object", + "required": ["staging_id"], + "properties": { + "staging_id": {"$ref": "#/$defs/id"}, + "files": { + "x-rule": "selection-keys", + "type": "array", + "uniqueItems": true, + "items": {"$ref": "#/$defs/path"} + }, + "target_change": {"$ref": "#/$defs/id"} + } + }, + "submission": { + "description": "One entry of the submission or the completion station. files lists what it is made of, as a selection's files do, and the tile opens onto them.", + "x-rule": "submission-keys", + "type": "object", + "required": ["id", "status"], + "properties": { + "id": {"$ref": "#/$defs/id"}, + "status": {"$ref": "#/$defs/submission_status"}, + "files": { + "x-rule": "submission-keys", + "type": "array", + "uniqueItems": true, + "items": {"$ref": "#/$defs/path"} + } + } + }, + "keyword_entry": { + "description": "declared_doc_count counts the documents whose topics carry the keyword.", + "x-rule": "keyword-entry-keys", + "type": "object", + "required": ["keyword", "declared_doc_count"], + "properties": { + "keyword": {"$ref": "#/$defs/topic"}, + "declared_doc_count": {"x-rule": "keyword-entry-keys", "type": "integer", "minimum": 0} + } + } + }, + "x-rules": [ + { + "id": "envelope-keys", + "class": "shape", + "says": "A snapshot is an object carrying schema_version, kind, repository, generation and the five station sections: documents, clusters, possibles, staged_topics and changes." + }, + {"id": "schema-version-is-1", "class": "shape", "says": "schema_version is 1."}, + { + "id": "kind-is-opendox-snapshot", + "class": "shape", + "says": "kind is opendox-snapshot. The governed generator's ideation-dashboard-snapshot is a different contract, and this schema refuses it." + }, + { + "id": "repository-is-text", + "class": "shape", + "says": "repository, the canonical id of the repository the snapshot projects, is non-empty text." + }, + { + "id": "generation-anchored", + "class": "shape", + "says": "generation is an object carrying source_revision, the revision of the tree the snapshot projects, as non-empty text; generator_version, when present, is non-empty text too." + }, + { + "id": "generated-at-is-rfc3339", + "class": "shape", + "says": "generation.generated_at, when present, is an RFC 3339 date-time on a day the calendar has (the pattern knows each month's length and the leap years), with no control character. It is narrower than RFC 3339 in two places. Its year is never 0000, which Python's datetime cannot hold. Its seconds run from 00 to 59 and are never a leap second's 60, which a git commit date cannot hold and neither Python's datetime nor a browser's Date can read. jsonschema's date-time checker refuses both. It is fixed by the source revision, never read from the clock." + }, + { + "id": "section-is-a-list", + "class": "shape", + "says": "Each section is a list: documents, clusters, possibles, staged_topics, changes and, when present, keyword_index. A station with nothing in it is an empty list." + }, + { + "id": "id-is-text", + "class": "shape", + "says": "Every entry id, and every reference to one, is non-empty text." + }, + { + "id": "document-keys", + "class": "shape", + "says": "A document is an object carrying id, path, stage and topics." + }, + { + "id": "path-is-repo-relative", + "class": "shape", + "says": "A path is relative to the repository root: it does not start with a slash or with a drive letter and a colon (C:/x or C:x, which Windows joins onto a root as a path outside it), holds no backslash and no control character (U+0000 to U+001F, U+007F to U+009F), and has no .. segment." + }, + { + "id": "stage-is-a-station-role", + "class": "shape", + "says": "A document's stage is one of the six station role keys: source, grouping, candidate, selection, submission, completion." + }, + { + "id": "title-and-summary-are-text", + "class": "shape", + "says": "A document's title and summary are each non-empty text, or null." + }, + { + "id": "topics-are-unique", + "class": "shape", + "says": "A document's topics are a list that names each topic once." + }, + { + "id": "topic-is-trimmed-text", + "class": "shape", + "says": "A topic is non-empty text with no leading or trailing whitespace and no control character (U+0000 to U+001F, U+007F to U+009F). Whitespace is what both Python and a browser count as whitespace, U+FEFF included, so both refuse the same topics." + }, + { + "id": "group-keys", + "class": "shape", + "says": "A group (a clusters entry) is an object carrying id, name, topics and document_edges. Its name is non-empty text, and its edges are a list." + }, + { + "id": "group-has-a-topic", + "class": "shape", + "says": "A group's topics name at least one topic, each once: a group forms around topics that its documents share." + }, + { + "id": "edge-keys", + "class": "shape", + "says": "A group's document edge is an object that names the document and at least one matched topic, each once." + }, + { + "id": "candidate-keys", + "class": "shape", + "says": "A candidate (a possibles entry) is an object carrying id, title and state. Its title, and its claim when present, are non-empty text; claiming_clusters, when present, names each group once." + }, + { + "id": "candidate-state-is-known", + "class": "shape", + "says": "A candidate's state is one of unselected, selected, declined and replaced." + }, + { + "id": "selected-candidate-has-pick", + "class": "shape", + "says": "A selected candidate carries pick, an object whose staging_id names the selection it went to." + }, + { + "id": "closed-candidate-has-reason", + "class": "shape", + "says": "A declined or replaced candidate carries reason, non-empty text that says why." + }, + { + "id": "selection-keys", + "class": "shape", + "says": "A selection (a staged_topics entry) is an object carrying staging_id; files, when present, lists repository-relative paths, each once." + }, + { + "id": "submission-keys", + "class": "shape", + "says": "A changes entry, a submission or a completed item, is an object carrying id and status; files, when present, lists repository-relative paths, each once." + }, + { + "id": "submission-status-is-known", + "class": "shape", + "says": "A changes entry's status is active (the submission station) or archived (the completion station)." + }, + { + "id": "keyword-entry-keys", + "class": "shape", + "says": "A keyword_index entry is an object carrying keyword and declared_doc_count, a whole number no less than 0." + }, + { + "id": "ids-are-unique", + "class": "reference", + "says": "Within each section, entry ids are unique: documents, clusters, possibles and changes by id, and staged_topics by staging_id." + }, + { + "id": "edge-names-a-document", + "class": "reference", + "says": "Every group edge names a document in documents." + }, + { + "id": "one-edge-per-document", + "class": "reference", + "says": "Within one group, the edges name each document once: one edge per member document. A document may feed several groups." + }, + { + "id": "candidate-names-a-group", + "class": "reference", + "says": "Every group that a candidate's claiming_clusters names is in clusters." + }, + { + "id": "pick-names-a-selection", + "class": "reference", + "says": "A candidate's pick.staging_id names a selection in staged_topics." + }, + { + "id": "target-names-a-submission", + "class": "reference", + "says": "A selection's target_change names an entry in changes." + }, + { + "id": "keyword-index-matches-topics", + "class": "reference", + "says": "keyword_index, when present, agrees with the documents: every topic a document carries has an entry, no keyword has two, and each entry's declared_doc_count is the number of documents that carry its keyword, 0 for a keyword that none carries." + } + ] +} diff --git a/src/opendox/contracts/schemas/xfactory-workbench-chat-turn.schema.yaml b/src/opendox/contracts/schemas/xfactory-workbench-chat-turn.schema.yaml new file mode 100644 index 00000000..9efaa118 --- /dev/null +++ b/src/opendox/contracts/schemas/xfactory-workbench-chat-turn.schema.yaml @@ -0,0 +1,414 @@ +$schema: "https://json-schema.org/draft/2020-12/schema" +$id: "xfactory-workbench-chat-turn.schema.yaml" +title: "doxBench grounded chat turn (request / success / fixed failure)" +contract_schema_version: 1 +description: >- + The POST /actions/workbench/chat-turn wire family + (add-workbench-integrated-editor-chat task 2.1; consumer contract + codexFactory specs/010-doxbench-editor-chat/contracts/chat-turn.md). One + file, three closed envelopes discriminated by `kind`: the request + (workbench-chat-turn-v2), the validated success + (workbench-chat-turn-v2-success), and the fixed redacted failure + (workbench-chat-turn-v2-failure). Every object + is closed; a failure NEVER carries prompt text, buffer text, provider + payload, credential, endpoint, secret name, or exception detail — only + limit failures carry measured data, and only numeric dimension/values (the + delegated validator enforces that pairing). `working_subject` is free-form + NON-IDENTITY authoring focus (FR-012): a string, never an actor, customer, + patient, tenant, authority, credential, or routing key — identity-shaped + values are refused by type alone, and no identity-bearing sibling field + exists to smuggle one. Content identity is exact-UTF-8 lowercase SHA-256 + over the full text (R4); the validator recomputes request-buffer hashes, so + a mismatched content_hash is refused rather than trusted. The outline plus + at least one loaded document buffer accompany every request (FR-013): both + complete, never truncated, selected, summarized, or omitted. A path that + does not exist yet is `null`, never a fabricated or guessed string + (`buffer_state.path`, the not-yet-created artifact, data-model S4): a + REQUIRED key with a nullable value — absent and null are + different facts, and only null is expressible. + + THE DEPRECATED v1 FAMILY IS REMOVED (contract-v3.0, + retire-doxbench-chat-turn-v1). Until this release the file also carried three + co-resident v1 envelopes — `workbench-chat-turn`, + `workbench-chat-turn-success` and `workbench-chat-turn-failure` — added at + contract-v1.31, DEPRECATED at contract-v1.34 by the widened family above, and + declared removable at a stated target through a top-level + `deprecated_envelopes` block. Thirteen minors and one major passed with the + contract validator warning on every packaged instance, far past the "at least + one full published release" floor the versioning policy sets. The three + `$defs`, their `oneOf` entries and the whole `deprecated_envelopes` block + leave with them: a block declaring the deprecation of kinds this file no + longer defines names nothing. The shared definitions the removed envelopes + reached are RETAINED, measured from the surviving family's own reference + closure rather than assumed. A consumer pinned below contract-v3.0 keeps + validating v1 instances against the bytes its pin names — compatibility flows + from the consumer, and no new release reaches backwards into an old pin. The + migration path is `contracts/CHANGELOG.md` and + `docs/contract-versioning-policy.md` § Deprecations Executed. + + THE ASSEMBLED CONTEXT'S POSTURE (contract-v1.40, add-doxbench-editing-phase-b + task 10.7). `success_v2` gains ONE optional property, `context_packet`, and + the widened family gains nothing else: the posture (`full | reduced`) under + which the turn's bounded context packet was assembled, plus the reduction's + own reason when there is one. It exists so the ratified "with the reduced + posture STATED" clause is readable by a consumer and showable to the human + rather than stated only inside the packet. ADDITIVE: the key is OPTIONAL, its + ABSENCE is not a posture claim (it means a producer older than v1.40), and no + existing shape changes. (That release also recorded a limitation of the then + DEPRECATED v1 success envelope, which was deliberately not widened; the + envelope is gone at contract-v3.0 and the limitation with it.) +oneOf: + - $ref: "#/$defs/request_v2" + - $ref: "#/$defs/success_v2" + - $ref: "#/$defs/failure_v2" +$defs: + content_hash: + type: string + pattern: "^[0-9a-f]{64}$" + confined_path: + # Repository-relative, confined: never absolute, never an escape. The + # delegated validator re-checks `..` traversal segment-wise. + type: string + minLength: 1 + maxLength: 512 + pattern: "^[^/].*$" + scope_key: + type: object + additionalProperties: false + required: [repository, ref, tile_kind, tile_id] + properties: + repository: { type: string, minLength: 1, maxLength: 128 } + ref: { type: string, minLength: 1, maxLength: 256 } + tile_kind: { enum: [cluster, possible, staged] } + tile_id: { type: string, minLength: 1, maxLength: 256 } + buffer_state: + # One COMPLETE authoring buffer (FR-005/FR-013): descriptor + full text + + # both identities. `base_hash` is the loaded base's identity; the + # stale-authority comparisons happen against `content_hash`. + type: object + additionalProperties: false + required: [kind, repository, path, base_ref, base_revision, + base_hash, content_hash, content, dirty] + properties: + kind: { enum: [outline, document] } + repository: { type: string, minLength: 1, maxLength: 128 } + # Nullable: a not-yet-created artifact has no path until its first Save + # (the null-path -> create lifecycle, data-model S4). + path: + oneOf: + - { type: "null" } + - { $ref: "#/$defs/confined_path" } + base_ref: { type: string, minLength: 1, maxLength: 256 } + base_revision: { type: string, minLength: 1, maxLength: 128 } + base_hash: { $ref: "#/$defs/content_hash" } + content_hash: { $ref: "#/$defs/content_hash" } + content: { type: string, maxLength: 1048576 } + dirty: { type: boolean } + transcript_turn: + type: object + additionalProperties: false + required: [role, content] + properties: + role: { enum: [human, assistant] } + content: { type: string, maxLength: 65536 } + turn_id: { type: string, minLength: 1, maxLength: 128 } + typed_proposal: + # A human-reviewable replacement for exactly ONE buffer, bound to the + # content identity the model saw (FR-026..FR-029). Untyped replacement + # content — a proposal with no target — is refused at the schema. + # + # RETAINED AT contract-v3.0 AND NO LONGER REACHED, deliberately and on the + # record (retire-doxbench-chat-turn-v1 task 2.1). The measured reference + # closure of the surviving family does NOT contain this definition: + # `keyed_typed_proposal` below RESTATES the shape with a buffer-key target + # rather than `$ref`-ing this one, so removing the v1 envelopes leaves this + # block unreferenced. The ratified requirement names `typed_proposal` among + # the shared definitions that "do NOT leave with the envelopes", on a + # factual premise the measurement contradicts. Task 2.1 rules that "any + # surprise is a finding, not a licence", so the surprise is RECORDED here + # and the definition STAYS: the realization does not remove bytes the + # ratified text names as retained. Disposing of it is the cut's decision, + # taken against this measurement rather than against the premise. + type: object + additionalProperties: false + required: [target, base_hash, summary, content] + properties: + target: { enum: [outline, document] } + base_hash: { $ref: "#/$defs/content_hash" } + summary: { type: string, minLength: 1, maxLength: 500 } + content: { type: string, maxLength: 1048576 } + # ---- the widened, co-resident family (contract-v1.34) ---- + buffer_key: + # A buffer's KEY in the widened family (design D1): the permanently + # reserved `outline`, the reserved `document` key that holds the ONE + # not-yet-created artifact of the create flow, or a document's own + # repository-relative path. Both reserved spellings already satisfy the + # confined-path shape, so ONE constraint expresses all three and no `oneOf` + # can match a key twice. WHICH keys a given turn actually holds is the + # request's own statement, checked by the delegated validator against the + # buffer set the request supplies — a schema cannot know a loaded set. + $ref: "#/$defs/confined_path" + keyed_observed_hashes: + # The recomputed identity of EVERY buffer the request supplied, keyed by + # buffer key. v1 required exactly the two keys `outline` and `document` + # because the set held exactly those two buffers; the widened record states + # one identity per loaded document, so a reader can ask which text the model + # saw for any of them. `outline` stays REQUIRED: its key is permanently + # reserved and every turn carries it. + type: object + minProperties: 2 + maxProperties: 25 + required: [outline] + propertyNames: { $ref: "#/$defs/buffer_key" } + additionalProperties: { $ref: "#/$defs/content_hash" } + keyed_typed_proposal: + # v1's `typed_proposal` with its two-value `target` enum widened to a BUFFER + # KEY. Every other field, bound and rule is unchanged. A target the request + # did not supply is unroutable and is refused by the delegated validator + # rather than guessed at. + type: object + additionalProperties: false + required: [target, base_hash, summary, content] + properties: + target: { $ref: "#/$defs/buffer_key" } + base_hash: { $ref: "#/$defs/content_hash" } + summary: { type: string, minLength: 1, maxLength: 500 } + content: { type: string, maxLength: 1048576 } + selected_model: + # WHAT THE HUMAN CHOSE, beside `model_id`'s WHAT ANSWERED. The two differ + # exactly when the chosen catalog entry is a ROUTING RULE this capability + # owns (an `auto` entry) rather than a provider model, which is why + # `routing_rule` is stated rather than inferred from the two ids being + # unequal. `data_handling` is the chosen entry's own catalog badge, carried + # so a transcript states the handling posture the human was shown at send + # time rather than the one the catalog happens to declare when it is read + # back. The resolved model is NOT repeated here: it is `model_id`, once. + type: object + additionalProperties: false + required: [requested_model_id, routing_rule, data_handling] + properties: + requested_model_id: { type: string, minLength: 1, maxLength: 128 } + routing_rule: { type: boolean } + data_handling: { type: string, minLength: 1, maxLength: 500 } + context_packet: + # WHAT THE TURN RAN UNDER (contract-v1.40, add-doxbench-editing-phase-b task + # 10.7). The ratified knowledge-service requirement ends "Where the knowledge + # service is unavailable the turn SHALL degrade to a declared reduced packet + # — the selected thread and the loaded buffers, with the reduced posture + # STATED". Until this release the posture was stated only INSIDE the + # assembled context, which no reader of the record and no human on the + # surface could consult: `success_v2` is a CLOSED envelope and had no field + # for it. This is that field. + # + # ITS VOCABULARY IS THE PACKET'S OWN, deliberately, so no third spelling of + # the same fact exists: `posture` and `reduced_reason` are the two members + # of `ContextPacket` this states, with the SAME two-value posture and the + # SAME truth-pairing the packet enforces at construction — a reduced packet + # STATES its reason and a full packet carries none. That pairing is not + # advisory here either: the two conditionals below refuse each half of it. + # + # WHY ONE OBJECT rather than two sibling keys on the envelope, on the + # `selected_model` precedent one $def above: the posture and its reason are + # one fact about one thing, grouping keeps the "present iff" rule local to + # the object that owns it, and it keeps the ENVELOPE's key set identical for + # a full turn and a reduced one — which is the ratified independence claim + # ("MUST NOT ... make the editors unusable") read on the wire, where the two + # turns differ in what the record SAYS and not in the shape it arrives in. + # + # `reduced_reason` is FREE PROSE with the same 500-`maxLength` ceiling every + # other authored string in this family carries (`data_handling`, a proposal + # `summary`, a failure `message`). THE UNIT IS CODE POINTS, which is what + # JSON Schema's `maxLength` counts — so a 500-character CJK reason is + # conformant at roughly 1,500 UTF-8 bytes, and a consumer sizing a buffer + # must size it in bytes rather than in `maxLength`. Producers are expected + # to refuse an over-long reason BEFORE emitting it rather than discovering + # the ceiling at validation; this repository's own producer does, in the one + # place the reason is carried onto the record. It is a statement about the + # ASSEMBLY, not + # about content: it names why the packet is reduced and, in the reasons this + # capability ships, says in as many words that nothing unbounded was + # substituted and no rail was bypassed. The delegated validator scans it for + # public-only violations exactly as it scans a failure's `message`. + type: object + additionalProperties: false + required: [posture] + properties: + posture: { enum: [full, reduced] } + reduced_reason: { type: string, minLength: 1, maxLength: 500 } + allOf: + # A REDUCTION NOBODY CAN READ IS A SILENT DEGRADATION -- the packet's own + # words, and its own refusal, restated on the wire. + - if: + required: [posture] + properties: { posture: { const: reduced } } + then: + required: [reduced_reason] + # AND A FULL PACKET CARRIES NO REASON. A record that declared `full` while + # naming a reduction would state two contradictory facts and let a reader + # pick; the packet refuses that construction, so the record refuses it too. + # This is a SEPARATE conditional and not the inverse of the one above: + # they guard different instances, and each has its own packaged negative. + - if: + required: [posture] + properties: { posture: { const: full } } + then: + not: { required: [reduced_reason] } + provider_retry: + # A MID-TURN RE-MINT AND THE PAID RETRY IT BOUGHT (contract-v1.45, + # add-doxchat-model-intake task 3.6; handed here by + # add-model-provider-broker task 2.4). + # + # Brett ruled on 2026-08-26 that when a minted token expires part-way + # through a turn the dashboard re-mints and retries ONCE, "with the re-mint + # and the paid retry VISIBLY RECORDED in the turn record" — a second paid + # call the human cannot see is exactly the decision that ruling was made to + # avoid. Three records already existed on the server side (the port's + # content-free mint ledger, the console's stderr notice, and the broker's own + # audit trail correlated by `--retry-of`) and the browser can read NONE of + # them, so the fact never reached the person paying for it. This is the field + # that carries it to them. + # + # THE REDACTED FACT AND NOTHING MORE. It states that the turn re-minted once + # and made one further paid provider call, and it names the re-mint's audit + # reference. It carries no token, no token prefix, no token hash, no + # provider status, no provider words, no endpoint and no timing that would + # let one be reconstructed — the same redaction discipline the failure + # envelope keeps, for the same reason: this record is stored, mirrored into a + # thread sidecar and rendered in a page. + # + # `audit_ref` IS DISCLOSABLE BY CONSTRUCTION rather than by care: the + # broker's own declaration records no token material against an audit + # reference, and it is the identifier the broker's trail is keyed by — so a + # reader of this record and a reader of `broker-audit.jsonl` can be shown to + # be reading about the same issuance, which is what makes "visibly recorded" + # mean something on both sides of the seam instead of only on the server's. + # + # `retried: true` is a CONST rather than a boolean anyone may set false. A + # turn that did not retry omits the whole object; a `retried: false` would be + # a second, weaker spelling of an absence that is already unambiguous, and + # two spellings of one fact is how a consumer comes to read the wrong one. + # + # WHAT `at_most_once` RECORDS: that the ruling's bound HELD. A second expiry + # inside one turn does not buy a third call — it surfaces the standard + # refusal, and that turn produces a FAILURE envelope rather than this one — + # so a success record carrying this object is a record of exactly one retry. + # Stated on the record rather than left to a reader who would have to know + # the port's internals to infer it. + type: object + additionalProperties: false + required: [retried, at_most_once] + properties: + retried: { const: true } + at_most_once: { const: true } + audit_ref: { type: string, minLength: 1, maxLength: 200 } + request_v2: + # The widened request. It carries the outline plus every loaded document, + # and it DECLARES which of them the chat is bound to. `active_document_path` + # is deliberately absent: the binding is stated, never inferred from an + # adjacent field that answers a different question (design D17), and the two + # questions had different answers exactly when a human worked the outline + # with a document loaded. + type: object + additionalProperties: false + required: [schema_version, kind, client_turn_id, scope, bound_buffer, + working_subject, message, model_id, last_assistant_turn_id, + transcript, buffers] + properties: + schema_version: { const: 1 } + kind: { const: workbench-chat-turn-v2 } + client_turn_id: { type: string, minLength: 1, maxLength: 128 } + scope: { $ref: "#/$defs/scope_key" } + # The DECLARED binding: what the chat is working ON. It MUST name one of + # the buffers this same request supplies (delegated validator), and it + # never narrows what the turn is GROUNDED on — every loaded buffer rides + # the request regardless of which one is bound. + bound_buffer: { $ref: "#/$defs/buffer_key" } + working_subject: { type: string, maxLength: 512 } + message: { type: string, minLength: 1, maxLength: 65536 } + model_id: { type: string, minLength: 1, maxLength: 128 } + last_assistant_turn_id: + oneOf: [ { type: "null" }, { type: string, maxLength: 128 } ] + transcript: + type: array + maxItems: 128 + items: { $ref: "#/$defs/transcript_turn" } + buffers: + # The outline plus one to twenty-four document buffers. The lower bound + # is unchanged from v1 (one outline and at least one document buffer, + # which the delegated validator pairs as before); the upper bound is the + # surface's own declared loaded-set bound plus the reserved outline, so + # a request can never carry more buffers than a human is allowed to + # load. Reaching the bound REFUSES the load with the measured bound + # stated; nothing is evicted, because every loaded buffer may hold + # unsaved human text. + type: array + minItems: 2 + maxItems: 25 + items: { $ref: "#/$defs/buffer_state" } + success_v2: + # The widened, validated success — the turn's durable RECORD. It names the + # bound buffer (from the request's DECLARED binding, never derived), states + # every buffer's observed identity by key, and carries the selected-model + # metadata beside the model that answered. + type: object + additionalProperties: false + required: [schema_version, kind, client_turn_id, assistant_turn_id, + model_id, selected_model, bound_buffer, observed_hashes, + assistant_prose, proposals] + properties: + schema_version: { const: 1 } + kind: { const: workbench-chat-turn-v2-success } + client_turn_id: { type: string, minLength: 1, maxLength: 128 } + assistant_turn_id: { type: string, minLength: 1, maxLength: 128 } + # The model that ANSWERED. With a routing-rule entry this is the model the + # rule resolved to, which is what makes a transcript state which model + # produced which turn instead of leaving it to be inferred. + model_id: { type: string, minLength: 1, maxLength: 128 } + selected_model: { $ref: "#/$defs/selected_model" } + # WHAT THE TURN RAN UNDER (contract-v1.40, task 10.7). OPTIONAL, which is + # what makes this release additive: a record produced before v1.40 omits + # it and stays valid, and no reader of an older record has to change. + # OMISSION IS NOT A POSTURE CLAIM -- it means the producer predates this + # release, never that the context was full. A consumer that needs the + # posture must read this key and treat its absence as unknown. + context_packet: { $ref: "#/$defs/context_packet" } + # WHAT THE TURN COST (contract-v1.45, add-doxchat-model-intake task 3.6). + # OPTIONAL, which is what makes this release additive, and PRESENT ONLY + # WHEN IT HAPPENED: a turn that minted once and answered once carries no + # such key, and its absence is not a claim — it means "nothing to report, + # or a producer older than v1.45". Present on the v2 envelope ONLY: the v1 + # family is deprecated and its promise is byte-identical stability. + provider_retry: { $ref: "#/$defs/provider_retry" } + bound_buffer: { $ref: "#/$defs/buffer_key" } + observed_hashes: { $ref: "#/$defs/keyed_observed_hashes" } + assistant_prose: { type: string, maxLength: 900000 } + # At most one proposal per supplied buffer (the delegated validator holds + # that rule, which is expressed over the REQUEST's own buffer count); the + # transport ceiling here is the largest buffer set a request may carry. + proposals: + type: array + maxItems: 25 + items: { $ref: "#/$defs/keyed_typed_proposal" } + failure_v2: + # The fixed redacted failure for a v2 turn. Structurally identical to the v1 + # failure and deliberately so — a refusal discloses nothing whichever family + # it answers — but it carries its own `kind`, so the whole v1 family can be + # deprecated as a unit and a v2 turn is never answered in a deprecated + # envelope. + type: object + additionalProperties: false + required: [schema_version, kind, client_turn_id, error, message] + properties: + schema_version: { const: 1 } + kind: { const: workbench-chat-turn-v2-failure } + client_turn_id: { type: string, minLength: 1, maxLength: 128 } + error: { type: string, pattern: "^[a-z][a-z0-9_]{2,63}$" } + message: { type: string, minLength: 1, maxLength: 500 } + limit: + type: object + additionalProperties: false + required: [dimension, measured, maximum] + properties: + dimension: { type: string, minLength: 1, maxLength: 64 } + measured: { type: integer, minimum: 0 } + maximum: { type: integer, minimum: 0 } diff --git a/src/opendox/contracts/schemas/xfactory-workbench-model-catalog.schema.yaml b/src/opendox/contracts/schemas/xfactory-workbench-model-catalog.schema.yaml new file mode 100644 index 00000000..e4399c59 --- /dev/null +++ b/src/opendox/contracts/schemas/xfactory-workbench-model-catalog.schema.yaml @@ -0,0 +1,230 @@ +$schema: "https://json-schema.org/draft/2020-12/schema" +$id: "xfactory-workbench-model-catalog.schema.yaml" +title: "doxBench approved model catalog (wire envelope)" +contract_schema_version: 1 +description: >- + The GET /workbench/model-catalog success envelope + (add-workbench-integrated-editor-chat task 2.1; consumer contract + codexFactory specs/010-doxbench-editor-chat/contracts/model-catalog.md). + PUBLIC-ONLY BY CONSTRUCTION: every object is closed + (additionalProperties: false), so credentials, raw endpoints, secret or + environment-variable names, request templates, provider-native options, and + deployment resource names are structurally impossible rather than merely + discouraged; the delegated validator additionally scans string values for + credential/endpoint spellings. An empty `models` array is a SUCCESSFUL + editor-only posture (FR-025/SC-008), never an error. Model ids are opaque + UI handles, not provider model names; selecting one grants no lasting + authority (the id is revalidated on every turn). Instance kinds use the + retained `workbench-*` identifier family (compatibility-migration checklist + NOTES); the file name carries the manifest's `xfactory-` artifact prefix. + Since contract-v1.38 an entry MAY additionally declare itself a ROUTING + RULE (`routing_rule`/`routes_to`/`resolved_model_id`, all optional and all + three travelling together) — see `$defs/model_entry`. That growth changes + nothing about the credential posture: a routing entry names only opaque + catalog ids, and an API-backed entry's credential is provisioned into the + `doxbench-bridge` harness profile by the ratified broker lane + (`add-model-provider-broker`). The adapter neither holds nor fetches a raw + secret — its child environment is an allowlist no credential-shaped + variable can pass — and a self-hosted, keyless provider needs none. No + field here has ever carried a credential and none may be added that could. +type: object +additionalProperties: false +required: [schema_version, kind, models] +properties: + schema_version: { const: 1 } + kind: { const: workbench-model-catalog } + models: + type: array + maxItems: 64 + items: { $ref: "#/$defs/model_entry" } +$defs: + model_entry: + # The public allowlist, exactly: the SEVEN REQUIRED base fields + # (data-model.md Section 6) plus, since contract-v1.38, the THREE OPTIONAL + # routing-declaration fields below, and since contract-v2.2 the ONE + # OPTIONAL `modalities` capability declaration. An entry that declares + # none of the four optional fields is byte-for-byte the entry this schema + # has always accepted, which is what makes each growth additive. + type: object + additionalProperties: false + required: [model_id, label, provider_class, available, + input_limit_bytes, output_limit_bytes, data_handling] + properties: + model_id: + type: string + minLength: 1 + maxLength: 128 + pattern: "^[A-Za-z0-9][A-Za-z0-9._-]*$" + label: { type: string, minLength: 1, maxLength: 200 } + provider_class: { type: string, minLength: 1, maxLength: 64 } + available: { type: boolean } + # A catalog entry may only NARROW the server ceilings (plan.md + # Constraints): request bound 1,048,576 bytes; output bound 900,000. + # + # For an AVAILABLE ROUTING entry these must additionally not exceed the + # limits of `resolved_model_id`'s entry — the model that ANSWERS — because + # the effective turn limit is computed from the SELECTED entry, which for + # a routed turn is the rule. The bound is the RESOLVED model's alone and + # deliberately NOT the minimum over `routes_to`: an un-resolved + # destination takes no turn while the rule resolves elsewhere. Delegated + # to the validator, which the shape cannot express. + input_limit_bytes: { type: integer, minimum: 1, maximum: 1048576 } + output_limit_bytes: { type: integer, minimum: 1, maximum: 900000 } + # For a ROUTING entry this text is a SEPARATOR-JOINED LIST of badge + # segments, and it MUST carry the badge of every model the rule may route + # to as one of them. THE SEPARATOR IS SPACE SLASH SPACE (" / "). + # + # The covering rule is SEGMENT MEMBERSHIP, not substring containment: for + # each id in `routes_to`, that entry's own `data_handling` must equal one + # segment of this one, compared with whitespace collapsed, case folded and + # trailing `.;,` dropped — and with interior characters never rewritten, + # so `on-tenant` and `non-tenant` stay different. Extra segments are + # permitted: a rule may carry its own lead-in beside the badges it must + # carry. A ROUTED entry whose own badge contains the separator is refused, + # because such a badge could never be one segment. The delegated validator + # (`scripts/validate-ideation-dashboard-contracts.py`) enforces all of + # this; the shape can express none of it. + # + # The 500-byte ceiling is unchanged and therefore bounds how many distinct + # badges one rule can carry — a rule whose list does not fit is a rule + # that must be split, or whose members' badges must be written more + # tightly. + data_handling: { type: string, minLength: 1, maxLength: 500 } + # --- the input-modality declaration (contract-v2.2, additive) --- + # + # OPTIONAL, and it says WHAT KIND of input the model accepts — the one + # thing `input_limit_bytes` and `output_limit_bytes` cannot say, because + # they describe how MUCH a model accepts and never what kind. It exists + # so a routing decision can ask whether a candidate can carry what a turn + # actually contains, rather than inferring capability from a model's name + # or from `provider_class`, which is a governance classification. + # + # ABSENCE IS NOT A CLAIM IN EITHER DIRECTION. An entry that declares + # nothing is a producer that predates this field, exactly as an absent + # `context_posture` means "a producer older than contract-v1.40" in the + # chat-turn family rather than a stated posture. A reader treats an + # undeclared entry as text-only FOR ROUTING — the safe reading — while + # recording that no declaration was made, so a conservative default is + # never mistaken for a stated capability. Requiring the field would have + # invalidated every catalog released before it, which an additive growth + # must not do. + # + # THE VOCABULARY IS CLOSED AND EXTENDS ONLY BY THE CHANGE THAT GOVERNS A + # NEW MEMBER — the rule `admission_surface` states for the client-identity + # roster. `image` enters because a turn carrying an image is the named + # near-term consumer. Audio, video, tool-calling, structured output, + # latency class and cost class do NOT enter: nothing consumes them, and a + # vocabulary guessed ahead of its consumers is one nothing validates + # against. INPUT ACCEPTANCE ONLY — output modality and tool/structured + # output are different questions with different consumers, and one set + # answering several would mean different things to different readers. + # + # `contains: {const: text}` is load-bearing and is NOT redundant with + # `minItems`: without it this shape would accept `modalities: [image]`, + # which the catalog TYPE refuses, leaving the WIRE GATE THE WEAKEST ONE — + # the very divergence class this release closes in the other direction. + # A chat turn always carries text, so a model that cannot accept text is + # not routable here at all. + # + # THE TYPE IS THE ONLY OTHER GATE, and that is worth stating exactly. The + # delegated validator does NOT restate the modality rules — they are all + # expressible here, so it enforces them BY APPLYING THESE BYTES. This + # clause is therefore the whole of the file-side refusal, not a second + # opinion beside one: remove it and the packaged image-only negative + # stops being refused at all. + # + # No `maxItems`: `uniqueItems` over a two-member closed enum already + # bounds the array at two, and a `maxItems` literal would be a second + # place to edit when the vocabulary grows by a governing change. + modalities: + type: array + minItems: 1 + uniqueItems: true + items: + enum: [text, image] + contains: + const: text + description: >- + The input modalities this model accepts, from the closed vocabulary + `text` and `image`. Optional; absence means the producer predates + the field and is read as text-only for routing while being recorded + as no declaration. Where present the set is non-empty, carries no + repeats, and must contain `text`. + # --- the routing-rule declaration (contract-v1.38, additive) --- + # + # OPTIONAL, and all three travel together (see dependentRequired and the + # two conditionals below). Present-and-true means this entry is a ROUTING + # RULE this capability owns — an `auto` entry that maps a turn to a model + # by role — rather than a directly-answering provider model. ABSENT means + # exactly what absence has always meant: a plain model, with no new + # obligation of any kind. + # + # They carry NO provider surface. `routes_to` and `resolved_model_id` are + # opaque catalog handles drawn from this same catalog's `model_id` values, + # so a routing declaration can name nothing a plain entry could not + # already name. An API-backed entry's credential comes from the ratified + # broker lane (`add-model-provider-broker`), which provisions the + # `doxbench-bridge` profile; the adapter holds and fetches no secret, and + # a keyless self-hosted provider needs none. + routing_rule: + type: boolean + description: >- + True IFF this entry is a routing rule rather than a directly + answering model. A routing entry must declare `routes_to` and + `resolved_model_id`; a plain entry must declare neither. + routes_to: + type: array + minItems: 1 + maxItems: 64 + uniqueItems: true + description: >- + Every model this rule MAY route to, as `model_id` references into + this same catalog. The delegated validator resolves each one and + refuses a dangling reference, a self-reference, a target that is + itself a routing rule, a target whose own `data_handling` this + entry's `data_handling` does not carry as a " / "-separated SEGMENT, + and a target whose badge holds that separator itself. + items: + type: string + minLength: 1 + maxLength: 128 + pattern: "^[A-Za-z0-9][A-Za-z0-9._-]*$" + resolved_model_id: + type: string + minLength: 1 + maxLength: 128 + pattern: "^[A-Za-z0-9][A-Za-z0-9._-]*$" + description: >- + The model this rule CURRENTLY resolves to — the one that ANSWERS the + turn, is recorded as the turn's `model_id` beside the requested id in + `selected_model`, and is the model the turn's thread sidecar names. + Must be a member of `routes_to` — so the badge covering above has + necessarily checked it — must be available whenever the routing entry + itself is available, and must accept at least the limits the routing + entry declares. All three are delegated-validator rules: the shape has + no operator for any of them. + dependentRequired: + # Neither routing field may ride on an entry that does not declare what + # it is, and neither is meaningful without the other. + routes_to: [routing_rule, resolved_model_id] + resolved_model_id: [routing_rule, routes_to] + allOf: + # BOTH conditionals constrain ONLY the three fields this release adds: + # each `if` requires `routing_rule` to be PRESENT, so an entry that + # declares no routing rule matches neither and is judged exactly as it + # was before contract-v1.38. + - if: + required: [routing_rule] + properties: + routing_rule: { const: true } + then: + required: [routes_to, resolved_model_id] + - if: + required: [routing_rule] + properties: + routing_rule: { const: false } + then: + not: + anyOf: + - required: [routes_to] + - required: [resolved_model_id] diff --git a/src/opendox/validator.py b/src/opendox/validator.py new file mode 100644 index 00000000..a20c6d2d --- /dev/null +++ b/src/opendox/validator.py @@ -0,0 +1,808 @@ +"""OPENDOX'S OWN VALIDATOR: its own document kinds, read from its own packaged +copies of their schemas. + +WHY THIS FILE EXISTS. #1144's requirement 7 asks for *"a validator openDox can +run"*, and its 7.1 says to *"NARROW THE INPUT SET FIRST, then acquire what +remains"*: openDox's validator validates openDox's OWN document kinds, whose +schemas its own spec leg owns. T007's batch G amends 7.1 on R1Q11 (a) and +R1Q12 (a) (`openxFactory#656` comment `5850003126`), so the set is FOUR +schemas: 7.1's three, `ideation-workbench`, `xfactory-workbench-chat-turn` and +`xfactory-workbench-model-catalog`, and the neutral snapshot contract, +`opendox-snapshot`, which T053 adds to openDox-spec. This module is plan 034's +T057. `opendox.contracts` beside it carries the four as package data, each one +digest-checked before it is read. + +THE INPUT SET, AND WHAT IS LEFT OUT (7.1, 7.1b). `KIND_ENTRIES` maps each +instance kind openDox validates to the copy that holds its schema. Nothing else +is in it. + +* openXdox-spec's three, `ideation-dashboard-snapshot`, its `-index` and + `gate-action-record`, *"belong to the CONSUMER's validator and openDox never + needs them"* (7.1). The consumer locates them itself (7.3, plan 034's T061). +* openxFactory's four are not in it either, and 7.1b names two of them as + unavailable by any route. `gate-intent` is an intent-plane schema, which + requirement 1 keeps with openxFactory. `ideation-possibles-register` is + filed `stays_openxfactory_adapter`, as openxFactory's own candidate + register. Vendoring either would meet requirement 7 by breaching + requirement 1. No openDox verb needs one, so neither is carried. + +WHY THE CONSUMER'S SCRIPT IS NOT REUSED (7.1a). The validator the carve left, +`scripts/validate-ideation-dashboard-contracts.py`, lives in openXdox-code. It +was run at openXdox-code `4610bca5` for this record. + +1. It finds its schemas from its own position. `ROOT` is + `Path(__file__).resolve().parents[1]`, and it reads `ROOT / "contracts"`, + or the directory `CONTRACTS_DIR` names. #1144 measured the first half + (*"run from openXdox-code it exits 2 with `ERROR .../contracts/schemas not + found`"*), and C3's openXdox-code#28 has added the second since. Run at + `4610bca5` with no `CONTRACTS_DIR`, it still exits 2: `ERROR harness + failure: .../contracts/schemas carries none of the family's 10 schemas`. + An installed openDox carries neither the script nor a `contracts/` beside + it, and no environment variable is part of `pip install`, which is the + checkout requirement 7 means (7.1). That is 7.1a's defect, and it is the + consumer's to fix for its own set (T061). +2. It names TEN schemas, of three owners (`SCHEMA_FILENAMES`). openDox's + validator reads four, all openDox-spec's (7.1). +3. It is the CONSUMER's file. openDox pins nothing of openXdox's, and a reach + into it is the direction requirements 2 and 5 close. +4. It needs `jsonschema`, `referencing` and `rfc3339-validator`. openDox-code + declares PyYAML alone, and this module needs no more. +5. It knows no `opendox-snapshot`. Given openDox-spec's contracts through + `CONTRACTS_DIR`, it refuses openDox-spec's own six-station example with + exit 1, `ERROR [kind] ...: unrecognized document (no known kind ...)`, + because its `KIND_TO_SCHEMA` has no entry for the neutral kind. +6. It is a script, run as a subprocess over a file. openDox-code has no + `scripts/` directory (7.2), and this validator is a library call that + answers violations. + +NEW SURFACE (7.2). This file is CREATED at the code leg, not relocated, and it +has no row in openxFactory's `docs/opendox-carve-manifest.yaml`, because the +manifest declares what LEAVES openxFactory and never what a destination +assembles (RULED OQ-C). It adds no verb, and #1144 adds none. Validation is the +post-render step inside the generate verbs, which plan 034's T058 routes here. + +THE EVALUATOR. It evaluates JSON Schema draft 2020-12, and exactly the keywords +the four copies use (`KEYWORDS`), as that draft defines them. JSON equality +holds throughout: `true` is not `1`, `1` is `1.0`, and key order does not +count. `format` is ASSERTED, as the consumer's validator asserts it with its +format checker, and `date-time` is the one format the four use. A copy that +uses a keyword, a format, a dialect or a reference this module does not +evaluate is REFUSED when its validator is built (`SchemaNotEvaluable`). It is +never evaluated with that keyword left out, which would pass whatever the +keyword refuses. + +RULE IDENTIFIERS. Every violation names the rule it broke. The neutral snapshot +contract gives each rule an id, carried as `x-rule` by the subschema that +enforces it, and that id is the violation's `rule`. A subschema that names none +(the three older contracts do not use the convention) is identified by the JSON +Schema keyword that failed. `Violation.line()` renders +`[] : `, with `where` a JSON pointer into the instance, so +a refusal carries the identifier a caller can match. T051's `EXPECTED_RULE` is +one such identifier, and F7.2 greps the verbs' stderr for it. + +THE NEUTRAL SNAPSHOT'S REFERENCE RULES. Seven of the contract's 32 rules are +cross-references that no JSON Schema keyword can state, so this module +implements them (`REFERENCE_RULES`). The contract catalogues each rule with its +class in `x-rules`. A snapshot validator is REFUSED when that catalog names a +reference rule this module does not implement, or when this module implements +one the catalog does not name. So a validator never enforces less than the +contract says, or more. + +IDENTITY BEFORE USE, ON EVERY CALL. `validator_for()` has +`opendox.contracts.verified_bytes()` prove each copy before its bytes are +parsed. A compiled validator is cached under the digest that proof returned, +never under a name or a time. So a changed byte is refused on the very call +that sees it, as openxFactory's `doxbench_contracts.validators()` refuses one. + +jsonschema's SHAPE, FOR THE doxBench SEAM. `KindValidator.iter_errors()` +yields violations whose `validator` (the failed keyword) and `absolute_path` +read as a `jsonschema` error's do. Those are the two fields +`serve_workbench`'s readers of the doxBench-validators seam read. +`validators()` answers one validator per wire kind: the model catalog's whole +document, and each chat-turn envelope's own `$defs` entry, as openxFactory's +`doxbench_contracts` builds them. So T085 can register it +(`serve_wire.register_doxbench_validators`). The doxBench kinds' semantic +rules are T085's. Here they are validated structurally, as openxFactory's +`validators()` validates them. + +IMPORT WEIGHT. The standard library, and `opendox.contracts`, whose record and +copies are read with PyYAML only when a validator is built. It names no +sibling. +""" + +from __future__ import annotations + +import calendar +import hashlib +import re +import threading +from dataclasses import dataclass +from typing import Any, Callable, Iterable, Iterator, Mapping + +from opendox import contracts + +__all__ = [ + "DIALECT", + "FORMATS", + "KEYWORDS", + "KINDS", + "KIND_ENTRIES", + "KindValidator", + "REFERENCE_RULES", + "SchemaNotEvaluable", + "UnknownKind", + "ValidatorUnavailable", + "Violation", + "report", + "validate", + "validator_for", + "validators", +] + +#: The one dialect the four copies declare, and the one this module evaluates. +DIALECT = "https://json-schema.org/draft/2020-12/schema" + +#: THE INPUT SET (7.1, as batch G amends it): each instance kind openDox +#: validates, and where its schema is, as (packaged copy id, JSON pointer into +#: that copy). "" is the copy's whole document. The chat-turn copy holds three +#: envelopes, and each wire kind is validated against its own envelope, as +#: openxFactory's `doxbench_contracts.CHAT_TURN_DEFS` names them. +KIND_ENTRIES: Mapping[str, tuple[str, str]] = { + "ideation-workbench": ("ideation-workbench", ""), + "opendox-snapshot": ("opendox-snapshot", ""), + "workbench-chat-turn-v2": ("xfactory-workbench-chat-turn", "/$defs/request_v2"), + "workbench-chat-turn-v2-failure": ("xfactory-workbench-chat-turn", + "/$defs/failure_v2"), + "workbench-chat-turn-v2-success": ("xfactory-workbench-chat-turn", + "/$defs/success_v2"), + "workbench-model-catalog": ("xfactory-workbench-model-catalog", ""), +} + +#: The instance kinds, sorted. +KINDS: tuple[str, ...] = tuple(sorted(KIND_ENTRIES)) + +#: The keywords that can FAIL, and that this module evaluates. +_ASSERTING = frozenset({ + "const", "dependentRequired", "enum", "format", "maxItems", "maxLength", + "maxProperties", "maximum", "minItems", "minLength", "minProperties", + "minimum", "pattern", "required", "type", "uniqueItems"}) +#: The keywords that route a value into subschemas. +_APPLYING = frozenset({ + "$ref", "additionalProperties", "allOf", "anyOf", "contains", "if", "items", + "not", "oneOf", "properties", "propertyNames", "then"}) +#: The keywords that only carry or annotate. +_ANNOTATING = frozenset({ + "$defs", "$id", "$schema", "contract_schema_version", "description", + "title", "x-rule", "x-rules"}) +#: Every keyword this module evaluates or knows to carry nothing. A copy that +#: uses any other is refused (`SchemaNotEvaluable`). +KEYWORDS = _ASSERTING | _APPLYING | _ANNOTATING + +_TYPES = frozenset({"array", "boolean", "integer", "null", "number", "object", + "string"}) + +#: How much of a value a violation's detail quotes. +_BRIEF = 80 + + +class ValidatorUnavailable(RuntimeError): + """openDox's validator cannot run for a kind. + + Either a packaged copy failed its identity check (`opendox.contracts` + refused it), or it uses something this module does not evaluate. No + verdict was reached, and no verdict is never a pass: a caller reports the + validator as unavailable, and `--strict` makes that fatal (T058).""" + + +class SchemaNotEvaluable(ValidatorUnavailable): + """A packaged copy uses a keyword, format, dialect or reference this + module does not evaluate, or declares reference rules it does not + implement. It is refused when its validator is built.""" + + +class UnknownKind(ValueError): + """The instance's kind is not one of openDox's own kinds (7.1). + + Raised instead of answering an empty list, which would read as valid. An + unknown kind has no schema here, so it has no verdict.""" + + +@dataclass(frozen=True) +class Violation: + """One broken rule, at one place in the instance.""" + + rule: str # the broken rule's identifier + path: tuple[str | int, ...] # where, as the instance's own keys and indices + keyword: str # the schema keyword that failed, or "reference" + detail: str # what was found + + @property + def where(self) -> str: + """`path` as a JSON pointer ("" is the instance itself).""" + return "".join("/" + str(part).replace("~", "~0").replace("/", "~1") + for part in self.path) + + # jsonschema's names for the same fields, for the doxBench seam's readers. + @property + def validator(self) -> str: + return self.keyword + + @property + def absolute_path(self) -> tuple[str | int, ...]: + return self.path + + @property + def message(self) -> str: + return self.detail + + def line(self) -> str: + """`[] : `, the form a refusal prints.""" + return f"[{self.rule}] {self.where or ''}: {self.detail}" + + +def report(violations: Iterable[Violation]) -> list[str]: + """Each violation's `line()`, in the order they were found.""" + return [violation.line() for violation in violations] + + +# --------------------------------------------------------------------------- +# JSON values +# --------------------------------------------------------------------------- + +def _brief(value: Any) -> str: + text = repr(value) + return text if len(text) <= _BRIEF else text[:_BRIEF - 3] + "..." + + +def _is_type(value: Any, name: str) -> bool: + if name == "object": + return isinstance(value, dict) + if name == "array": + return isinstance(value, list) + if name == "string": + return isinstance(value, str) + if name == "null": + return value is None + if name == "boolean": + return isinstance(value, bool) + if isinstance(value, bool): # JSON true is not the number 1 + return False + if name == "integer": + return isinstance(value, int) or (isinstance(value, float) and value.is_integer()) + return isinstance(value, (int, float)) # "number"; _TYPES was checked at build + + +def _is_number(value: Any) -> bool: + return isinstance(value, (int, float)) and not isinstance(value, bool) + + +def _canon(value: Any) -> Any: + """JSON equality: `true` is not `1`, `1` is `1.0`, and key order is noise.""" + if isinstance(value, bool): + return ("boolean", value) + if isinstance(value, (int, float)): + return ("number", value) + if isinstance(value, str): + return ("string", value) + if value is None: + return ("null",) + if isinstance(value, list): + return ("array", tuple(_canon(v) for v in value)) + if isinstance(value, dict): + # A key is text in JSON. A YAML instance can carry another scalar as a + # key, so each key is tagged, and a mixed mapping still sorts. + return ("object", tuple(sorted( + ((("text", k) if isinstance(k, str) else ("other", repr(k))), _canon(v)) + for k, v in value.items()))) + return ("other", repr(value)) # never equal to a JSON value + + +_DATE_TIME = re.compile( + r"([0-9]{4})-(0[1-9]|1[0-2])-([0-9]{2})T(?:[01][0-9]|2[0-3]):[0-5][0-9]:[0-5][0-9]" + r"(?:\.[0-9]+)?(?:Z|[+-](?:[01][0-9]|2[0-3]):[0-5][0-9])") + + +def _is_date_time(value: str) -> bool: + """An RFC 3339 date-time, as the consumer's validator checks one. + + That is `jsonschema`'s `date-time` checker, which is `rfc3339-validator` + over the upper-cased value: a year other than 0000, a day the month has, + and no leap second. It differs in one place, deliberately: the whole value + must match. `rfc3339-validator` anchors its pattern with `$`, which also + matches before a final newline, so it admits `...Z` followed by a + newline, and this does not.""" + match = _DATE_TIME.fullmatch(value.upper()) + if match is None: + return False + year, month, day = (int(group) for group in match.groups()) + return year != 0 and 1 <= day <= calendar.monthrange(year, month)[1] + + +#: The formats this module asserts. A copy naming another is refused. +FORMATS: Mapping[str, Callable[[str], bool]] = {"date-time": _is_date_time} + + +# --------------------------------------------------------------------------- +# the neutral snapshot's reference rules +# --------------------------------------------------------------------------- + +def _entries(snap: Any, section: str) -> list[tuple[int, dict[str, Any]]]: + items = snap.get(section) if isinstance(snap, dict) else None + if not isinstance(items, list): + return [] # absent or not a list is a shape rule's to report + return [(i, entry) for i, entry in enumerate(items) if isinstance(entry, dict)] + + +def _ids(snap: Any, section: str, key: str = "id") -> set[str]: + return {entry[key] for _i, entry in _entries(snap, section) + if isinstance(entry.get(key), str)} + + +def _broken(rule: str, path: tuple[str | int, ...], detail: str) -> Violation: + return Violation(rule, path, "reference", detail) + + +def _ids_are_unique(snap: Any) -> Iterator[Violation]: + for section, key in (("documents", "id"), ("clusters", "id"), ("possibles", "id"), + ("staged_topics", "staging_id"), ("changes", "id")): + seen: set[str] = set() + for i, entry in _entries(snap, section): + value = entry.get(key) + if isinstance(value, str): + if value in seen: + yield _broken("ids-are-unique", (section, i, key), + f"{_brief(value)} is already an id in {section}") + seen.add(value) + + +def _edges(group: dict[str, Any]) -> list[tuple[int, Any]]: + edges = group.get("document_edges") + return list(enumerate(edges if isinstance(edges, list) else [])) + + +def _edge_names_a_document(snap: Any) -> Iterator[Violation]: + known = _ids(snap, "documents") + for gi, group in _entries(snap, "clusters"): + for ei, edge in _edges(group): + ref = edge.get("document") if isinstance(edge, dict) else None + if isinstance(ref, str) and ref not in known: + yield _broken("edge-names-a-document", + ("clusters", gi, "document_edges", ei, "document"), + f"no document has the id {_brief(ref)}") + + +def _one_edge_per_document(snap: Any) -> Iterator[Violation]: + for gi, group in _entries(snap, "clusters"): + seen: set[str] = set() + for ei, edge in _edges(group): + ref = edge.get("document") if isinstance(edge, dict) else None + if isinstance(ref, str): + if ref in seen: + yield _broken("one-edge-per-document", + ("clusters", gi, "document_edges", ei, "document"), + f"{_brief(ref)} already has an edge in this group") + seen.add(ref) + + +def _candidate_names_a_group(snap: Any) -> Iterator[Violation]: + known = _ids(snap, "clusters") + for pi, candidate in _entries(snap, "possibles"): + refs = candidate.get("claiming_clusters") + for ri, ref in enumerate(refs if isinstance(refs, list) else []): + if isinstance(ref, str) and ref not in known: + yield _broken("candidate-names-a-group", + ("possibles", pi, "claiming_clusters", ri), + f"no group has the id {_brief(ref)}") + + +def _pick_names_a_selection(snap: Any) -> Iterator[Violation]: + known = _ids(snap, "staged_topics", "staging_id") + for pi, candidate in _entries(snap, "possibles"): + pick = candidate.get("pick") + ref = pick.get("staging_id") if isinstance(pick, dict) else None + if isinstance(ref, str) and ref not in known: + yield _broken("pick-names-a-selection", ("possibles", pi, "pick", "staging_id"), + f"no selection has the staging_id {_brief(ref)}") + + +def _target_names_a_submission(snap: Any) -> Iterator[Violation]: + known = _ids(snap, "changes") + for ti, selection in _entries(snap, "staged_topics"): + ref = selection.get("target_change") + if isinstance(ref, str) and ref not in known: + yield _broken("target-names-a-submission", ("staged_topics", ti, "target_change"), + f"no changes entry has the id {_brief(ref)}") + + +def _keyword_index_matches_topics(snap: Any) -> Iterator[Violation]: + """Present, the index never contradicts the documents: every topic they + carry has one entry, and each entry counts the documents that carry its + keyword. An entry for a keyword no document carries is lawful at 0.""" + index = snap.get("keyword_index") if isinstance(snap, dict) else None + if not isinstance(index, list): + return # absent is lawful; not a list is section-is-a-list's + carried: dict[str, int] = {} + for _i, document in _entries(snap, "documents"): + topics = document.get("topics") + for topic in ({t for t in topics if isinstance(t, str)} + if isinstance(topics, list) else ()): + carried[topic] = carried.get(topic, 0) + 1 + rule = "keyword-index-matches-topics" + listed: set[str] = set() + for ki, entry in enumerate(index): + keyword = entry.get("keyword") if isinstance(entry, dict) else None + if not isinstance(keyword, str): + continue # keyword-entry-keys and topic-is-trimmed-text say why + if keyword in listed: + yield _broken(rule, ("keyword_index", ki, "keyword"), + f"{_brief(keyword)} already has an entry") + continue + listed.add(keyword) + count = entry.get("declared_doc_count") + if _is_type(count, "integer") and count != carried.get(keyword, 0): + yield _broken(rule, ("keyword_index", ki, "declared_doc_count"), + f"{count} documents, but {carried.get(keyword, 0)} carry " + f"{_brief(keyword)}") + unlisted = sorted(set(carried) - listed) + if unlisted: + yield _broken(rule, ("keyword_index",), + f"topics the documents carry and the index does not list: " + f"{_brief(unlisted)}") + + +#: The reference rules this module implements, per packaged copy. Only the +#: neutral snapshot contract declares any. +REFERENCE_RULES: Mapping[str, Mapping[str, Callable[[Any], Iterator[Violation]]]] = { + "opendox-snapshot": { + "ids-are-unique": _ids_are_unique, + "edge-names-a-document": _edge_names_a_document, + "one-edge-per-document": _one_edge_per_document, + "candidate-names-a-group": _candidate_names_a_group, + "pick-names-a-selection": _pick_names_a_selection, + "target-names-a-submission": _target_names_a_submission, + "keyword-index-matches-topics": _keyword_index_matches_topics, + }, +} + + +# --------------------------------------------------------------------------- +# a copy, compiled +# --------------------------------------------------------------------------- + +def _unescape(token: str) -> str: + return token.replace("~1", "/").replace("~0", "~") + + +def _at_pointer(document: Any, pointer: str) -> Any: + """The node `pointer` (a JSON pointer, "" for the root) names in `document`.""" + node = document + if pointer == "": + return node + if not pointer.startswith("/"): + raise KeyError(pointer) + for token in pointer[1:].split("/"): + token = _unescape(token) + if isinstance(node, dict): + node = node[token] + elif isinstance(node, list): + node = node[int(token)] + else: + raise KeyError(pointer) + return node + + +class KindValidator: + """The validator of one kind, built over one proved copy. + + `iter_errors(instance)` yields every `Violation`: the schema's, in the + order the schema is walked, and then the reference rules', in the + contract's catalog order. `violations()` lists them and `is_valid()` asks + whether there are none.""" + + def __init__(self, kind: str, copy_id: str, pointer: str, document: Any, + digest: str) -> None: + self.kind = kind + self.copy_id = copy_id + self.pointer = pointer + self.digest = digest + self._document = document + self._patterns: dict[str, re.Pattern[str]] = {} + self._refuse_what_is_not_evaluated(document) + try: + self._entry = _at_pointer(document, pointer) + except (KeyError, IndexError, ValueError) as exc: + raise self._not_evaluable(f"it has no {pointer!r} for {kind}") from exc + self._reference = self._reference_rules(document) + + # -- building ----------------------------------------------------------- + + def _not_evaluable(self, detail: str) -> SchemaNotEvaluable: + return SchemaNotEvaluable( + f"openDox's validator cannot evaluate its packaged copy of " + f"{self.copy_id} for {self.kind}: {detail}") + + def _refuse_what_is_not_evaluated(self, document: Any) -> None: + if not isinstance(document, dict): + raise self._not_evaluable(f"it is a {type(document).__name__}, not a schema") + if document.get("$schema") != DIALECT: + raise self._not_evaluable( + f"its dialect is {document.get('$schema')!r}, not {DIALECT!r}") + for at, node in _subschemas(document): + unknown = sorted(set(node) - KEYWORDS) + if unknown: + raise self._not_evaluable(f"{at or ''} uses {unknown}, which " + "this module does not evaluate") + if "type" in node: + names = node["type"] if isinstance(node["type"], list) else [node["type"]] + if not names or not set(names) <= _TYPES: + raise self._not_evaluable(f"{at} names the type(s) {names}") + if "format" in node and node["format"] not in FORMATS: + raise self._not_evaluable( + f"{at} names the format {node['format']!r}, which this module " + f"does not assert (it asserts {sorted(FORMATS)})") + if "pattern" in node: + try: + self._patterns[node["pattern"]] = re.compile(node["pattern"]) + except (re.error, TypeError) as exc: + raise self._not_evaluable( + f"{at}'s pattern does not compile ({exc})") from exc + if "$ref" in node: + ref = node["$ref"] + if not isinstance(ref, str) or not (ref == "#" or ref.startswith("#/")): + raise self._not_evaluable( + f"{at} refers to {ref!r}; only a reference inside the copy " + "is evaluated") + try: + target = _at_pointer(document, ref[1:]) + except (KeyError, IndexError, ValueError) as exc: + raise self._not_evaluable(f"{at}'s reference {ref!r} names " + "nothing in the copy") from exc + if not isinstance(target, (dict, bool)): + raise self._not_evaluable(f"{at}'s reference {ref!r} names a " + f"{type(target).__name__}, not a schema") + + def _reference_rules(self, document: dict[str, Any] + ) -> tuple[Callable[[Any], Iterator[Violation]], ...]: + implemented = REFERENCE_RULES.get(self.copy_id, {}) + catalog = document.get("x-rules", []) + if not isinstance(catalog, list) or not all( + isinstance(rule, dict) and isinstance(rule.get("id"), str) + for rule in catalog): + raise self._not_evaluable("its x-rules catalog is not a list of rules " + "that each carry an id") + declared = [rule["id"] for rule in catalog if rule.get("class") == "reference"] + if sorted(declared) != sorted(implemented): + raise self._not_evaluable( + f"its catalog declares the reference rules {sorted(declared)}, and " + f"this module implements {sorted(implemented)}; a reference rule " + "is enforced only when both name it") + return tuple(implemented[rule] for rule in declared) + + # -- evaluating --------------------------------------------------------- + + def iter_errors(self, instance: Any) -> Iterator[Violation]: + yield from self._evaluate(instance, self._entry, ()) + for check in self._reference: + yield from check(instance) + + def violations(self, instance: Any) -> list[Violation]: + return list(self.iter_errors(instance)) + + def is_valid(self, instance: Any) -> bool: + return next(self.iter_errors(instance), None) is None + + def _valid(self, value: Any, schema: Any, path: tuple[str | int, ...]) -> bool: + return next(self._evaluate(value, schema, path), None) is None + + def _pattern(self, pattern: str) -> re.Pattern[str]: + """The compiled pattern. Every pattern in the copy compiled when this + validator was built, so one is compiled here only if it is reached + through a reference the build's walk did not pass.""" + compiled = self._patterns.get(pattern) + if compiled is None: + try: + compiled = self._patterns[pattern] = re.compile(pattern) + except re.error as exc: + raise self._not_evaluable(f"a pattern does not compile ({exc})") from exc + return compiled + + def _evaluate(self, value: Any, schema: Any, + path: tuple[str | int, ...]) -> Iterator[Violation]: + if schema is True: + return + if schema is False: + yield Violation("false", path, "false", "no value is valid here") + return + named = schema.get("x-rule") + + def broken(keyword: str, detail: str) -> Violation: + return Violation(named or keyword, path, keyword, detail) + + if "$ref" in schema: + # Draft 2020-12: a reference applies BESIDE its sibling keywords. + yield from self._evaluate(value, _at_pointer(self._document, schema["$ref"][1:]), + path) + if "type" in schema: + names = schema["type"] if isinstance(schema["type"], list) else [schema["type"]] + if not any(_is_type(value, name) for name in names): + yield broken("type", f"{_brief(value)} is not of type {' or '.join(names)}") + if "const" in schema and _canon(value) != _canon(schema["const"]): + yield broken("const", f"{_brief(value)} is not {_brief(schema['const'])}") + if "enum" in schema and _canon(value) not in {_canon(v) for v in schema["enum"]}: + yield broken("enum", f"{_brief(value)} is not one of {_brief(schema['enum'])}") + if isinstance(value, str): + yield from self._string(value, schema, broken) + if _is_number(value): + if "minimum" in schema and value < schema["minimum"]: + yield broken("minimum", f"{_brief(value)} is less than {schema['minimum']}") + if "maximum" in schema and value > schema["maximum"]: + yield broken("maximum", f"{_brief(value)} is more than {schema['maximum']}") + if isinstance(value, list): + yield from self._array(value, schema, path, broken) + if isinstance(value, dict): + yield from self._object(value, schema, path, broken) + for sub in schema.get("allOf", ()): + yield from self._evaluate(value, sub, path) + if "anyOf" in schema: + if not any(self._valid(value, sub, path) for sub in schema["anyOf"]): + yield broken("anyOf", f"{_brief(value)} is valid under none of the " + f"{len(schema['anyOf'])} subschemas") + if "oneOf" in schema: + valid = [i for i, sub in enumerate(schema["oneOf"]) if self._valid(value, sub, path)] + if len(valid) != 1: + yield broken("oneOf", f"{_brief(value)} is valid under " + + ("none" if not valid else f"subschemas {valid}") + + f" of the {len(schema['oneOf'])}, not exactly one") + if "not" in schema and self._valid(value, schema["not"], path): + yield broken("not", f"{_brief(value)} is valid under the subschema `not` refuses") + # `if` is a test, never a failure: it only chooses whether `then` applies. + if "if" in schema and "then" in schema and self._valid(value, schema["if"], path): + yield from self._evaluate(value, schema["then"], path) + + def _string(self, value: str, schema: dict[str, Any], + broken: Callable[[str, str], Violation]) -> Iterator[Violation]: + if len(value) < schema.get("minLength", 0): + yield broken("minLength", f"{_brief(value)} is shorter than {schema['minLength']}") + if "maxLength" in schema and len(value) > schema["maxLength"]: + yield broken("maxLength", f"{_brief(value)} is longer than {schema['maxLength']}") + if "pattern" in schema and not self._pattern(schema["pattern"]).search(value): + yield broken("pattern", f"{_brief(value)} does not match the rule's pattern") + if "format" in schema and not FORMATS[schema["format"]](value): + yield broken("format", f"{_brief(value)} is not a {schema['format']}") + + def _array(self, value: list[Any], schema: dict[str, Any], + path: tuple[str | int, ...], + broken: Callable[[str, str], Violation]) -> Iterator[Violation]: + if len(value) < schema.get("minItems", 0): + yield broken("minItems", f"{len(value)} items, fewer than {schema['minItems']}") + if "maxItems" in schema and len(value) > schema["maxItems"]: + yield broken("maxItems", f"{len(value)} items, more than {schema['maxItems']}") + if schema.get("uniqueItems") and len({_canon(v) for v in value}) != len(value): + yield broken("uniqueItems", f"{_brief(value)} repeats an item") + if "items" in schema: + for i, item in enumerate(value): + yield from self._evaluate(item, schema["items"], path + (i,)) + if "contains" in schema and not any( + self._valid(item, schema["contains"], path + (i,)) + for i, item in enumerate(value)): + yield broken("contains", "no item is valid under the subschema `contains` asks for") + + def _object(self, value: dict[str, Any], schema: dict[str, Any], + path: tuple[str | int, ...], + broken: Callable[[str, str], Violation]) -> Iterator[Violation]: + for key in schema.get("required", ()): + if key not in value: + yield broken("required", f"{key!r} is required") + if len(value) < schema.get("minProperties", 0): + yield broken("minProperties", + f"{len(value)} properties, fewer than {schema['minProperties']}") + if "maxProperties" in schema and len(value) > schema["maxProperties"]: + yield broken("maxProperties", + f"{len(value)} properties, more than {schema['maxProperties']}") + for key, needed in schema.get("dependentRequired", {}).items(): + if key in value: + for other in needed: + if other not in value: + yield broken("dependentRequired", + f"{other!r} is required beside {key!r}") + properties = schema.get("properties", {}) + for key, sub in properties.items(): + if key in value: + yield from self._evaluate(value[key], sub, path + (key,)) + if "additionalProperties" in schema: + extra = [key for key in value if key not in properties] + extra_schema = schema["additionalProperties"] + if extra_schema is False: + if extra: + yield broken("additionalProperties", + f"unexpected properties {_brief(sorted(extra))}") + else: + for key in extra: + yield from self._evaluate(value[key], extra_schema, path + (key,)) + if "propertyNames" in schema: + # jsonschema does not extend the path for a name: the object is where. + for key in value: + for found in self._evaluate(key, schema["propertyNames"], path): + yield Violation(found.rule, path, found.keyword, + f"the property name {_brief(key)}: {found.detail}") + + +def _subschemas(node: Any, at: str = "") -> Iterator[tuple[str, dict[str, Any]]]: + """Every subschema of `node` that is an object, with its location.""" + if not isinstance(node, dict): + return + yield at, node + for key in ("properties", "$defs"): + for name, sub in node.get(key, {}).items() if isinstance(node.get(key), dict) else (): + yield from _subschemas(sub, f"{at}/{key}/{name}") + for key in ("additionalProperties", "items", "contains", "propertyNames", "not", + "if", "then"): + if key in node: + yield from _subschemas(node[key], f"{at}/{key}") + for key in ("allOf", "anyOf", "oneOf"): + for i, sub in enumerate(node.get(key, ())): + yield from _subschemas(sub, f"{at}/{key}/{i}") + + +# --------------------------------------------------------------------------- +# the validators, over copies proved on every call +# --------------------------------------------------------------------------- + +_CACHE: dict[tuple[str, str], KindValidator] = {} +_CACHE_LOCK = threading.Lock() + + +def validator_for(kind: str) -> KindValidator: + """The validator of `kind`, over its packaged copy, proved on this call. + + Raises `UnknownKind` for a kind outside the input set, and + `ValidatorUnavailable` when the copy fails its identity check or uses + something this module does not evaluate.""" + if kind not in KIND_ENTRIES: + raise UnknownKind( + f"{kind!r} is not one of openDox's own kinds; openDox's validator " + f"validates {list(KINDS)} (#1144 7.1)") + copy_id, pointer = KIND_ENTRIES[kind] + try: + data = contracts.verified_bytes(copy_id) + except contracts.CopyRefused as exc: + raise ValidatorUnavailable(str(exc)) from exc + key = (kind, hashlib.sha256(data).hexdigest()) + with _CACHE_LOCK: + cached = _CACHE.get(key) + if cached is not None: + return cached + import yaml + + try: + document = yaml.safe_load(data) + except yaml.YAMLError as exc: + raise ValidatorUnavailable( + f"the packaged copy of {copy_id} matches its digest but is not YAML " + f"({exc.__class__.__name__})") from exc + built = KindValidator(kind, copy_id, pointer, document, key[1]) + with _CACHE_LOCK: + return _CACHE.setdefault(key, built) + + +def validators() -> dict[str, KindValidator]: + """One validator per kind, every copy proved on this call. A fresh dict, + so a caller changing its copy changes no other caller's.""" + return {kind: validator_for(kind) for kind in KINDS} + + +def validate(instance: Any, *, kind: str | None = None) -> list[Violation]: + """Every rule of `kind`'s contract that `instance` breaks. + + `kind` names the contract the instance must meet, as a generator declares + the contract it writes (`generator_seam.SnapshotGenerator.contract`). Given + one, the instance's own `kind` field is checked by that contract. Without + one, the instance's own `kind` field picks the contract, and a kind that is + not openDox's raises `UnknownKind`.""" + if kind is None: + kind = instance.get("kind") if isinstance(instance, dict) else None + if not isinstance(kind, str) or kind not in KIND_ENTRIES: + raise UnknownKind( + f"the instance's kind is {_brief(kind)}, which is not one of " + f"openDox's own kinds {list(KINDS)} (#1144 7.1)") + return validator_for(kind).violations(instance) diff --git a/tests/fixtures/spec-examples/ideation-workbench-adhoc-human-seen.example.yaml b/tests/fixtures/spec-examples/ideation-workbench-adhoc-human-seen.example.yaml new file mode 100644 index 00000000..87e1d6c5 --- /dev/null +++ b/tests/fixtures/spec-examples/ideation-workbench-adhoc-human-seen.example.yaml @@ -0,0 +1,29 @@ +# VALID ideation-workbench manifest — the named case "ad-hoc human-seen cluster +# submission" (add-ideation-dashboard task 2.4). A human hand-picked docs from the +# doc list (seed.kind=ad-hoc), then ran draft-organize to package the set as a +# human-seen cluster submission for organizer/cataloger review (tasks 3.5/3.10). +# - every manual-include member carries a recorded reason (human override = evidence); +# - no recipe block: an ad-hoc set need not be intensional (recipe only required +# when seed.kind=recipe); +# - action_history records the draft-organize that produced the submission packet. +schema_version: 1 +kind: ideation-workbench +repository: openxFactory +name: Human-seen cluster - gate console lineage +created: "2026-07-14T10:05:00Z" +updated: "2026-07-14T10:30:00Z" +members: + - document: doc-gate-console + via: manual-include + reason: Reader noticed the gate-console docs share a lineage the clusterer had not linked. + - document: doc-next-step-kickoff + via: manual-include + reason: Kickoff dispatch is the downstream half of the same human-seen theme. +seed: + kind: ad-hoc +action_history: + - action: draft-organize + at: "2026-07-14T10:30:00Z" + reference: staging/gate-console-lineage/PENDING-REVIEW-skeleton +notebook: + alias: xf-wb-gate-console-lineage diff --git a/tests/fixtures/spec-examples/ideation-workbench-cluster-seeded.example.yaml b/tests/fixtures/spec-examples/ideation-workbench-cluster-seeded.example.yaml new file mode 100644 index 00000000..7d3e9f85 --- /dev/null +++ b/tests/fixtures/spec-examples/ideation-workbench-cluster-seeded.example.yaml @@ -0,0 +1,41 @@ +# VALID ideation-workbench manifest, cluster-seeded with a saved recipe. +# - seed.kind=cluster-seeded carries cluster_id (schema allOf); +# - a manual-include member carries its recorded reason (never a silent edit); +# - an excluded entry carries its reason (negative evidence); +# - recipe.pinned is a SUBSET of recipe.checked (validator rule W1); +# - recipe.new_candidates is DISJOINT from members and excluded (validator rule W2); +# - the notebook alias matches xf-wb-*. +# This file lives under examples/ (static reference material), so the +# committed-manifest guard deliberately excludes it — a real saved manifest +# would live under gitignored ideation/workbench/. +schema_version: 1 +kind: ideation-workbench +repository: openxFactory +name: Dashboard readiness sweep +created: "2026-07-14T08:15:00Z" +updated: "2026-07-14T09:40:00Z" +members: + - {document: doc-keyword-lens, via: cluster-seed} + - {document: doc-cluster-canvas, via: cluster-seed} + - document: doc-ideation-dashboard-brainstorm + via: manual-include + reason: The originating brainstorm belongs in the readiness sweep even though it predates the cluster. +excluded: + - document: doc-lifecycle-notebook-projection + reason: Notebook-projection doc matched the recipe but is out of the dashboard readiness scope. +seed: + kind: cluster-seeded + cluster_id: cl-ideation-dashboard +recipe: + checked: [ideation-dashboard, keyword-lens, cluster-canvas] + pinned: [ideation-dashboard] + last_run: + source_revision: a1b2c3d4e5f6a7b8c9d0e1f2a3b4c5d6e7f8a9b0 + at: "2026-07-14T09:40:00Z" + new_candidates: [doc-workflow-visualization] +action_history: + - {action: readiness, at: "2026-07-14T09:00:00Z", reference: readiness/xf-wb-run-0007} + - {action: doc-health, at: "2026-07-14T09:20:00Z", reference: health/scoped/xf-wb-0007} + - {action: draft-organize, at: "2026-07-14T09:40:00Z", reference: staging/dashboard-readiness/DRAFT-skeleton} +notebook: + alias: xf-wb-dashboard-readiness diff --git a/tests/fixtures/spec-examples/negative/opendox-snapshot-candidate-keys.negative.yaml b/tests/fixtures/spec-examples/negative/opendox-snapshot-candidate-keys.negative.yaml new file mode 100644 index 00000000..ca2d3fc3 --- /dev/null +++ b/tests/fixtures/spec-examples/negative/opendox-snapshot-candidate-keys.negative.yaml @@ -0,0 +1,50 @@ +# expected_failure: candidate-keys +# INVALID: a candidate carries no `title`. +# The no-front-matter example with this one change, so it breaks this +# shape rule and no other. +{ + "schema_version": 1, + "kind": "opendox-snapshot", + "repository": "plain-notes", + "generation": {"source_revision": "0a1b2c3d4e5f60718293a4b5c6d7e8f901234567"}, + "documents": [ + { + "id": "kitchen", + "path": "kitchen.md", + "stage": "source", + "title": null, + "summary": null, + "topics": ["bread", "oven"] + }, + { + "id": "bakery-visit", + "path": "bakery-visit.md", + "stage": "source", + "title": null, + "summary": null, + "topics": ["bread", "flour"] + }, + { + "id": "garden", + "path": "garden.md", + "stage": "source", + "title": null, + "summary": null, + "topics": ["herbs"] + } + ], + "clusters": [ + { + "id": "bread", + "name": "bread", + "topics": ["bread"], + "document_edges": [ + {"document": "kitchen", "matched_topics": ["bread"]}, + {"document": "bakery-visit", "matched_topics": ["bread"]} + ] + } + ], + "possibles": [{"id": "loaf-club", "state": "unselected"}], + "staged_topics": [], + "changes": [] +} diff --git a/tests/fixtures/spec-examples/negative/opendox-snapshot-candidate-names-a-group.negative.yaml b/tests/fixtures/spec-examples/negative/opendox-snapshot-candidate-names-a-group.negative.yaml new file mode 100644 index 00000000..8f8b0543 --- /dev/null +++ b/tests/fixtures/spec-examples/negative/opendox-snapshot-candidate-names-a-group.negative.yaml @@ -0,0 +1,57 @@ +# expected_failure: candidate-names-a-group +# INVALID: a candidate is claimed by a group the snapshot does not hold. +# The no-front-matter example with this one change, so it breaks this +# reference rule and no other. +{ + "schema_version": 1, + "kind": "opendox-snapshot", + "repository": "plain-notes", + "generation": {"source_revision": "0a1b2c3d4e5f60718293a4b5c6d7e8f901234567"}, + "documents": [ + { + "id": "kitchen", + "path": "kitchen.md", + "stage": "source", + "title": null, + "summary": null, + "topics": ["bread", "oven"] + }, + { + "id": "bakery-visit", + "path": "bakery-visit.md", + "stage": "source", + "title": null, + "summary": null, + "topics": ["bread", "flour"] + }, + { + "id": "garden", + "path": "garden.md", + "stage": "source", + "title": null, + "summary": null, + "topics": ["herbs"] + } + ], + "clusters": [ + { + "id": "bread", + "name": "bread", + "topics": ["bread"], + "document_edges": [ + {"document": "kitchen", "matched_topics": ["bread"]}, + {"document": "bakery-visit", "matched_topics": ["bread"]} + ] + } + ], + "possibles": [ + { + "id": "loaf-club", + "title": "A weekly loaf club", + "state": "unselected", + "claiming_clusters": ["no-such-group"] + } + ], + "staged_topics": [], + "changes": [] +} diff --git a/tests/fixtures/spec-examples/negative/opendox-snapshot-candidate-state-is-known.negative.yaml b/tests/fixtures/spec-examples/negative/opendox-snapshot-candidate-state-is-known.negative.yaml new file mode 100644 index 00000000..79ec8eb8 --- /dev/null +++ b/tests/fixtures/spec-examples/negative/opendox-snapshot-candidate-state-is-known.negative.yaml @@ -0,0 +1,50 @@ +# expected_failure: candidate-state-is-known +# INVALID: a candidate's state is not one of the four. +# The no-front-matter example with this one change, so it breaks this +# shape rule and no other. +{ + "schema_version": 1, + "kind": "opendox-snapshot", + "repository": "plain-notes", + "generation": {"source_revision": "0a1b2c3d4e5f60718293a4b5c6d7e8f901234567"}, + "documents": [ + { + "id": "kitchen", + "path": "kitchen.md", + "stage": "source", + "title": null, + "summary": null, + "topics": ["bread", "oven"] + }, + { + "id": "bakery-visit", + "path": "bakery-visit.md", + "stage": "source", + "title": null, + "summary": null, + "topics": ["bread", "flour"] + }, + { + "id": "garden", + "path": "garden.md", + "stage": "source", + "title": null, + "summary": null, + "topics": ["herbs"] + } + ], + "clusters": [ + { + "id": "bread", + "name": "bread", + "topics": ["bread"], + "document_edges": [ + {"document": "kitchen", "matched_topics": ["bread"]}, + {"document": "bakery-visit", "matched_topics": ["bread"]} + ] + } + ], + "possibles": [{"id": "loaf-club", "title": "A weekly loaf club", "state": "maybe"}], + "staged_topics": [], + "changes": [] +} diff --git a/tests/fixtures/spec-examples/negative/opendox-snapshot-closed-candidate-has-reason.negative.yaml b/tests/fixtures/spec-examples/negative/opendox-snapshot-closed-candidate-has-reason.negative.yaml new file mode 100644 index 00000000..489e6c63 --- /dev/null +++ b/tests/fixtures/spec-examples/negative/opendox-snapshot-closed-candidate-has-reason.negative.yaml @@ -0,0 +1,50 @@ +# expected_failure: closed-candidate-has-reason +# INVALID: a declined candidate carries no `reason`. +# The no-front-matter example with this one change, so it breaks this +# shape rule and no other. +{ + "schema_version": 1, + "kind": "opendox-snapshot", + "repository": "plain-notes", + "generation": {"source_revision": "0a1b2c3d4e5f60718293a4b5c6d7e8f901234567"}, + "documents": [ + { + "id": "kitchen", + "path": "kitchen.md", + "stage": "source", + "title": null, + "summary": null, + "topics": ["bread", "oven"] + }, + { + "id": "bakery-visit", + "path": "bakery-visit.md", + "stage": "source", + "title": null, + "summary": null, + "topics": ["bread", "flour"] + }, + { + "id": "garden", + "path": "garden.md", + "stage": "source", + "title": null, + "summary": null, + "topics": ["herbs"] + } + ], + "clusters": [ + { + "id": "bread", + "name": "bread", + "topics": ["bread"], + "document_edges": [ + {"document": "kitchen", "matched_topics": ["bread"]}, + {"document": "bakery-visit", "matched_topics": ["bread"]} + ] + } + ], + "possibles": [{"id": "loaf-club", "title": "A weekly loaf club", "state": "declined"}], + "staged_topics": [], + "changes": [] +} diff --git a/tests/fixtures/spec-examples/negative/opendox-snapshot-document-keys.negative.yaml b/tests/fixtures/spec-examples/negative/opendox-snapshot-document-keys.negative.yaml new file mode 100644 index 00000000..f05eb926 --- /dev/null +++ b/tests/fixtures/spec-examples/negative/opendox-snapshot-document-keys.negative.yaml @@ -0,0 +1,43 @@ +# expected_failure: document-keys +# INVALID: a document carries no `stage`. +# The no-front-matter example with this one change, so it breaks this +# shape rule and no other. +{ + "schema_version": 1, + "kind": "opendox-snapshot", + "repository": "plain-notes", + "generation": {"source_revision": "0a1b2c3d4e5f60718293a4b5c6d7e8f901234567"}, + "documents": [ + { + "id": "kitchen", + "path": "kitchen.md", + "stage": "source", + "title": null, + "summary": null, + "topics": ["bread", "oven"] + }, + { + "id": "bakery-visit", + "path": "bakery-visit.md", + "stage": "source", + "title": null, + "summary": null, + "topics": ["bread", "flour"] + }, + {"id": "garden", "path": "garden.md", "title": null, "summary": null, "topics": ["herbs"]} + ], + "clusters": [ + { + "id": "bread", + "name": "bread", + "topics": ["bread"], + "document_edges": [ + {"document": "kitchen", "matched_topics": ["bread"]}, + {"document": "bakery-visit", "matched_topics": ["bread"]} + ] + } + ], + "possibles": [], + "staged_topics": [], + "changes": [] +} diff --git a/tests/fixtures/spec-examples/negative/opendox-snapshot-edge-keys.negative.yaml b/tests/fixtures/spec-examples/negative/opendox-snapshot-edge-keys.negative.yaml new file mode 100644 index 00000000..c43d8146 --- /dev/null +++ b/tests/fixtures/spec-examples/negative/opendox-snapshot-edge-keys.negative.yaml @@ -0,0 +1,50 @@ +# expected_failure: edge-keys +# INVALID: a group edge carries no `matched_topics`. +# The no-front-matter example with this one change, so it breaks this +# shape rule and no other. +{ + "schema_version": 1, + "kind": "opendox-snapshot", + "repository": "plain-notes", + "generation": {"source_revision": "0a1b2c3d4e5f60718293a4b5c6d7e8f901234567"}, + "documents": [ + { + "id": "kitchen", + "path": "kitchen.md", + "stage": "source", + "title": null, + "summary": null, + "topics": ["bread", "oven"] + }, + { + "id": "bakery-visit", + "path": "bakery-visit.md", + "stage": "source", + "title": null, + "summary": null, + "topics": ["bread", "flour"] + }, + { + "id": "garden", + "path": "garden.md", + "stage": "source", + "title": null, + "summary": null, + "topics": ["herbs"] + } + ], + "clusters": [ + { + "id": "bread", + "name": "bread", + "topics": ["bread"], + "document_edges": [ + {"document": "kitchen"}, + {"document": "bakery-visit", "matched_topics": ["bread"]} + ] + } + ], + "possibles": [], + "staged_topics": [], + "changes": [] +} diff --git a/tests/fixtures/spec-examples/negative/opendox-snapshot-edge-names-a-document.negative.yaml b/tests/fixtures/spec-examples/negative/opendox-snapshot-edge-names-a-document.negative.yaml new file mode 100644 index 00000000..189d97e5 --- /dev/null +++ b/tests/fixtures/spec-examples/negative/opendox-snapshot-edge-names-a-document.negative.yaml @@ -0,0 +1,50 @@ +# expected_failure: edge-names-a-document +# INVALID: a group edge names a document the snapshot does not hold. +# The no-front-matter example with this one change, so it breaks this +# reference rule and no other. +{ + "schema_version": 1, + "kind": "opendox-snapshot", + "repository": "plain-notes", + "generation": {"source_revision": "0a1b2c3d4e5f60718293a4b5c6d7e8f901234567"}, + "documents": [ + { + "id": "kitchen", + "path": "kitchen.md", + "stage": "source", + "title": null, + "summary": null, + "topics": ["bread", "oven"] + }, + { + "id": "bakery-visit", + "path": "bakery-visit.md", + "stage": "source", + "title": null, + "summary": null, + "topics": ["bread", "flour"] + }, + { + "id": "garden", + "path": "garden.md", + "stage": "source", + "title": null, + "summary": null, + "topics": ["herbs"] + } + ], + "clusters": [ + { + "id": "bread", + "name": "bread", + "topics": ["bread"], + "document_edges": [ + {"document": "kitchen", "matched_topics": ["bread"]}, + {"document": "no-such-document", "matched_topics": ["bread"]} + ] + } + ], + "possibles": [], + "staged_topics": [], + "changes": [] +} diff --git a/tests/fixtures/spec-examples/negative/opendox-snapshot-envelope-keys.negative.yaml b/tests/fixtures/spec-examples/negative/opendox-snapshot-envelope-keys.negative.yaml new file mode 100644 index 00000000..2261d5f3 --- /dev/null +++ b/tests/fixtures/spec-examples/negative/opendox-snapshot-envelope-keys.negative.yaml @@ -0,0 +1,49 @@ +# expected_failure: envelope-keys +# INVALID: the snapshot has no `changes` section. +# The no-front-matter example with this one change, so it breaks this +# shape rule and no other. +{ + "schema_version": 1, + "kind": "opendox-snapshot", + "repository": "plain-notes", + "generation": {"source_revision": "0a1b2c3d4e5f60718293a4b5c6d7e8f901234567"}, + "documents": [ + { + "id": "kitchen", + "path": "kitchen.md", + "stage": "source", + "title": null, + "summary": null, + "topics": ["bread", "oven"] + }, + { + "id": "bakery-visit", + "path": "bakery-visit.md", + "stage": "source", + "title": null, + "summary": null, + "topics": ["bread", "flour"] + }, + { + "id": "garden", + "path": "garden.md", + "stage": "source", + "title": null, + "summary": null, + "topics": ["herbs"] + } + ], + "clusters": [ + { + "id": "bread", + "name": "bread", + "topics": ["bread"], + "document_edges": [ + {"document": "kitchen", "matched_topics": ["bread"]}, + {"document": "bakery-visit", "matched_topics": ["bread"]} + ] + } + ], + "possibles": [], + "staged_topics": [] +} diff --git a/tests/fixtures/spec-examples/negative/opendox-snapshot-generated-at-is-rfc3339.negative.yaml b/tests/fixtures/spec-examples/negative/opendox-snapshot-generated-at-is-rfc3339.negative.yaml new file mode 100644 index 00000000..5b94b526 --- /dev/null +++ b/tests/fixtures/spec-examples/negative/opendox-snapshot-generated-at-is-rfc3339.negative.yaml @@ -0,0 +1,53 @@ +# expected_failure: generated-at-is-rfc3339 +# INVALID: `generated_at` names a day that February does not have. +# The no-front-matter example with this one change, so it breaks this +# shape rule and no other. +{ + "schema_version": 1, + "kind": "opendox-snapshot", + "repository": "plain-notes", + "generation": { + "source_revision": "0a1b2c3d4e5f60718293a4b5c6d7e8f901234567", + "generated_at": "2026-02-31T12:00:00Z" + }, + "documents": [ + { + "id": "kitchen", + "path": "kitchen.md", + "stage": "source", + "title": null, + "summary": null, + "topics": ["bread", "oven"] + }, + { + "id": "bakery-visit", + "path": "bakery-visit.md", + "stage": "source", + "title": null, + "summary": null, + "topics": ["bread", "flour"] + }, + { + "id": "garden", + "path": "garden.md", + "stage": "source", + "title": null, + "summary": null, + "topics": ["herbs"] + } + ], + "clusters": [ + { + "id": "bread", + "name": "bread", + "topics": ["bread"], + "document_edges": [ + {"document": "kitchen", "matched_topics": ["bread"]}, + {"document": "bakery-visit", "matched_topics": ["bread"]} + ] + } + ], + "possibles": [], + "staged_topics": [], + "changes": [] +} diff --git a/tests/fixtures/spec-examples/negative/opendox-snapshot-generation-anchored.negative.yaml b/tests/fixtures/spec-examples/negative/opendox-snapshot-generation-anchored.negative.yaml new file mode 100644 index 00000000..e1e69529 --- /dev/null +++ b/tests/fixtures/spec-examples/negative/opendox-snapshot-generation-anchored.negative.yaml @@ -0,0 +1,50 @@ +# expected_failure: generation-anchored +# INVALID: `generation` carries no `source_revision`. +# The no-front-matter example with this one change, so it breaks this +# shape rule and no other. +{ + "schema_version": 1, + "kind": "opendox-snapshot", + "repository": "plain-notes", + "generation": {"generated_at": "2026-09-27T12:00:00Z"}, + "documents": [ + { + "id": "kitchen", + "path": "kitchen.md", + "stage": "source", + "title": null, + "summary": null, + "topics": ["bread", "oven"] + }, + { + "id": "bakery-visit", + "path": "bakery-visit.md", + "stage": "source", + "title": null, + "summary": null, + "topics": ["bread", "flour"] + }, + { + "id": "garden", + "path": "garden.md", + "stage": "source", + "title": null, + "summary": null, + "topics": ["herbs"] + } + ], + "clusters": [ + { + "id": "bread", + "name": "bread", + "topics": ["bread"], + "document_edges": [ + {"document": "kitchen", "matched_topics": ["bread"]}, + {"document": "bakery-visit", "matched_topics": ["bread"]} + ] + } + ], + "possibles": [], + "staged_topics": [], + "changes": [] +} diff --git a/tests/fixtures/spec-examples/negative/opendox-snapshot-group-has-a-topic.negative.yaml b/tests/fixtures/spec-examples/negative/opendox-snapshot-group-has-a-topic.negative.yaml new file mode 100644 index 00000000..90cbd4c1 --- /dev/null +++ b/tests/fixtures/spec-examples/negative/opendox-snapshot-group-has-a-topic.negative.yaml @@ -0,0 +1,50 @@ +# expected_failure: group-has-a-topic +# INVALID: a group names no topic. +# The no-front-matter example with this one change, so it breaks this +# shape rule and no other. +{ + "schema_version": 1, + "kind": "opendox-snapshot", + "repository": "plain-notes", + "generation": {"source_revision": "0a1b2c3d4e5f60718293a4b5c6d7e8f901234567"}, + "documents": [ + { + "id": "kitchen", + "path": "kitchen.md", + "stage": "source", + "title": null, + "summary": null, + "topics": ["bread", "oven"] + }, + { + "id": "bakery-visit", + "path": "bakery-visit.md", + "stage": "source", + "title": null, + "summary": null, + "topics": ["bread", "flour"] + }, + { + "id": "garden", + "path": "garden.md", + "stage": "source", + "title": null, + "summary": null, + "topics": ["herbs"] + } + ], + "clusters": [ + { + "id": "bread", + "name": "bread", + "topics": [], + "document_edges": [ + {"document": "kitchen", "matched_topics": ["bread"]}, + {"document": "bakery-visit", "matched_topics": ["bread"]} + ] + } + ], + "possibles": [], + "staged_topics": [], + "changes": [] +} diff --git a/tests/fixtures/spec-examples/negative/opendox-snapshot-group-keys.negative.yaml b/tests/fixtures/spec-examples/negative/opendox-snapshot-group-keys.negative.yaml new file mode 100644 index 00000000..0dd56c3a --- /dev/null +++ b/tests/fixtures/spec-examples/negative/opendox-snapshot-group-keys.negative.yaml @@ -0,0 +1,49 @@ +# expected_failure: group-keys +# INVALID: a group carries no `name`. +# The no-front-matter example with this one change, so it breaks this +# shape rule and no other. +{ + "schema_version": 1, + "kind": "opendox-snapshot", + "repository": "plain-notes", + "generation": {"source_revision": "0a1b2c3d4e5f60718293a4b5c6d7e8f901234567"}, + "documents": [ + { + "id": "kitchen", + "path": "kitchen.md", + "stage": "source", + "title": null, + "summary": null, + "topics": ["bread", "oven"] + }, + { + "id": "bakery-visit", + "path": "bakery-visit.md", + "stage": "source", + "title": null, + "summary": null, + "topics": ["bread", "flour"] + }, + { + "id": "garden", + "path": "garden.md", + "stage": "source", + "title": null, + "summary": null, + "topics": ["herbs"] + } + ], + "clusters": [ + { + "id": "bread", + "topics": ["bread"], + "document_edges": [ + {"document": "kitchen", "matched_topics": ["bread"]}, + {"document": "bakery-visit", "matched_topics": ["bread"]} + ] + } + ], + "possibles": [], + "staged_topics": [], + "changes": [] +} diff --git a/tests/fixtures/spec-examples/negative/opendox-snapshot-id-is-text.negative.yaml b/tests/fixtures/spec-examples/negative/opendox-snapshot-id-is-text.negative.yaml new file mode 100644 index 00000000..bf966223 --- /dev/null +++ b/tests/fixtures/spec-examples/negative/opendox-snapshot-id-is-text.negative.yaml @@ -0,0 +1,50 @@ +# expected_failure: id-is-text +# INVALID: an entry in `changes` has an empty id. +# The no-front-matter example with this one change, so it breaks this +# shape rule and no other. +{ + "schema_version": 1, + "kind": "opendox-snapshot", + "repository": "plain-notes", + "generation": {"source_revision": "0a1b2c3d4e5f60718293a4b5c6d7e8f901234567"}, + "documents": [ + { + "id": "kitchen", + "path": "kitchen.md", + "stage": "source", + "title": null, + "summary": null, + "topics": ["bread", "oven"] + }, + { + "id": "bakery-visit", + "path": "bakery-visit.md", + "stage": "source", + "title": null, + "summary": null, + "topics": ["bread", "flour"] + }, + { + "id": "garden", + "path": "garden.md", + "stage": "source", + "title": null, + "summary": null, + "topics": ["herbs"] + } + ], + "clusters": [ + { + "id": "bread", + "name": "bread", + "topics": ["bread"], + "document_edges": [ + {"document": "kitchen", "matched_topics": ["bread"]}, + {"document": "bakery-visit", "matched_topics": ["bread"]} + ] + } + ], + "possibles": [], + "staged_topics": [], + "changes": [{"id": "", "status": "active"}] +} diff --git a/tests/fixtures/spec-examples/negative/opendox-snapshot-ids-are-unique.negative.yaml b/tests/fixtures/spec-examples/negative/opendox-snapshot-ids-are-unique.negative.yaml new file mode 100644 index 00000000..a4bdf242 --- /dev/null +++ b/tests/fixtures/spec-examples/negative/opendox-snapshot-ids-are-unique.negative.yaml @@ -0,0 +1,50 @@ +# expected_failure: ids-are-unique +# INVALID: two documents share one id. +# The no-front-matter example with this one change, so it breaks this +# reference rule and no other. +{ + "schema_version": 1, + "kind": "opendox-snapshot", + "repository": "plain-notes", + "generation": {"source_revision": "0a1b2c3d4e5f60718293a4b5c6d7e8f901234567"}, + "documents": [ + { + "id": "kitchen", + "path": "kitchen.md", + "stage": "source", + "title": null, + "summary": null, + "topics": ["bread", "oven"] + }, + { + "id": "bakery-visit", + "path": "bakery-visit.md", + "stage": "source", + "title": null, + "summary": null, + "topics": ["bread", "flour"] + }, + { + "id": "kitchen", + "path": "garden.md", + "stage": "source", + "title": null, + "summary": null, + "topics": ["herbs"] + } + ], + "clusters": [ + { + "id": "bread", + "name": "bread", + "topics": ["bread"], + "document_edges": [ + {"document": "kitchen", "matched_topics": ["bread"]}, + {"document": "bakery-visit", "matched_topics": ["bread"]} + ] + } + ], + "possibles": [], + "staged_topics": [], + "changes": [] +} diff --git a/tests/fixtures/spec-examples/negative/opendox-snapshot-keyword-entry-keys.negative.yaml b/tests/fixtures/spec-examples/negative/opendox-snapshot-keyword-entry-keys.negative.yaml new file mode 100644 index 00000000..b616092e --- /dev/null +++ b/tests/fixtures/spec-examples/negative/opendox-snapshot-keyword-entry-keys.negative.yaml @@ -0,0 +1,56 @@ +# expected_failure: keyword-entry-keys +# INVALID: a keyword_index entry's count is text, not a whole number. +# The no-front-matter example with a correct keyword_index added, and +# this one change, so it breaks this shape rule and no other. +{ + "schema_version": 1, + "kind": "opendox-snapshot", + "repository": "plain-notes", + "generation": {"source_revision": "0a1b2c3d4e5f60718293a4b5c6d7e8f901234567"}, + "documents": [ + { + "id": "kitchen", + "path": "kitchen.md", + "stage": "source", + "title": null, + "summary": null, + "topics": ["bread", "oven"] + }, + { + "id": "bakery-visit", + "path": "bakery-visit.md", + "stage": "source", + "title": null, + "summary": null, + "topics": ["bread", "flour"] + }, + { + "id": "garden", + "path": "garden.md", + "stage": "source", + "title": null, + "summary": null, + "topics": ["herbs"] + } + ], + "clusters": [ + { + "id": "bread", + "name": "bread", + "topics": ["bread"], + "document_edges": [ + {"document": "kitchen", "matched_topics": ["bread"]}, + {"document": "bakery-visit", "matched_topics": ["bread"]} + ] + } + ], + "possibles": [], + "staged_topics": [], + "changes": [], + "keyword_index": [ + {"keyword": "bread", "declared_doc_count": 2}, + {"keyword": "flour", "declared_doc_count": 1}, + {"keyword": "herbs", "declared_doc_count": 1}, + {"keyword": "oven", "declared_doc_count": "one"} + ] +} diff --git a/tests/fixtures/spec-examples/negative/opendox-snapshot-keyword-index-matches-topics.negative.yaml b/tests/fixtures/spec-examples/negative/opendox-snapshot-keyword-index-matches-topics.negative.yaml new file mode 100644 index 00000000..1227fef6 --- /dev/null +++ b/tests/fixtures/spec-examples/negative/opendox-snapshot-keyword-index-matches-topics.negative.yaml @@ -0,0 +1,56 @@ +# expected_failure: keyword-index-matches-topics +# INVALID: keyword_index counts three documents for a topic two carry. +# The no-front-matter example with a correct keyword_index added, and +# this one change, so it breaks this reference rule and no other. +{ + "schema_version": 1, + "kind": "opendox-snapshot", + "repository": "plain-notes", + "generation": {"source_revision": "0a1b2c3d4e5f60718293a4b5c6d7e8f901234567"}, + "documents": [ + { + "id": "kitchen", + "path": "kitchen.md", + "stage": "source", + "title": null, + "summary": null, + "topics": ["bread", "oven"] + }, + { + "id": "bakery-visit", + "path": "bakery-visit.md", + "stage": "source", + "title": null, + "summary": null, + "topics": ["bread", "flour"] + }, + { + "id": "garden", + "path": "garden.md", + "stage": "source", + "title": null, + "summary": null, + "topics": ["herbs"] + } + ], + "clusters": [ + { + "id": "bread", + "name": "bread", + "topics": ["bread"], + "document_edges": [ + {"document": "kitchen", "matched_topics": ["bread"]}, + {"document": "bakery-visit", "matched_topics": ["bread"]} + ] + } + ], + "possibles": [], + "staged_topics": [], + "changes": [], + "keyword_index": [ + {"keyword": "bread", "declared_doc_count": 3}, + {"keyword": "flour", "declared_doc_count": 1}, + {"keyword": "herbs", "declared_doc_count": 1}, + {"keyword": "oven", "declared_doc_count": 1} + ] +} diff --git a/tests/fixtures/spec-examples/negative/opendox-snapshot-kind-is-opendox-snapshot.negative.yaml b/tests/fixtures/spec-examples/negative/opendox-snapshot-kind-is-opendox-snapshot.negative.yaml new file mode 100644 index 00000000..d8bdb8b3 --- /dev/null +++ b/tests/fixtures/spec-examples/negative/opendox-snapshot-kind-is-opendox-snapshot.negative.yaml @@ -0,0 +1,50 @@ +# expected_failure: kind-is-opendox-snapshot +# INVALID: `kind` names the governed generator's contract. +# The no-front-matter example with this one change, so it breaks this +# shape rule and no other. +{ + "schema_version": 1, + "kind": "ideation-dashboard-snapshot", + "repository": "plain-notes", + "generation": {"source_revision": "0a1b2c3d4e5f60718293a4b5c6d7e8f901234567"}, + "documents": [ + { + "id": "kitchen", + "path": "kitchen.md", + "stage": "source", + "title": null, + "summary": null, + "topics": ["bread", "oven"] + }, + { + "id": "bakery-visit", + "path": "bakery-visit.md", + "stage": "source", + "title": null, + "summary": null, + "topics": ["bread", "flour"] + }, + { + "id": "garden", + "path": "garden.md", + "stage": "source", + "title": null, + "summary": null, + "topics": ["herbs"] + } + ], + "clusters": [ + { + "id": "bread", + "name": "bread", + "topics": ["bread"], + "document_edges": [ + {"document": "kitchen", "matched_topics": ["bread"]}, + {"document": "bakery-visit", "matched_topics": ["bread"]} + ] + } + ], + "possibles": [], + "staged_topics": [], + "changes": [] +} diff --git a/tests/fixtures/spec-examples/negative/opendox-snapshot-one-edge-per-document.negative.yaml b/tests/fixtures/spec-examples/negative/opendox-snapshot-one-edge-per-document.negative.yaml new file mode 100644 index 00000000..d1cb95fe --- /dev/null +++ b/tests/fixtures/spec-examples/negative/opendox-snapshot-one-edge-per-document.negative.yaml @@ -0,0 +1,51 @@ +# expected_failure: one-edge-per-document +# INVALID: a group gives one document a second edge. +# The no-front-matter example with this one change, so it breaks this +# reference rule and no other. +{ + "schema_version": 1, + "kind": "opendox-snapshot", + "repository": "plain-notes", + "generation": {"source_revision": "0a1b2c3d4e5f60718293a4b5c6d7e8f901234567"}, + "documents": [ + { + "id": "kitchen", + "path": "kitchen.md", + "stage": "source", + "title": null, + "summary": null, + "topics": ["bread", "oven"] + }, + { + "id": "bakery-visit", + "path": "bakery-visit.md", + "stage": "source", + "title": null, + "summary": null, + "topics": ["bread", "flour"] + }, + { + "id": "garden", + "path": "garden.md", + "stage": "source", + "title": null, + "summary": null, + "topics": ["herbs"] + } + ], + "clusters": [ + { + "id": "bread", + "name": "bread", + "topics": ["bread"], + "document_edges": [ + {"document": "kitchen", "matched_topics": ["bread"]}, + {"document": "bakery-visit", "matched_topics": ["bread"]}, + {"document": "kitchen", "matched_topics": ["bread"]} + ] + } + ], + "possibles": [], + "staged_topics": [], + "changes": [] +} diff --git a/tests/fixtures/spec-examples/negative/opendox-snapshot-path-is-repo-relative.negative.yaml b/tests/fixtures/spec-examples/negative/opendox-snapshot-path-is-repo-relative.negative.yaml new file mode 100644 index 00000000..339b0c67 --- /dev/null +++ b/tests/fixtures/spec-examples/negative/opendox-snapshot-path-is-repo-relative.negative.yaml @@ -0,0 +1,50 @@ +# expected_failure: path-is-repo-relative +# INVALID: a document's path climbs out of the repository. +# The no-front-matter example with this one change, so it breaks this +# shape rule and no other. +{ + "schema_version": 1, + "kind": "opendox-snapshot", + "repository": "plain-notes", + "generation": {"source_revision": "0a1b2c3d4e5f60718293a4b5c6d7e8f901234567"}, + "documents": [ + { + "id": "kitchen", + "path": "kitchen.md", + "stage": "source", + "title": null, + "summary": null, + "topics": ["bread", "oven"] + }, + { + "id": "bakery-visit", + "path": "bakery-visit.md", + "stage": "source", + "title": null, + "summary": null, + "topics": ["bread", "flour"] + }, + { + "id": "garden", + "path": "../outside/garden.md", + "stage": "source", + "title": null, + "summary": null, + "topics": ["herbs"] + } + ], + "clusters": [ + { + "id": "bread", + "name": "bread", + "topics": ["bread"], + "document_edges": [ + {"document": "kitchen", "matched_topics": ["bread"]}, + {"document": "bakery-visit", "matched_topics": ["bread"]} + ] + } + ], + "possibles": [], + "staged_topics": [], + "changes": [] +} diff --git a/tests/fixtures/spec-examples/negative/opendox-snapshot-pick-names-a-selection.negative.yaml b/tests/fixtures/spec-examples/negative/opendox-snapshot-pick-names-a-selection.negative.yaml new file mode 100644 index 00000000..1d5d9514 --- /dev/null +++ b/tests/fixtures/spec-examples/negative/opendox-snapshot-pick-names-a-selection.negative.yaml @@ -0,0 +1,57 @@ +# expected_failure: pick-names-a-selection +# INVALID: a selected candidate's pick names no selection. +# The no-front-matter example with this one change, so it breaks this +# reference rule and no other. +{ + "schema_version": 1, + "kind": "opendox-snapshot", + "repository": "plain-notes", + "generation": {"source_revision": "0a1b2c3d4e5f60718293a4b5c6d7e8f901234567"}, + "documents": [ + { + "id": "kitchen", + "path": "kitchen.md", + "stage": "source", + "title": null, + "summary": null, + "topics": ["bread", "oven"] + }, + { + "id": "bakery-visit", + "path": "bakery-visit.md", + "stage": "source", + "title": null, + "summary": null, + "topics": ["bread", "flour"] + }, + { + "id": "garden", + "path": "garden.md", + "stage": "source", + "title": null, + "summary": null, + "topics": ["herbs"] + } + ], + "clusters": [ + { + "id": "bread", + "name": "bread", + "topics": ["bread"], + "document_edges": [ + {"document": "kitchen", "matched_topics": ["bread"]}, + {"document": "bakery-visit", "matched_topics": ["bread"]} + ] + } + ], + "possibles": [ + { + "id": "loaf-club", + "title": "A weekly loaf club", + "state": "selected", + "pick": {"staging_id": "no-such-selection"} + } + ], + "staged_topics": [], + "changes": [] +} diff --git a/tests/fixtures/spec-examples/negative/opendox-snapshot-repository-is-text.negative.yaml b/tests/fixtures/spec-examples/negative/opendox-snapshot-repository-is-text.negative.yaml new file mode 100644 index 00000000..4ba82e43 --- /dev/null +++ b/tests/fixtures/spec-examples/negative/opendox-snapshot-repository-is-text.negative.yaml @@ -0,0 +1,50 @@ +# expected_failure: repository-is-text +# INVALID: `repository` is empty. +# The no-front-matter example with this one change, so it breaks this +# shape rule and no other. +{ + "schema_version": 1, + "kind": "opendox-snapshot", + "repository": "", + "generation": {"source_revision": "0a1b2c3d4e5f60718293a4b5c6d7e8f901234567"}, + "documents": [ + { + "id": "kitchen", + "path": "kitchen.md", + "stage": "source", + "title": null, + "summary": null, + "topics": ["bread", "oven"] + }, + { + "id": "bakery-visit", + "path": "bakery-visit.md", + "stage": "source", + "title": null, + "summary": null, + "topics": ["bread", "flour"] + }, + { + "id": "garden", + "path": "garden.md", + "stage": "source", + "title": null, + "summary": null, + "topics": ["herbs"] + } + ], + "clusters": [ + { + "id": "bread", + "name": "bread", + "topics": ["bread"], + "document_edges": [ + {"document": "kitchen", "matched_topics": ["bread"]}, + {"document": "bakery-visit", "matched_topics": ["bread"]} + ] + } + ], + "possibles": [], + "staged_topics": [], + "changes": [] +} diff --git a/tests/fixtures/spec-examples/negative/opendox-snapshot-schema-version-is-1.negative.yaml b/tests/fixtures/spec-examples/negative/opendox-snapshot-schema-version-is-1.negative.yaml new file mode 100644 index 00000000..8c27d842 --- /dev/null +++ b/tests/fixtures/spec-examples/negative/opendox-snapshot-schema-version-is-1.negative.yaml @@ -0,0 +1,50 @@ +# expected_failure: schema-version-is-1 +# INVALID: `schema_version` is 2. +# The no-front-matter example with this one change, so it breaks this +# shape rule and no other. +{ + "schema_version": 2, + "kind": "opendox-snapshot", + "repository": "plain-notes", + "generation": {"source_revision": "0a1b2c3d4e5f60718293a4b5c6d7e8f901234567"}, + "documents": [ + { + "id": "kitchen", + "path": "kitchen.md", + "stage": "source", + "title": null, + "summary": null, + "topics": ["bread", "oven"] + }, + { + "id": "bakery-visit", + "path": "bakery-visit.md", + "stage": "source", + "title": null, + "summary": null, + "topics": ["bread", "flour"] + }, + { + "id": "garden", + "path": "garden.md", + "stage": "source", + "title": null, + "summary": null, + "topics": ["herbs"] + } + ], + "clusters": [ + { + "id": "bread", + "name": "bread", + "topics": ["bread"], + "document_edges": [ + {"document": "kitchen", "matched_topics": ["bread"]}, + {"document": "bakery-visit", "matched_topics": ["bread"]} + ] + } + ], + "possibles": [], + "staged_topics": [], + "changes": [] +} diff --git a/tests/fixtures/spec-examples/negative/opendox-snapshot-section-is-a-list.negative.yaml b/tests/fixtures/spec-examples/negative/opendox-snapshot-section-is-a-list.negative.yaml new file mode 100644 index 00000000..183a28ce --- /dev/null +++ b/tests/fixtures/spec-examples/negative/opendox-snapshot-section-is-a-list.negative.yaml @@ -0,0 +1,50 @@ +# expected_failure: section-is-a-list +# INVALID: `possibles` is an object, not a list. +# The no-front-matter example with this one change, so it breaks this +# shape rule and no other. +{ + "schema_version": 1, + "kind": "opendox-snapshot", + "repository": "plain-notes", + "generation": {"source_revision": "0a1b2c3d4e5f60718293a4b5c6d7e8f901234567"}, + "documents": [ + { + "id": "kitchen", + "path": "kitchen.md", + "stage": "source", + "title": null, + "summary": null, + "topics": ["bread", "oven"] + }, + { + "id": "bakery-visit", + "path": "bakery-visit.md", + "stage": "source", + "title": null, + "summary": null, + "topics": ["bread", "flour"] + }, + { + "id": "garden", + "path": "garden.md", + "stage": "source", + "title": null, + "summary": null, + "topics": ["herbs"] + } + ], + "clusters": [ + { + "id": "bread", + "name": "bread", + "topics": ["bread"], + "document_edges": [ + {"document": "kitchen", "matched_topics": ["bread"]}, + {"document": "bakery-visit", "matched_topics": ["bread"]} + ] + } + ], + "possibles": {}, + "staged_topics": [], + "changes": [] +} diff --git a/tests/fixtures/spec-examples/negative/opendox-snapshot-selected-candidate-has-pick.negative.yaml b/tests/fixtures/spec-examples/negative/opendox-snapshot-selected-candidate-has-pick.negative.yaml new file mode 100644 index 00000000..9e80bdde --- /dev/null +++ b/tests/fixtures/spec-examples/negative/opendox-snapshot-selected-candidate-has-pick.negative.yaml @@ -0,0 +1,50 @@ +# expected_failure: selected-candidate-has-pick +# INVALID: a selected candidate carries no `pick`. +# The no-front-matter example with this one change, so it breaks this +# shape rule and no other. +{ + "schema_version": 1, + "kind": "opendox-snapshot", + "repository": "plain-notes", + "generation": {"source_revision": "0a1b2c3d4e5f60718293a4b5c6d7e8f901234567"}, + "documents": [ + { + "id": "kitchen", + "path": "kitchen.md", + "stage": "source", + "title": null, + "summary": null, + "topics": ["bread", "oven"] + }, + { + "id": "bakery-visit", + "path": "bakery-visit.md", + "stage": "source", + "title": null, + "summary": null, + "topics": ["bread", "flour"] + }, + { + "id": "garden", + "path": "garden.md", + "stage": "source", + "title": null, + "summary": null, + "topics": ["herbs"] + } + ], + "clusters": [ + { + "id": "bread", + "name": "bread", + "topics": ["bread"], + "document_edges": [ + {"document": "kitchen", "matched_topics": ["bread"]}, + {"document": "bakery-visit", "matched_topics": ["bread"]} + ] + } + ], + "possibles": [{"id": "loaf-club", "title": "A weekly loaf club", "state": "selected"}], + "staged_topics": [], + "changes": [] +} diff --git a/tests/fixtures/spec-examples/negative/opendox-snapshot-selection-keys.negative.yaml b/tests/fixtures/spec-examples/negative/opendox-snapshot-selection-keys.negative.yaml new file mode 100644 index 00000000..d2c54587 --- /dev/null +++ b/tests/fixtures/spec-examples/negative/opendox-snapshot-selection-keys.negative.yaml @@ -0,0 +1,50 @@ +# expected_failure: selection-keys +# INVALID: a selection carries no `staging_id`. +# The no-front-matter example with this one change, so it breaks this +# shape rule and no other. +{ + "schema_version": 1, + "kind": "opendox-snapshot", + "repository": "plain-notes", + "generation": {"source_revision": "0a1b2c3d4e5f60718293a4b5c6d7e8f901234567"}, + "documents": [ + { + "id": "kitchen", + "path": "kitchen.md", + "stage": "source", + "title": null, + "summary": null, + "topics": ["bread", "oven"] + }, + { + "id": "bakery-visit", + "path": "bakery-visit.md", + "stage": "source", + "title": null, + "summary": null, + "topics": ["bread", "flour"] + }, + { + "id": "garden", + "path": "garden.md", + "stage": "source", + "title": null, + "summary": null, + "topics": ["herbs"] + } + ], + "clusters": [ + { + "id": "bread", + "name": "bread", + "topics": ["bread"], + "document_edges": [ + {"document": "kitchen", "matched_topics": ["bread"]}, + {"document": "bakery-visit", "matched_topics": ["bread"]} + ] + } + ], + "possibles": [], + "staged_topics": [{"files": ["garden.md"]}], + "changes": [] +} diff --git a/tests/fixtures/spec-examples/negative/opendox-snapshot-stage-is-a-station-role.negative.yaml b/tests/fixtures/spec-examples/negative/opendox-snapshot-stage-is-a-station-role.negative.yaml new file mode 100644 index 00000000..51299b5b --- /dev/null +++ b/tests/fixtures/spec-examples/negative/opendox-snapshot-stage-is-a-station-role.negative.yaml @@ -0,0 +1,50 @@ +# expected_failure: stage-is-a-station-role +# INVALID: a document's stage is not one of the six role keys. +# The no-front-matter example with this one change, so it breaks this +# shape rule and no other. +{ + "schema_version": 1, + "kind": "opendox-snapshot", + "repository": "plain-notes", + "generation": {"source_revision": "0a1b2c3d4e5f60718293a4b5c6d7e8f901234567"}, + "documents": [ + { + "id": "kitchen", + "path": "kitchen.md", + "stage": "source", + "title": null, + "summary": null, + "topics": ["bread", "oven"] + }, + { + "id": "bakery-visit", + "path": "bakery-visit.md", + "stage": "source", + "title": null, + "summary": null, + "topics": ["bread", "flour"] + }, + { + "id": "garden", + "path": "garden.md", + "stage": "idea", + "title": null, + "summary": null, + "topics": ["herbs"] + } + ], + "clusters": [ + { + "id": "bread", + "name": "bread", + "topics": ["bread"], + "document_edges": [ + {"document": "kitchen", "matched_topics": ["bread"]}, + {"document": "bakery-visit", "matched_topics": ["bread"]} + ] + } + ], + "possibles": [], + "staged_topics": [], + "changes": [] +} diff --git a/tests/fixtures/spec-examples/negative/opendox-snapshot-submission-keys.negative.yaml b/tests/fixtures/spec-examples/negative/opendox-snapshot-submission-keys.negative.yaml new file mode 100644 index 00000000..fd28e69d --- /dev/null +++ b/tests/fixtures/spec-examples/negative/opendox-snapshot-submission-keys.negative.yaml @@ -0,0 +1,50 @@ +# expected_failure: submission-keys +# INVALID: an entry in `changes` carries no `status`. +# The no-front-matter example with this one change, so it breaks this +# shape rule and no other. +{ + "schema_version": 1, + "kind": "opendox-snapshot", + "repository": "plain-notes", + "generation": {"source_revision": "0a1b2c3d4e5f60718293a4b5c6d7e8f901234567"}, + "documents": [ + { + "id": "kitchen", + "path": "kitchen.md", + "stage": "source", + "title": null, + "summary": null, + "topics": ["bread", "oven"] + }, + { + "id": "bakery-visit", + "path": "bakery-visit.md", + "stage": "source", + "title": null, + "summary": null, + "topics": ["bread", "flour"] + }, + { + "id": "garden", + "path": "garden.md", + "stage": "source", + "title": null, + "summary": null, + "topics": ["herbs"] + } + ], + "clusters": [ + { + "id": "bread", + "name": "bread", + "topics": ["bread"], + "document_edges": [ + {"document": "kitchen", "matched_topics": ["bread"]}, + {"document": "bakery-visit", "matched_topics": ["bread"]} + ] + } + ], + "possibles": [], + "staged_topics": [], + "changes": [{"id": "bake-sale"}] +} diff --git a/tests/fixtures/spec-examples/negative/opendox-snapshot-submission-status-is-known.negative.yaml b/tests/fixtures/spec-examples/negative/opendox-snapshot-submission-status-is-known.negative.yaml new file mode 100644 index 00000000..8f05c328 --- /dev/null +++ b/tests/fixtures/spec-examples/negative/opendox-snapshot-submission-status-is-known.negative.yaml @@ -0,0 +1,50 @@ +# expected_failure: submission-status-is-known +# INVALID: an entry in `changes` has a status neither station reads. +# The no-front-matter example with this one change, so it breaks this +# shape rule and no other. +{ + "schema_version": 1, + "kind": "opendox-snapshot", + "repository": "plain-notes", + "generation": {"source_revision": "0a1b2c3d4e5f60718293a4b5c6d7e8f901234567"}, + "documents": [ + { + "id": "kitchen", + "path": "kitchen.md", + "stage": "source", + "title": null, + "summary": null, + "topics": ["bread", "oven"] + }, + { + "id": "bakery-visit", + "path": "bakery-visit.md", + "stage": "source", + "title": null, + "summary": null, + "topics": ["bread", "flour"] + }, + { + "id": "garden", + "path": "garden.md", + "stage": "source", + "title": null, + "summary": null, + "topics": ["herbs"] + } + ], + "clusters": [ + { + "id": "bread", + "name": "bread", + "topics": ["bread"], + "document_edges": [ + {"document": "kitchen", "matched_topics": ["bread"]}, + {"document": "bakery-visit", "matched_topics": ["bread"]} + ] + } + ], + "possibles": [], + "staged_topics": [], + "changes": [{"id": "bake-sale", "status": "open"}] +} diff --git a/tests/fixtures/spec-examples/negative/opendox-snapshot-target-names-a-submission.negative.yaml b/tests/fixtures/spec-examples/negative/opendox-snapshot-target-names-a-submission.negative.yaml new file mode 100644 index 00000000..b7653024 --- /dev/null +++ b/tests/fixtures/spec-examples/negative/opendox-snapshot-target-names-a-submission.negative.yaml @@ -0,0 +1,50 @@ +# expected_failure: target-names-a-submission +# INVALID: a selection's target names no entry in `changes`. +# The no-front-matter example with this one change, so it breaks this +# reference rule and no other. +{ + "schema_version": 1, + "kind": "opendox-snapshot", + "repository": "plain-notes", + "generation": {"source_revision": "0a1b2c3d4e5f60718293a4b5c6d7e8f901234567"}, + "documents": [ + { + "id": "kitchen", + "path": "kitchen.md", + "stage": "source", + "title": null, + "summary": null, + "topics": ["bread", "oven"] + }, + { + "id": "bakery-visit", + "path": "bakery-visit.md", + "stage": "source", + "title": null, + "summary": null, + "topics": ["bread", "flour"] + }, + { + "id": "garden", + "path": "garden.md", + "stage": "source", + "title": null, + "summary": null, + "topics": ["herbs"] + } + ], + "clusters": [ + { + "id": "bread", + "name": "bread", + "topics": ["bread"], + "document_edges": [ + {"document": "kitchen", "matched_topics": ["bread"]}, + {"document": "bakery-visit", "matched_topics": ["bread"]} + ] + } + ], + "possibles": [], + "staged_topics": [{"staging_id": "loaf-club", "target_change": "no-such-submission"}], + "changes": [] +} diff --git a/tests/fixtures/spec-examples/negative/opendox-snapshot-title-and-summary-are-text.negative.yaml b/tests/fixtures/spec-examples/negative/opendox-snapshot-title-and-summary-are-text.negative.yaml new file mode 100644 index 00000000..8701fbb4 --- /dev/null +++ b/tests/fixtures/spec-examples/negative/opendox-snapshot-title-and-summary-are-text.negative.yaml @@ -0,0 +1,50 @@ +# expected_failure: title-and-summary-are-text +# INVALID: a document's title is empty text. +# The no-front-matter example with this one change, so it breaks this +# shape rule and no other. +{ + "schema_version": 1, + "kind": "opendox-snapshot", + "repository": "plain-notes", + "generation": {"source_revision": "0a1b2c3d4e5f60718293a4b5c6d7e8f901234567"}, + "documents": [ + { + "id": "kitchen", + "path": "kitchen.md", + "stage": "source", + "title": null, + "summary": null, + "topics": ["bread", "oven"] + }, + { + "id": "bakery-visit", + "path": "bakery-visit.md", + "stage": "source", + "title": null, + "summary": null, + "topics": ["bread", "flour"] + }, + { + "id": "garden", + "path": "garden.md", + "stage": "source", + "title": "", + "summary": null, + "topics": ["herbs"] + } + ], + "clusters": [ + { + "id": "bread", + "name": "bread", + "topics": ["bread"], + "document_edges": [ + {"document": "kitchen", "matched_topics": ["bread"]}, + {"document": "bakery-visit", "matched_topics": ["bread"]} + ] + } + ], + "possibles": [], + "staged_topics": [], + "changes": [] +} diff --git a/tests/fixtures/spec-examples/negative/opendox-snapshot-topic-is-trimmed-text.negative.yaml b/tests/fixtures/spec-examples/negative/opendox-snapshot-topic-is-trimmed-text.negative.yaml new file mode 100644 index 00000000..c8a7b2a1 --- /dev/null +++ b/tests/fixtures/spec-examples/negative/opendox-snapshot-topic-is-trimmed-text.negative.yaml @@ -0,0 +1,50 @@ +# expected_failure: topic-is-trimmed-text +# INVALID: a document's topic carries trailing whitespace. +# The no-front-matter example with this one change, so it breaks this +# shape rule and no other. +{ + "schema_version": 1, + "kind": "opendox-snapshot", + "repository": "plain-notes", + "generation": {"source_revision": "0a1b2c3d4e5f60718293a4b5c6d7e8f901234567"}, + "documents": [ + { + "id": "kitchen", + "path": "kitchen.md", + "stage": "source", + "title": null, + "summary": null, + "topics": ["bread", "oven"] + }, + { + "id": "bakery-visit", + "path": "bakery-visit.md", + "stage": "source", + "title": null, + "summary": null, + "topics": ["bread", "flour"] + }, + { + "id": "garden", + "path": "garden.md", + "stage": "source", + "title": null, + "summary": null, + "topics": ["herbs "] + } + ], + "clusters": [ + { + "id": "bread", + "name": "bread", + "topics": ["bread"], + "document_edges": [ + {"document": "kitchen", "matched_topics": ["bread"]}, + {"document": "bakery-visit", "matched_topics": ["bread"]} + ] + } + ], + "possibles": [], + "staged_topics": [], + "changes": [] +} diff --git a/tests/fixtures/spec-examples/negative/opendox-snapshot-topics-are-unique.negative.yaml b/tests/fixtures/spec-examples/negative/opendox-snapshot-topics-are-unique.negative.yaml new file mode 100644 index 00000000..9880af37 --- /dev/null +++ b/tests/fixtures/spec-examples/negative/opendox-snapshot-topics-are-unique.negative.yaml @@ -0,0 +1,50 @@ +# expected_failure: topics-are-unique +# INVALID: a document names one topic twice. +# The no-front-matter example with this one change, so it breaks this +# shape rule and no other. +{ + "schema_version": 1, + "kind": "opendox-snapshot", + "repository": "plain-notes", + "generation": {"source_revision": "0a1b2c3d4e5f60718293a4b5c6d7e8f901234567"}, + "documents": [ + { + "id": "kitchen", + "path": "kitchen.md", + "stage": "source", + "title": null, + "summary": null, + "topics": ["bread", "oven"] + }, + { + "id": "bakery-visit", + "path": "bakery-visit.md", + "stage": "source", + "title": null, + "summary": null, + "topics": ["bread", "flour"] + }, + { + "id": "garden", + "path": "garden.md", + "stage": "source", + "title": null, + "summary": null, + "topics": ["herbs", "herbs"] + } + ], + "clusters": [ + { + "id": "bread", + "name": "bread", + "topics": ["bread"], + "document_edges": [ + {"document": "kitchen", "matched_topics": ["bread"]}, + {"document": "bakery-visit", "matched_topics": ["bread"]} + ] + } + ], + "possibles": [], + "staged_topics": [], + "changes": [] +} diff --git a/tests/fixtures/spec-examples/opendox-snapshot-no-front-matter.example.yaml b/tests/fixtures/spec-examples/opendox-snapshot-no-front-matter.example.yaml new file mode 100644 index 00000000..296f9648 --- /dev/null +++ b/tests/fixtures/spec-examples/opendox-snapshot-no-front-matter.example.yaml @@ -0,0 +1,54 @@ +# VALID: a plain repository whose documents carry no front matter at all. +# +# Every document is a source, and title and summary are null. The topics are +# the ones a generator's topic rule derived (T054 names that rule; this +# contract does not). One group forms around the topic two documents share, +# which is the grouping tile AT-R1 opens the chat pane from. The other stations +# are empty lists, keyword_index is absent, and so is generated_at: this is the +# smallest shape the contract admits. +{ + "schema_version": 1, + "kind": "opendox-snapshot", + "repository": "plain-notes", + "generation": {"source_revision": "0a1b2c3d4e5f60718293a4b5c6d7e8f901234567"}, + "documents": [ + { + "id": "kitchen", + "path": "kitchen.md", + "stage": "source", + "title": null, + "summary": null, + "topics": ["bread", "oven"] + }, + { + "id": "bakery-visit", + "path": "bakery-visit.md", + "stage": "source", + "title": null, + "summary": null, + "topics": ["bread", "flour"] + }, + { + "id": "garden", + "path": "garden.md", + "stage": "source", + "title": null, + "summary": null, + "topics": ["herbs"] + } + ], + "clusters": [ + { + "id": "bread", + "name": "bread", + "topics": ["bread"], + "document_edges": [ + {"document": "kitchen", "matched_topics": ["bread"]}, + {"document": "bakery-visit", "matched_topics": ["bread"]} + ] + } + ], + "possibles": [], + "staged_topics": [], + "changes": [] +} diff --git a/tests/fixtures/spec-examples/opendox-snapshot-six-stations.example.yaml b/tests/fixtures/spec-examples/opendox-snapshot-six-stations.example.yaml new file mode 100644 index 00000000..188e669c --- /dev/null +++ b/tests/fixtures/spec-examples/opendox-snapshot-six-stations.example.yaml @@ -0,0 +1,164 @@ +# VALID: a plain repository whose documents sit in all six stations. +# +# Each document declares its station with the neutral `stage:` key (R1Q13 (a)); +# the three that declare `source` could as well have declared nothing. Three +# groups form around shared topics. The candidates show all four states, and +# the selected one's pick names the selection, whose target names the entry in +# the submission station. Each changes entry lists its files, as the selection +# does. keyword_index is present, which it need not be. +{ + "schema_version": 1, + "kind": "opendox-snapshot", + "repository": "community-garden", + "generation": { + "source_revision": "4f2b9c0d1e3a5b7c9d0e2f4a6b8c0d1e3f5a7b9c", + "generated_at": "2026-09-27T12:00:00Z", + "generator_version": "opendox-neutral-1" + }, + "documents": [ + { + "id": "notes/soil-test", + "path": "notes/soil-test.md", + "stage": "source", + "title": "Soil test results", + "summary": "pH and nutrient readings from the three beds.", + "topics": ["compost", "soil"] + }, + { + "id": "notes/compost-bins", + "path": "notes/compost-bins.md", + "stage": "source", + "title": "Compost bins", + "summary": "How the two bins are turned, and when.", + "topics": ["compost", "reuse"] + }, + { + "id": "notes/rain-barrels", + "path": "notes/rain-barrels.md", + "stage": "source", + "title": "Rain barrels", + "summary": null, + "topics": ["reuse", "water"] + }, + { + "id": "notes/watering", + "path": "notes/watering.md", + "stage": "grouping", + "title": "Watering, gathered", + "summary": "Everything on watering, in one place.", + "topics": ["water"] + }, + { + "id": "ideas/tool-library", + "path": "ideas/tool-library.md", + "stage": "candidate", + "title": "A shared tool library", + "summary": "Lend tools between plots instead of buying twice.", + "topics": ["reuse", "tools"] + }, + { + "id": "plans/rain-harvest", + "path": "plans/rain-harvest.md", + "stage": "selection", + "title": "Rain harvest plan", + "summary": "Chosen: gutters to barrels to beds.", + "topics": ["water"] + }, + { + "id": "submissions/rain-harvest-build", + "path": "submissions/rain-harvest-build.md", + "stage": "submission", + "title": "Build the rain harvest", + "summary": null, + "topics": ["water"] + }, + { + "id": "done/compost-rota", + "path": "done/compost-rota.md", + "stage": "completion", + "title": "Compost turning rota", + "summary": "Adopted in the spring.", + "topics": ["compost"] + } + ], + "clusters": [ + { + "id": "compost", + "name": "compost", + "topics": ["compost"], + "document_edges": [ + {"document": "notes/soil-test", "matched_topics": ["compost"]}, + {"document": "notes/compost-bins", "matched_topics": ["compost"]} + ] + }, + { + "id": "reuse", + "name": "reuse", + "topics": ["reuse"], + "document_edges": [ + {"document": "notes/compost-bins", "matched_topics": ["reuse"]}, + {"document": "notes/rain-barrels", "matched_topics": ["reuse"]} + ] + }, + { + "id": "watering", + "name": "Watering, gathered", + "topics": ["water"], + "document_edges": [ + {"document": "notes/rain-barrels", "matched_topics": ["water"]}, + {"document": "notes/watering", "matched_topics": ["water"]} + ] + } + ], + "possibles": [ + { + "id": "tool-library", + "title": "A shared tool library", + "claim": "Lend tools between plots instead of buying twice.", + "state": "unselected", + "claiming_clusters": ["reuse"] + }, + { + "id": "rain-harvest", + "title": "Harvest rain from the shed roof", + "state": "selected", + "claiming_clusters": ["watering"], + "pick": {"staging_id": "rain-harvest"} + }, + { + "id": "paved-paths", + "title": "Pave the paths", + "state": "declined", + "reason": "Paving stops the rain soaking into the beds.", + "claiming_clusters": ["watering"] + }, + { + "id": "bagged-compost", + "title": "Buy bagged compost", + "state": "replaced", + "reason": "The two bins make enough; the turning rota took its place." + } + ], + "staged_topics": [ + { + "staging_id": "rain-harvest", + "files": ["plans/rain-harvest.md"], + "target_change": "rain-harvest-build" + } + ], + "changes": [ + { + "id": "rain-harvest-build", + "status": "active", + "files": ["work/rain-harvest-build/plan.md", "work/rain-harvest-build/steps.md"] + }, + {"id": "compost-rota", "status": "archived", "files": ["work/compost-rota/plan.md"]} + ], + "keyword_index": [ + {"keyword": "compost", "declared_doc_count": 3}, + {"keyword": "reuse", "declared_doc_count": 3}, + {"keyword": "soil", "declared_doc_count": 1}, + {"keyword": "tools", "declared_doc_count": 1}, + {"keyword": "water", "declared_doc_count": 4} + ] +} diff --git a/tests/fixtures/spec-examples/workbench-chat-turn-v2-full-context.example.yaml b/tests/fixtures/spec-examples/workbench-chat-turn-v2-full-context.example.yaml new file mode 100644 index 00000000..7d3d3a3e --- /dev/null +++ b/tests/fixtures/spec-examples/workbench-chat-turn-v2-full-context.example.yaml @@ -0,0 +1,33 @@ +# VALID widened success (contract-v1.40): the ORDINARY posture, STATED. The +# knowledge service answered, the packet was assembled full, and the record says +# so explicitly rather than leaving it to be inferred from a missing key. +# +# THE PAIR TO THIS INSTANCE IS `workbench-chat-turn-v2-success.example.yaml`, +# which carries NO `context_packet` at all and is still valid: that is what makes +# this release additive, and it is why absence must never be read as `full`. A +# record without the key means its producer predates contract-v1.40; a record +# with `posture: full` means the packet was assembled full and someone checked. +# Both are conformant, and a consumer must be able to tell them apart — which it +# can, because they are different bytes. +{ + "schema_version": 1, + "kind": "workbench-chat-turn-v2-success", + "client_turn_id": "turn-v2-0004", + "assistant_turn_id": "srv-v2-0004", + "model_id": "local-authoring-1", + "selected_model": { + "requested_model_id": "local-authoring-1", + "routing_rule": false, + "data_handling": "Local process only; no content leaves this machine." + }, + "context_packet": { + "posture": "full" + }, + "bound_buffer": "ideation/staging/demo-topic/note.md", + "observed_hashes": { + "outline": "74d2e440df9974e0c1d82412165e497bb95611142b3ed1235c0bd74b3c2277ba", + "ideation/staging/demo-topic/note.md": "9859eb7d779f1dee12579980045bb94ecbc2cc487d8160aa239bc1bfe3416825" + }, + "assistant_prose": "The staged set has two notes that bear on the boundary the outline claims; both are behind this answer.", + "proposals": [] +} diff --git a/tests/fixtures/spec-examples/workbench-chat-turn-v2-loaded-set.example.yaml b/tests/fixtures/spec-examples/workbench-chat-turn-v2-loaded-set.example.yaml new file mode 100644 index 00000000..bace8815 --- /dev/null +++ b/tests/fixtures/spec-examples/workbench-chat-turn-v2-loaded-set.example.yaml @@ -0,0 +1,58 @@ +# VALID widened request (contract-v1.34, add-doxbench-editing-phase-b design +# D15): the outline plus TWO document buffers — one loaded by path, one the +# reserved not-yet-created slot — and a DECLARED `bound_buffer` naming the +# loaded one. Binding says what the chat is working ON; grounding is unchanged +# and still carries every loaded buffer, which is why all three ride the request +# while only one is bound. +{ + "schema_version": 1, + "kind": "workbench-chat-turn-v2", + "client_turn_id": "turn-v2-0001", + "scope": { + "repository": "openxFactory", + "ref": "doxbench/session/demo-topic", + "tile_kind": "staged", + "tile_id": "demo-topic" + }, + "bound_buffer": "ideation/staging/demo-topic/note.md", + "working_subject": "Clarify the acceptance boundary", + "message": "Does the note still say what the outline claims?", + "model_id": "local-authoring-1", + "last_assistant_turn_id": null, + "transcript": [], + "buffers": [ + { + "kind": "outline", + "repository": "openxFactory", + "path": "ideation/staging/demo-topic/demo-topic.md", + "base_ref": "doxbench/session/demo-topic", + "base_revision": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "base_hash": "74d2e440df9974e0c1d82412165e497bb95611142b3ed1235c0bd74b3c2277ba", + "content_hash": "74d2e440df9974e0c1d82412165e497bb95611142b3ed1235c0bd74b3c2277ba", + "content": "# Demo topic\n\nOutline line.\n", + "dirty": false + }, + { + "kind": "document", + "repository": "openxFactory", + "path": "ideation/staging/demo-topic/note.md", + "base_ref": "doxbench/session/demo-topic", + "base_revision": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "base_hash": "9859eb7d779f1dee12579980045bb94ecbc2cc487d8160aa239bc1bfe3416825", + "content_hash": "9859eb7d779f1dee12579980045bb94ecbc2cc487d8160aa239bc1bfe3416825", + "content": "# Note\n\nBody.\n", + "dirty": false + }, + { + "kind": "document", + "repository": "openxFactory", + "path": null, + "base_ref": "doxbench/session/demo-topic", + "base_revision": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "base_hash": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "content_hash": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "content": "", + "dirty": false + } + ] +} diff --git a/tests/fixtures/spec-examples/workbench-chat-turn-v2-provider-retry.example.yaml b/tests/fixtures/spec-examples/workbench-chat-turn-v2-provider-retry.example.yaml new file mode 100644 index 00000000..8c2733a4 --- /dev/null +++ b/tests/fixtures/spec-examples/workbench-chat-turn-v2-provider-retry.example.yaml @@ -0,0 +1,49 @@ +# VALID widened success (contract-v1.45): the turn that COST TWO PAID PROVIDER +# CALLS, and says so. +# +# Brett ruled on 2026-08-26 that when a minted token expires part-way through a +# turn the console re-mints and retries ONCE, "with the re-mint and the paid +# retry VISIBLY RECORDED in the turn record" — because a second paid call the +# human cannot see is exactly the decision that ruling was made to avoid. Three +# server-side records of it already existed (the port's content-free mint +# ledger, the console's own stderr notice, and the broker's audit trail +# correlated by `--retry-of`) and the browser could read NONE of them. This key +# is how the fact reaches the person paying for it. +# +# WHAT IT CARRIES AND WHAT IT REFUSES TO: that it happened, that it happened at +# most once (the ruling's bound — a second expiry in one turn refuses instead of +# buying a third call, and that turn produces a FAILURE envelope, never this +# one), and the re-mint's audit reference. Never the token, never a prefix or a +# hash of it, never a provider status and never the provider's words. +# +# THE PAIR TO THIS INSTANCE is every other v2 success example, which carries no +# `provider_retry` at all and stays valid: that is what makes contract-v1.45 +# additive. Absence means "nothing to report, or a producer older than v1.45" — +# never `retried: false`, which this shape cannot even express. +{ + "schema_version": 1, + "kind": "workbench-chat-turn-v2-success", + "client_turn_id": "turn-v2-0009", + "assistant_turn_id": "srv-v2-0009", + "model_id": "authoring-model", + "selected_model": { + "requested_model_id": "authoring-model", + "routing_rule": false, + "data_handling": "leaves this host: a hosted provider reached with a short-lived token the credential broker minted" + }, + "context_packet": { + "posture": "full" + }, + "provider_retry": { + "retried": true, + "at_most_once": true, + "audit_ref": "openprofiler-audit-7b21ee08" + }, + "bound_buffer": "ideation/staging/demo-topic/note.md", + "observed_hashes": { + "outline": "74d2e440df9974e0c1d82412165e497bb95611142b3ed1235c0bd74b3c2277ba", + "ideation/staging/demo-topic/note.md": "9859eb7d779f1dee12579980045bb94ecbc2cc487d8160aa239bc1bfe3416825" + }, + "assistant_prose": "The boundary the outline claims is narrower than the two staged notes support; both are behind this answer.", + "proposals": [] +} diff --git a/tests/fixtures/spec-examples/workbench-chat-turn-v2-reduced-context.example.yaml b/tests/fixtures/spec-examples/workbench-chat-turn-v2-reduced-context.example.yaml new file mode 100644 index 00000000..6bb621d4 --- /dev/null +++ b/tests/fixtures/spec-examples/workbench-chat-turn-v2-reduced-context.example.yaml @@ -0,0 +1,33 @@ +# VALID widened success (contract-v1.40): the DEGRADED POSTURE, on the wire. +# The staged-set knowledge service was unavailable, so the turn ran on the +# declared reduced packet — the selected thread and the loaded buffers, no +# corpus evidence — and it SUCCEEDED, which is the ratified requirement's "MUST +# NOT ... make the editors unusable" half. What contract-v1.40 adds is the other +# half: the posture is STATED where a reader and a human can consult it, and the +# reason says in as many words that nothing unbounded was substituted and no rail +# was bypassed. The reason is carried VERBATIM from the packet, never +# re-derived, and this instance's is the shipped +# `doxbench_packet.REDUCED_NO_KNOWLEDGE_SERVICE` text. +{ + "schema_version": 1, + "kind": "workbench-chat-turn-v2-success", + "client_turn_id": "turn-v2-0003", + "assistant_turn_id": "srv-v2-0003", + "model_id": "local-authoring-1", + "selected_model": { + "requested_model_id": "local-authoring-1", + "routing_rule": false, + "data_handling": "Local process only; no content leaves this machine." + }, + "context_packet": { + "posture": "reduced", + "reduced_reason": "the staged-set knowledge service is unavailable, so this packet carries the selected thread and the loaded buffers only, with NO corpus evidence; no unbounded context was substituted and no rail was bypassed to reach a provider" + }, + "bound_buffer": "ideation/staging/demo-topic/note.md", + "observed_hashes": { + "outline": "74d2e440df9974e0c1d82412165e497bb95611142b3ed1235c0bd74b3c2277ba", + "ideation/staging/demo-topic/note.md": "9859eb7d779f1dee12579980045bb94ecbc2cc487d8160aa239bc1bfe3416825" + }, + "assistant_prose": "Working from the note and the outline alone this turn: the staged-set index was not reachable, so nothing from the wider corpus is behind this answer.", + "proposals": [] +} diff --git a/tests/fixtures/spec-examples/workbench-chat-turn-v2-success.example.yaml b/tests/fixtures/spec-examples/workbench-chat-turn-v2-success.example.yaml new file mode 100644 index 00000000..b05a432c --- /dev/null +++ b/tests/fixtures/spec-examples/workbench-chat-turn-v2-success.example.yaml @@ -0,0 +1,42 @@ +# VALID widened success (contract-v1.34): the turn's durable RECORD. It names +# the DECLARED bound buffer — carried on the wire, never derived from which +# document happened to be supplied — states every buffer's observed identity BY +# KEY, targets its one proposal at a buffer key, and carries the selected-model +# metadata beside the model that answered. Here the human chose the `auto` +# ROUTING RULE, so `selected_model.requested_model_id` and `model_id` differ and +# `routing_rule` says why. +# +# AND IT CARRIES NO `context_packet`, DELIBERATELY, since contract-v1.40: this +# instance is the pre-release record shape, unchanged byte for byte, and its +# continued validity IS the additive claim. Absence here is NOT a `full` +# posture — it is a producer that predates v1.40 and states no posture at all. +# See `workbench-chat-turn-v2-full-context.example.yaml` for a producer that +# does, and `workbench-chat-turn-v2-reduced-context.example.yaml` for the +# degraded one. +{ + "schema_version": 1, + "kind": "workbench-chat-turn-v2-success", + "client_turn_id": "turn-v2-0001", + "assistant_turn_id": "srv-v2-0001", + "model_id": "local-authoring-1", + "selected_model": { + "requested_model_id": "auto", + "routing_rule": true, + "data_handling": "Local process only; no content leaves this machine." + }, + "bound_buffer": "ideation/staging/demo-topic/note.md", + "observed_hashes": { + "outline": "74d2e440df9974e0c1d82412165e497bb95611142b3ed1235c0bd74b3c2277ba", + "ideation/staging/demo-topic/note.md": "9859eb7d779f1dee12579980045bb94ecbc2cc487d8160aa239bc1bfe3416825", + "document": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + }, + "assistant_prose": "The note omits the boundary the outline states.", + "proposals": [ + { + "target": "ideation/staging/demo-topic/note.md", + "base_hash": "9859eb7d779f1dee12579980045bb94ecbc2cc487d8160aa239bc1bfe3416825", + "summary": "Names the boundary the outline already claims.", + "content": "# Note\n\nBody.\n\nBoundary: explicit.\n" + } + ] +} diff --git a/tests/fixtures/spec-examples/workbench-model-catalog-empty.example.yaml b/tests/fixtures/spec-examples/workbench-model-catalog-empty.example.yaml new file mode 100644 index 00000000..635ec974 --- /dev/null +++ b/tests/fixtures/spec-examples/workbench-model-catalog-empty.example.yaml @@ -0,0 +1,6 @@ +# VALID: the editor-only posture — zero approved models is SUCCESS (FR-025). +{ + "schema_version": 1, + "kind": "workbench-model-catalog", + "models": [] +} diff --git a/tests/fixtures/spec-examples/workbench-model-catalog-hosted-zero-retention.example.yaml b/tests/fixtures/spec-examples/workbench-model-catalog-hosted-zero-retention.example.yaml new file mode 100644 index 00000000..ed8d6325 --- /dev/null +++ b/tests/fixtures/spec-examples/workbench-model-catalog-hosted-zero-retention.example.yaml @@ -0,0 +1,16 @@ +# VALID: an explicitly enabled zero-retention hosted model with NARROW limits. +{ + "schema_version": 1, + "kind": "workbench-model-catalog", + "models": [ + { + "model_id": "hosted-zr-1", + "label": "Hosted zero-retention model (explicitly enabled)", + "provider_class": "hosted-zero-retention", + "available": true, + "input_limit_bytes": 2048, + "output_limit_bytes": 8192, + "data_handling": "Zero retention; content leaves the tenant boundary for inference only." + } + ] +} diff --git a/tests/fixtures/spec-examples/workbench-model-catalog-local.example.yaml b/tests/fixtures/spec-examples/workbench-model-catalog-local.example.yaml new file mode 100644 index 00000000..b84d093c --- /dev/null +++ b/tests/fixtures/spec-examples/workbench-model-catalog-local.example.yaml @@ -0,0 +1,25 @@ +# VALID: one approved local/on-tenant model, the seven required base fields +# only — no routing declaration, which is the shape every producer that +# predates contract-v1.38 emits and which stays valid unchanged. +# +# It is also the ABSENCE case for contract-v2.2's `modalities`: this entry +# declares no modalities and is valid exactly as it was. Its silence is not a +# claim in either direction — it says the producer predates the field, not that +# the model rejects images — and a reader treats it as text-only FOR ROUTING +# while recording that no declaration was made. The declaring counterpart is +# `workbench-model-catalog-multimodal.example.yaml`. +{ + "schema_version": 1, + "kind": "workbench-model-catalog", + "models": [ + { + "model_id": "local-authoring-1", + "label": "Approved authoring model", + "provider_class": "on-tenant", + "available": true, + "input_limit_bytes": 800000, + "output_limit_bytes": 900000, + "data_handling": "Processed in the approved tenant boundary; no retention." + } + ] +} diff --git a/tests/fixtures/spec-examples/workbench-model-catalog-multimodal.example.yaml b/tests/fixtures/spec-examples/workbench-model-catalog-multimodal.example.yaml new file mode 100644 index 00000000..0446bbaf --- /dev/null +++ b/tests/fixtures/spec-examples/workbench-model-catalog-multimodal.example.yaml @@ -0,0 +1,27 @@ +# VALID: one approved model that DECLARES the input modalities it accepts +# (contract-v2.2). The set is non-empty, carries no repeat, and contains +# `text` — a chat turn always carries text, so a model that could not accept +# it would not be routable here at all. +# +# This is the POSITIVE half of the closed vocabulary. Its negatives are +# `negative/workbench-model-catalog-modality-outside-the-vocabulary`, +# `negative/workbench-model-catalog-modality-image-only` and +# `negative/workbench-model-catalog-modality-empty-set`. The ABSENCE case — an +# entry that declares nothing and stays valid — is +# `workbench-model-catalog-local.example.yaml`. +{ + "schema_version": 1, + "kind": "workbench-model-catalog", + "models": [ + { + "model_id": "local-vision-1", + "label": "Approved authoring model (accepts images)", + "provider_class": "on-tenant", + "available": true, + "input_limit_bytes": 800000, + "output_limit_bytes": 900000, + "data_handling": "Processed in the approved tenant boundary; no retention.", + "modalities": ["text", "image"] + } + ] +} diff --git a/tests/fixtures/spec-examples/workbench-model-catalog-routing-rule-wider-than-a-non-resolved-member.example.yaml b/tests/fixtures/spec-examples/workbench-model-catalog-routing-rule-wider-than-a-non-resolved-member.example.yaml new file mode 100644 index 00000000..2a1fc612 --- /dev/null +++ b/tests/fixtures/spec-examples/workbench-model-catalog-routing-rule-wider-than-a-non-resolved-member.example.yaml @@ -0,0 +1,52 @@ +# VALID (contract-v1.38, rule 5' — Brett's ruling 2026-08-21): the rule declares +# an input budget of 800000, which EXCEEDS that of `fit-narrow-spare` (2048), a +# model it may route to. That is lawful, and it is the POINT of the ruling: the +# bound is the RESOLVED model's alone. +# +# `fit-wide-resolved` is what answers, and it accepts 800000, so the menu's +# declared limits are honoured by the model that actually serves the turn. The +# narrow member takes no turn while the rule resolves elsewhere, so capping +# against it would constrain a promise nobody can call in — and would bake in +# semantics that contradict the sanctioned per-turn fit-aware router staged as +# `ideation/staging/doxchat-auto-fit-routing/`, under which a rule's ceiling is +# the widest thing it can serve rather than the narrowest. +# +# The first shipped form of this rule (a MINIMUM over `routes_to`) would have +# REFUSED this catalog. Its mirror-image negative is +# `negative/workbench-model-catalog-routing-rule-wider-than-its-resolution`. +{ + "schema_version": 1, + "kind": "workbench-model-catalog", + "models": [ + { + "model_id": "auto-fit", + "label": "Automatic (routes by role)", + "provider_class": "routing-rule", + "available": true, + "input_limit_bytes": 800000, + "output_limit_bytes": 900000, + "data_handling": "Routes by role. / Processed in the approved tenant boundary; no retention. / Zero retention; content leaves the tenant boundary for inference only.", + "routing_rule": true, + "routes_to": ["fit-wide-resolved", "fit-narrow-spare"], + "resolved_model_id": "fit-wide-resolved" + }, + { + "model_id": "fit-wide-resolved", + "label": "Approved authoring model (the resolution)", + "provider_class": "on-tenant", + "available": true, + "input_limit_bytes": 800000, + "output_limit_bytes": 900000, + "data_handling": "Processed in the approved tenant boundary; no retention." + }, + { + "model_id": "fit-narrow-spare", + "label": "Hosted zero-retention model (routable, narrower, not resolved)", + "provider_class": "hosted-zero-retention", + "available": true, + "input_limit_bytes": 2048, + "output_limit_bytes": 8192, + "data_handling": "Zero retention; content leaves the tenant boundary for inference only." + } + ] +} diff --git a/tests/fixtures/spec-examples/workbench-model-catalog-routing-rule.example.yaml b/tests/fixtures/spec-examples/workbench-model-catalog-routing-rule.example.yaml new file mode 100644 index 00000000..8b03418e --- /dev/null +++ b/tests/fixtures/spec-examples/workbench-model-catalog-routing-rule.example.yaml @@ -0,0 +1,49 @@ +# VALID (contract-v1.38): an `auto` ROUTING RULE beside the two approved models +# it may route to. Its own badge CARRIES BOTH of theirs as SEGMENTS (the ratified +# "carry the handling badge of every model it may route to"), it resolves to one +# of them, and that one is available. +# +# Its declared limits (2048/8192) are bounded by `routed-on-tenant-1`, THE MODEL +# IT RESOLVES TO (rule 5'), which accepts 800000/900000 — so the rule sits far +# under its bound. They happen to EQUAL the limits of `routed-hosted-zr-1`, the +# member it does NOT resolve to, which is also the narrowest member; so the old +# min-over-`routes_to` form accepts this catalog too. THIS INSTANCE THEREFORE +# DOES NOT DISCRIMINATE rule 5 from rule 5'. The one that does is +# `workbench-model-catalog-routing-rule-wider-than-a-non-resolved-member`, where +# the rule is WIDER than a non-resolved member and lawful only under 5'. +{ + "schema_version": 1, + "kind": "workbench-model-catalog", + "models": [ + { + "model_id": "auto", + "label": "Automatic (routes by role)", + "provider_class": "routing-rule", + "available": true, + "input_limit_bytes": 2048, + "output_limit_bytes": 8192, + "data_handling": "Routes by role. / Processed in the approved tenant boundary; no retention. / Zero retention; content leaves the tenant boundary for inference only.", + "routing_rule": true, + "routes_to": ["routed-on-tenant-1", "routed-hosted-zr-1"], + "resolved_model_id": "routed-on-tenant-1" + }, + { + "model_id": "routed-on-tenant-1", + "label": "Approved authoring model (routable)", + "provider_class": "on-tenant", + "available": true, + "input_limit_bytes": 800000, + "output_limit_bytes": 900000, + "data_handling": "Processed in the approved tenant boundary; no retention." + }, + { + "model_id": "routed-hosted-zr-1", + "label": "Hosted zero-retention model (routable)", + "provider_class": "hosted-zero-retention", + "available": true, + "input_limit_bytes": 2048, + "output_limit_bytes": 8192, + "data_handling": "Zero retention; content leaves the tenant boundary for inference only." + } + ] +} diff --git a/tests/test_validator.py b/tests/test_validator.py new file mode 100644 index 00000000..5c0f99b5 --- /dev/null +++ b/tests/test_validator.py @@ -0,0 +1,552 @@ +"""openDox's own validator, `opendox.validator` (plan 034's T057). + +`tests/test_validator_input_set.py` holds WHAT the validator reads: its four +packaged copies, each proved before it is read, and nothing else. This module +holds HOW it judges an instance against them. + +THE CORPUS. `tests/fixtures/spec-examples/` is openDox-spec's own examples, +copied byte for byte from openDox-spec#16 at `cd49eb25` +(`examples/ideation-dashboard/`, T053). They are every positive example of the +four kinds, and the neutral snapshot contract's 32 negatives, one per rule. +Each negative's `# expected_failure:` line names the rule it breaks, so the +corpus asks the validator for the rule itself, not for a wording it guesses at. + +WHAT IT HOLDS. + +1. THE NEUTRAL CONTRACT'S RULES, ALL 32. Each negative is refused for exactly + the rule it names, at one place, and its report names that rule as + `[]`. Each positive is refused for nothing. The contract's catalog + and this module's reference rules name the same seven, and every + subschema that can fail names a catalogued shape rule, so no refusal of + the neutral kind is left without an identifier. +2. THE OTHER THREE KINDS, STRUCTURALLY. Every positive example of theirs + validates, and each chat-turn wire kind is judged against its own + envelope. +3. THE EVALUATOR'S SEMANTICS, keyword by keyword, over small schemas of its + own: JSON equality, the date-time format, the applicators, and where a + violation is reported. +4. FAIL CLOSED. A copy that uses what this module does not evaluate is refused + when its validator is built, never evaluated with a keyword left out. +5. jsonschema's SHAPE, FOR THE doxBench SEAM. `serve_workbench`'s two readers + of the seam, run over `validators()`, judge the chat turn as they judge it + over openxFactory's jsonschema validators, so plan 034's T085 can register + them. +6. OVER openDox's OWN PROJECTION. T050's fixture, projected by openDox's own + generator, breaks no rule, and T051's malformed fixture breaks exactly its + `EXPECTED_RULE`. This is F7.2's judgment, in process, and T058 wires it + into the generate verbs. +7. NO REACH. The validator imports, and validates, with every sibling blocked + and no third-party module but PyYAML. + +A CREATED file: no carve-manifest row (RULED OQ-C). +""" + +from __future__ import annotations + +import json +import os +import re +import shutil +import subprocess +import sys +import textwrap +from pathlib import Path +from typing import Any + +import pytest +import yaml + +from opendox import contracts +from opendox import corpus_adapter +from opendox import default_generator +from opendox import domain_profile +from opendox import generator_seam as gs +from opendox import serve_workbench +from opendox import validator as V +from opendox.runtime import local_git_adapter as lga + +ROOT = Path(__file__).resolve().parents[1] +SRC = ROOT / "src" +FIXTURES = ROOT / "tests" / "fixtures" +EXAMPLES = FIXTURES / "spec-examples" +POSITIVE = sorted(EXAMPLES.glob("*.example.yaml")) +NEGATIVE = sorted((EXAMPLES / "negative").glob("opendox-snapshot-*.negative.yaml")) +PLAIN = FIXTURES / "plain-documents" # T050, openDox-code#53 +MALFORMED = FIXTURES / "malformed" # T051, openDox-code#56 +SNAPSHOT = gs.NEUTRAL_SNAPSHOT_KIND + +#: The four packages a neutral openDox must import without (#1144's F2.1). +SIBLINGS = ("openxdox", "ideation_dashboard", "doc_health", + "corpus_adapter_openxfactory") + + +def _read(path: Path) -> Any: + return yaml.safe_load(path.read_text(encoding="utf-8")) + + +def _expected_failure(path: Path) -> str: + named = re.findall(r"^# expected_failure: (\S+)\s*$", + path.read_text(encoding="utf-8"), re.M) + assert len(named) == 1, f"{path.name} must name exactly one expected_failure: {named}" + return named[0] + + +def _contract() -> dict[str, Any]: + return contracts.load("opendox-snapshot") + + +def _built(schema: dict[str, Any], *, copy_id: str = "a-test-schema", + pointer: str = "") -> V.KindValidator: + """A validator over a schema of the test's own, in the copies' dialect.""" + return V.KindValidator("a-test-kind", copy_id, pointer, + {"$schema": V.DIALECT, **schema}, "0" * 64) + + +def _found(schema: dict[str, Any], instance: Any) -> set[tuple[str, str]]: + return {(v.keyword, v.where) for v in _built(schema).iter_errors(instance)} + + +# --------------------------------------------------------------------------- +# 1. the neutral contract's 32 rules +# --------------------------------------------------------------------------- + +def test_the_corpus_is_one_negative_per_rule_and_the_two_positives() -> None: + catalog = [rule["id"] for rule in _contract()["x-rules"]] + assert len(catalog) == 32 + assert sorted(_expected_failure(path) for path in NEGATIVE) == sorted(catalog) + assert [p.name for p in POSITIVE if p.name.startswith("opendox-snapshot-")] == [ + "opendox-snapshot-no-front-matter.example.yaml", + "opendox-snapshot-six-stations.example.yaml"] + + +@pytest.mark.parametrize("path", NEGATIVE, ids=lambda p: p.name) +def test_a_negative_is_refused_for_its_rule_alone_and_its_report_names_it( + path: Path) -> None: + """Validated against the neutral contract, as openDox's generator declares + it writes, each negative breaks exactly the rule its header names, at one + place, and every line of the report opens with `[]`.""" + expected = _expected_failure(path) + found = V.validate(_read(path), kind=SNAPSHOT) + assert {v.rule for v in found} == {expected}, V.report(found) + assert len({v.where for v in found}) == 1, V.report(found) + assert all(line.startswith(f"[{expected}] ") for line in V.report(found)) + + +@pytest.mark.parametrize("path", [p for p in POSITIVE if p.name.startswith("opendox-")], + ids=lambda p: p.name) +def test_a_positive_breaks_no_rule(path: Path) -> None: + assert V.validate(_read(path)) == [] + assert V.validate(_read(path), kind=SNAPSHOT) == [] + + +def test_the_catalog_and_the_reference_rules_are_one_set() -> None: + """25 shape rules and 7 reference rules. This module implements exactly + the seven the catalog declares, and in the catalog's order.""" + catalog = _contract()["x-rules"] + reference = [rule["id"] for rule in catalog if rule["class"] == "reference"] + shape = [rule["id"] for rule in catalog if rule["class"] == "shape"] + assert (len(shape), len(reference)) == (25, 7) + assert list(V.REFERENCE_RULES["opendox-snapshot"]) == reference + assert set(V.REFERENCE_RULES) == {"opendox-snapshot"} + + +def test_every_subschema_of_the_neutral_contract_that_can_fail_names_its_rule() -> None: + """So every refusal of the neutral kind carries the contract's identifier, + and none falls back to a bare keyword.""" + catalog = {rule["id"]: rule["class"] for rule in _contract()["x-rules"]} + can_fail = V._ASSERTING | {"anyOf", "oneOf", "not", "contains"} + named = set() + for at, node in V._subschemas(_contract()): + if "if" in at.split("/"): + continue # an `if` is a test and never reports + if can_fail & set(node): + assert "x-rule" in node, f"{at or ''} can fail and names no rule" + if "x-rule" in node: + assert catalog.get(node["x-rule"]) == "shape", (at, node["x-rule"]) + named.add(node["x-rule"]) + assert named == {rule for rule, cls in catalog.items() if cls == "shape"} + + +def test_the_governed_kind_is_refused_as_a_rule_or_as_unknown() -> None: + """A snapshot of the consumer's governed kind is not openDox's. Asked for + the neutral contract, the validator refuses it for + `kind-is-opendox-snapshot`; asked to pick a contract by the instance's own + kind, it has none, and says so rather than answering an empty list.""" + governed = _read(EXAMPLES / "negative" / + "opendox-snapshot-kind-is-opendox-snapshot.negative.yaml") + assert governed["kind"] == "ideation-dashboard-snapshot" + assert {v.rule for v in V.validate(governed, kind=SNAPSHOT)} == {"kind-is-opendox-snapshot"} + with pytest.raises(V.UnknownKind): + V.validate(governed) + for not_an_instance in ([], "opendox-snapshot", None, {"kind": 1}): + with pytest.raises(V.UnknownKind): + V.validate(not_an_instance) + with pytest.raises(V.UnknownKind): + V.validate({}, kind="ideation-dashboard-snapshot") + + +def test_a_report_line_names_the_rule_the_place_and_what_was_found() -> None: + snap = _read(EXAMPLES / "opendox-snapshot-no-front-matter.example.yaml") + snap["documents"][0]["title"] = "" + snap["clusters"][0]["document_edges"][0]["document"] = "nowhere.md" + lines = V.report(V.validate(snap, kind=SNAPSHOT)) + assert lines == [ + "[title-and-summary-are-text] /documents/0/title: '' is shorter than 1", + "[edge-names-a-document] /clusters/0/document_edges/0/document: " + "no document has the id 'nowhere.md'", + ], lines + + +def test_a_pointer_escapes_its_keys_and_the_root_is_named() -> None: + violation = V.Violation("r", ("a/b", "c~d", 0), "type", "detail") + assert violation.where == "/a~1b/c~0d/0" + assert V.Violation("envelope-keys", (), "type", "x").line() == "[envelope-keys] : x" + + +# --------------------------------------------------------------------------- +# 2. the other three kinds, structurally +# --------------------------------------------------------------------------- + +@pytest.mark.parametrize("path", [p for p in POSITIVE if not p.name.startswith("opendox-")], + ids=lambda p: p.name) +def test_every_positive_example_of_the_other_kinds_validates(path: Path) -> None: + instance = _read(path) + assert instance["kind"] in V.KIND_ENTRIES + assert V.validate(instance) == [], V.report(V.validate(instance)) + + +def test_each_chat_turn_kind_is_judged_against_its_own_envelope() -> None: + """A success envelope is valid as a success and not as a request: each + wire kind is its own envelope, as openxFactory's `CHAT_TURN_DEFS` names + them, and not the chat-turn file's three-way `oneOf`.""" + success = _read(EXAMPLES / "workbench-chat-turn-v2-success.example.yaml") + assert V.validate(success, kind="workbench-chat-turn-v2-success") == [] + as_request = V.validate(success, kind="workbench-chat-turn-v2") + assert ("const", "/kind") in {(v.keyword, v.where) for v in as_request} + assert "oneOf" not in {v.keyword for v in as_request} + + +# --------------------------------------------------------------------------- +# 3. the evaluator, keyword by keyword +# --------------------------------------------------------------------------- + +@pytest.mark.parametrize("types, value, ok", [ + ("integer", 2, True), ("integer", 2.0, True), ("integer", 2.5, False), + ("integer", True, False), ("number", 1.5, True), ("number", False, False), + ("boolean", 0, False), ("null", None, True), ("string", None, False), + (["string", "null"], None, True), ("array", {}, False), ("object", [], False), +], ids=repr) +def test_type_is_json_type(types: Any, value: Any, ok: bool) -> None: + """JSON's types: `true` is not a number, and 2.0 is the integer 2.""" + assert (not _found({"type": types}, value)) is ok + + +@pytest.mark.parametrize("schema, value, ok", [ + ({"const": 1}, 1.0, True), ({"const": 1}, True, False), + ({"const": {"a": 1, "b": [1, 2]}}, {"b": [1.0, 2], "a": 1}, True), + ({"const": [1, 2]}, [2, 1], False), + ({"enum": [False, "x"]}, 0, False), ({"enum": [False, "x"]}, False, True), + ({"uniqueItems": True}, [1, 1.0], False), ({"uniqueItems": True}, [1, True], True), + ({"uniqueItems": True}, [{"a": 1}, {"a": 1}], False), + ({"uniqueItems": True}, [{1: "a"}, {"1": "a"}], True), +], ids=repr) +def test_const_enum_and_uniqueness_are_json_equality(schema: dict, value: Any, + ok: bool) -> None: + assert (not _found(schema, value)) is ok + + +@pytest.mark.parametrize("value, ok", [ + ("2026-09-27T12:00:00Z", True), ("2026-09-27t12:00:00.25+05:30", True), + ("2024-02-29T00:00:00Z", True), ("2000-02-29T23:59:59z", True), + ("0001-01-01T00:00:00Z", True), + ("2026-02-29T00:00:00Z", False), ("1900-02-29T00:00:00Z", False), + ("2026-04-31T00:00:00Z", False), ("2026-13-01T00:00:00Z", False), + ("0000-01-01T00:00:00Z", False), ("2016-12-31T23:59:60Z", False), + ("2026-09-27T24:00:00Z", False), ("2026-09-27T12:00:00", False), + ("2026-09-27", False), ("2026-09-27T12:00:00+0530", False), + ("２０２６-09-27T12:00:00Z", False), + ("2026-09-27T12:00:00Z\n", False), +], ids=repr) +def test_the_date_time_format_is_asserted(value: str, ok: bool) -> None: + """RFC 3339, as the consumer's validator asserts it: ASCII digits, a year + other than 0000, a day the month has, no leap second. The whole value must + match, so a trailing newline is refused, which is the one place this is + stricter than `rfc3339-validator`. `format` holds only for text.""" + assert (not _found({"format": "date-time"}, value)) is ok + assert not _found({"format": "date-time"}, 12) + + +def test_lengths_count_characters_and_a_pattern_searches() -> None: + assert _found({"minLength": 2, "maxLength": 3}, "é") == {("minLength", "")} + assert not _found({"minLength": 2, "maxLength": 3}, "éé") + assert _found({"maxLength": 3}, "abcd") == {("maxLength", "")} + # A JSON Schema pattern is not anchored: `b` is found inside `abc`. + assert not _found({"pattern": "b"}, "abc") + assert _found({"pattern": "^b"}, "abc") == {("pattern", "")} + # A length or a pattern says nothing about a value that is not text. + assert not _found({"minLength": 5, "pattern": "^x$"}, 7) + + +def test_numbers_have_bounds_and_booleans_are_not_numbers() -> None: + assert _found({"minimum": 0, "maximum": 3}, -1) == {("minimum", "")} + assert _found({"minimum": 0, "maximum": 3}, 4) == {("maximum", "")} + assert not _found({"minimum": 0, "maximum": 3}, 3.0) + assert not _found({"minimum": 1}, False) + + +def test_array_keywords() -> None: + schema = {"minItems": 1, "maxItems": 2, "items": {"type": "string"}, + "contains": {"const": "x"}} + assert _found(schema, []) == {("minItems", ""), ("contains", "")} + assert _found(schema, ["x", "y", "z"]) == {("maxItems", "")} + assert _found(schema, ["x", 1]) == {("type", "/1")} + assert _found(schema, ["y"]) == {("contains", "")} + + +def test_object_keywords() -> None: + schema = {"required": ["a"], "minProperties": 1, "maxProperties": 2, + "dependentRequired": {"b": ["c"]}, + "properties": {"a": {"type": "string"}, "b": {}, "c": {}}, + "additionalProperties": False} + assert _found(schema, {}) == {("required", ""), ("minProperties", "")} + assert _found(schema, {"a": 1}) == {("type", "/a")} + assert _found(schema, {"a": "x", "b": 1}) == {("dependentRequired", "")} + assert _found(schema, {"a": "x", "b": 1, "c": 2}) == {("maxProperties", "")} + assert _found(schema, {"a": "x", "z": 1}) == {("additionalProperties", "")} + # An additionalProperties SCHEMA judges each unnamed property where it is. + assert _found({"properties": {"a": {}}, "additionalProperties": {"type": "integer"}}, + {"a": "x", "z": "y"}) == {("type", "/z")} + + +def test_property_names_are_judged_at_the_object() -> None: + """jsonschema reports a bad property NAME at the object, with the keyword + that refused the name, and so does this module.""" + found = _built({"properties": {"o": {"propertyNames": {"pattern": "^[a-z]+$"}, + "maxProperties": 5}}}).violations( + {"o": {"ok": 1, "Not OK": 2}}) + assert [(v.keyword, v.where) for v in found] == [("pattern", "/o")] + assert "Not OK" in found[0].detail + + +def test_applicators() -> None: + one_of = {"oneOf": [{"type": "integer"}, {"minimum": 0}]} + assert _found(one_of, -1) == set() # integer only + assert _found(one_of, 1) == {("oneOf", "")} # both + assert _found(one_of, "x") == set() # `minimum` holds for text + assert _found({"oneOf": [{"type": "integer"}, {"type": "null"}]}, "x") == {("oneOf", "")} + any_of = {"anyOf": [{"type": "integer"}, {"type": "null"}]} + assert _found(any_of, "x") == {("anyOf", "")} and not _found(any_of, None) + assert _found({"not": {"required": ["a"]}}, {"a": 1}) == {("not", "")} + assert _found({"allOf": [{"minimum": 1}, {"maximum": 0}]}, 0.5) == { + ("minimum", ""), ("maximum", "")} + # `if` never reports; it only chooses whether `then` applies. + conditional = {"if": {"properties": {"k": {"const": "a"}}, "required": ["k"]}, + "then": {"required": ["a"]}} + assert _found(conditional, {"k": "a"}) == {("required", "")} + assert not _found(conditional, {"k": "b"}) + assert not _found(conditional, {}) + + +def test_a_reference_applies_beside_its_siblings_and_names_its_own_rule() -> None: + """Draft 2020-12: `$ref` applies BESIDE the keywords next to it, and a + violation found through the reference carries the referenced subschema's + rule.""" + built = _built({"properties": {"n": {"$ref": "#/$defs/n", "x-rule": "outer", + "maximum": 5}}, + "$defs": {"n": {"x-rule": "inner", "type": "integer"}}}) + assert {(v.rule, v.keyword) for v in built.violations({"n": 6.5})} == { + ("inner", "type"), ("outer", "maximum")} + + +def test_a_boolean_schema() -> None: + assert not _found({"properties": {"a": True}}, {"a": object()}) + assert _found({"properties": {"a": False}}, {"a": 1}) == {("false", "/a")} + + +# --------------------------------------------------------------------------- +# 4. fail closed +# --------------------------------------------------------------------------- + +@pytest.mark.parametrize("schema, says", [ + ({"patternProperties": {"^x": {}}}, "does not evaluate"), + ({"properties": {"a": {"else": {}}}}, "does not evaluate"), + ({"format": "uri"}, "does not assert"), + ({"$ref": "other.schema.yaml#/$defs/x"}, "only a reference inside the copy"), + ({"$ref": "#/$defs/missing"}, "names nothing in the copy"), + ({"$ref": "#/title", "title": "t"}, "not a schema"), + ({"pattern": "("}, "does not compile"), + ({"type": "text"}, "names the type"), + ({"x-rules": [{"id": "r", "class": "reference", "says": "?"}]}, + "implements []"), + ({"x-rules": ["not a rule"]}, "each carry an id"), +], ids=["patternProperties", "else", "a format", "a remote reference", + "a dangling reference", "a reference to text", "a bad pattern", + "an unknown type", "an unimplemented reference rule", "a malformed catalog"]) +def test_what_is_not_evaluated_is_refused_when_the_validator_is_built( + schema: dict, says: str) -> None: + with pytest.raises(V.SchemaNotEvaluable) as refused: + _built(schema) + assert says in str(refused.value) + assert isinstance(refused.value, V.ValidatorUnavailable) + + +def test_another_dialect_is_refused() -> None: + with pytest.raises(V.SchemaNotEvaluable) as refused: + V.KindValidator("k", "c", "", {"$schema": "http://json-schema.org/draft-07/schema#"}, + "0" * 64) + assert "dialect" in str(refused.value) + + +def test_a_reference_rule_the_catalog_does_not_declare_is_not_enforced() -> None: + """The neutral contract's catalog, less one reference rule, is refused: + this module implements it and the contract would not declare it.""" + contract = _contract() + contract["x-rules"] = [r for r in contract["x-rules"] if r["id"] != "ids-are-unique"] + with pytest.raises(V.SchemaNotEvaluable) as refused: + V.KindValidator(SNAPSHOT, "opendox-snapshot", "", contract, "0" * 64) + assert "ids-are-unique" in str(refused.value) + + +def test_every_copy_builds_and_is_cached_by_its_proved_digest() -> None: + first = V.validators() + assert sorted(first) == list(V.KINDS) + again = V.validators() + assert first is not again and all(first[k] is again[k] for k in V.KINDS) + for kind, built in first.items(): + copy_id, pointer = V.KIND_ENTRIES[kind] + assert (built.kind, built.copy_id, built.pointer) == (kind, copy_id, pointer) + assert built.digest == contracts.record().copy(copy_id).sha256 + + +# --------------------------------------------------------------------------- +# 5. the doxBench seam's two readers, over these validators +# --------------------------------------------------------------------------- + +def test_the_doxbench_seams_readers_judge_the_chat_turn_through_these_validators() -> None: + """`serve_workbench` reads a validator's `iter_errors()` and each error's + `validator` and `absolute_path`, as jsonschema spells them. Over + `validators()`, a valid request conforms; a request carrying only the + outline breaks the buffers' `minItems` floor and nothing else; and one + that breaks something besides is told apart from it.""" + seam = serve_workbench.WorkbenchRoutes + validators = V.validators() + request = _read(EXAMPLES / "workbench-chat-turn-v2-loaded-set.example.yaml") + kind = request["kind"] + assert seam._doxbench_wire_conforms(validators, kind, request) + assert not seam._doxbench_violation_beside_the_buffers_floor(validators, kind, request) + + outline_only = {**request, "buffers": [b for b in request["buffers"] + if b.get("kind") == "outline"]} + assert len(outline_only["buffers"]) == 1 + errors = list(validators[kind].iter_errors(outline_only)) + assert [(e.validator, list(e.absolute_path)) for e in errors] == [("minItems", ["buffers"])] + assert not seam._doxbench_wire_conforms(validators, kind, outline_only) + assert not seam._doxbench_violation_beside_the_buffers_floor(validators, kind, + outline_only) + + and_more = {**outline_only, "message": ""} + assert seam._doxbench_violation_beside_the_buffers_floor(validators, kind, and_more) + # A kind with no validator is no verdict, and no verdict is not consent. + assert not seam._doxbench_wire_conforms(validators, "workbench-chat-turn", request) + + +# --------------------------------------------------------------------------- +# 6. over openDox's own projection (F7.2's judgment, in process) +# --------------------------------------------------------------------------- + +ANCHOR_DATE = "2026-09-27T12:00:00+00:00" + + +def _git(root: Path, *args: str) -> None: + env = {k: v for k, v in os.environ.items() if not k.startswith("GIT_")} + env.update({ + "GIT_AUTHOR_NAME": "fixture", "GIT_AUTHOR_EMAIL": "fixture@example.invalid", + "GIT_COMMITTER_NAME": "fixture", "GIT_COMMITTER_EMAIL": "fixture@example.invalid", + "GIT_AUTHOR_DATE": ANCHOR_DATE, "GIT_COMMITTER_DATE": ANCHOR_DATE, + "GIT_CONFIG_GLOBAL": os.devnull, "GIT_CONFIG_SYSTEM": os.devnull, + }) + subprocess.run(["git", "-C", str(root), *args], check=True, capture_output=True, env=env) + + +def _committed_copy(tmp_path: Path, fixture: Path) -> Path: + root = tmp_path / fixture.name + shutil.copytree(fixture, root) + _git(root, "-c", "init.defaultBranch=main", "init", "-q") + _git(root, "add", "-A") + _git(root, "commit", "-qm", "fixture") + return root + + +@pytest.fixture +def _default_home(): + """openDox's default home corpus registered, and the registry put back.""" + saved = corpus_adapter._home_factory + corpus_adapter.register_home(lambda root: ( + lga.WorkingTreeCorpus(), corpus_adapter.CorpusRef(name="home", location=str(root)))) + yield + corpus_adapter._home_factory = saved + + +def test_openDox_projection_of_the_plain_fixture_breaks_no_rule( + tmp_path: Path, _default_home) -> None: + snapshot = default_generator.generate(_committed_copy(tmp_path, PLAIN), "fixture") + assert snapshot["kind"] == SNAPSHOT + assert V.validate(snapshot, kind=default_generator.GENERATOR.contract) == [] + + +def test_openDox_projection_of_the_malformed_fixture_breaks_its_expected_rule( + tmp_path: Path, _default_home) -> None: + """T051's fixture breaks exactly one rule, and the report names the rule + `EXPECTED_RULE` holds, which is what F7.2 greps the verbs' output for.""" + rule = (MALFORMED / "EXPECTED_RULE").read_text(encoding="utf-8").strip() + snapshot = default_generator.generate(_committed_copy(tmp_path, MALFORMED), "fixture") + found = V.validate(snapshot, kind=SNAPSHOT) + assert [(v.rule, v.where) for v in found] == [(rule, "/documents/1/title")], V.report(found) + assert any(rule in line for line in V.report(found)) + + +# --------------------------------------------------------------------------- +# 7. no reach +# --------------------------------------------------------------------------- + +_BLOCKED = """ +import json, sys +for name in SIBLINGS: + sys.modules[name] = None +before = set(sys.modules) +import yaml # what PyYAML brings with it (its C binding's runtime) is PyYAML's +pyyaml = sorted({m.split(".")[0] for m in set(sys.modules) - before}) +from opendox import validator +built = validator.validators() +snap = {"schema_version": 1, "kind": "opendox-snapshot", "repository": "r", + "generation": {"source_revision": "HEAD"}, "documents": [], "clusters": [], + "possibles": [], "staged_topics": [], "changes": []} +print(json.dumps({"kinds": sorted(built), "violations": validator.report(validator.validate(snap)), + "pyyaml": pyyaml, + "siblings": [n for n in SIBLINGS if sys.modules.get(n) is not None], + "new": sorted({m.split(".")[0] for m in set(sys.modules) - before})})) +""" + + +def test_the_validator_imports_and_validates_with_every_sibling_blocked() -> None: + program = (f"import sys; sys.path.insert(0, {str(SRC)!r})\n" + f"SIBLINGS = {SIBLINGS!r}\n" + textwrap.dedent(_BLOCKED)) + done = subprocess.run([sys.executable, "-c", program], capture_output=True, text=True, + cwd=str(ROOT), timeout=120) + assert done.returncode == 0, done.stderr + out = json.loads(done.stdout.strip().splitlines()[-1]) + assert out["kinds"] == list(V.KINDS) + assert out["violations"] == [] + assert "yaml" in out["pyyaml"] + third_party = sorted(set(out["new"]) - set(sys.stdlib_module_names) + - {"opendox"} - set(out["pyyaml"]) - set(SIBLINGS)) + assert third_party == [], third_party + assert out["siblings"] == [], "a sibling was imported past its block" + + +def test_the_validator_leaves_the_registries_as_it_found_them() -> None: + """Validating registers nothing: no profile, no generator, no home.""" + before = (domain_profile._registered, gs._registered, corpus_adapter._home_factory) + V.validate(_read(EXAMPLES / "opendox-snapshot-six-stations.example.yaml")) + assert (domain_profile._registered, gs._registered, + corpus_adapter._home_factory) == before diff --git a/tests/test_validator_input_set.py b/tests/test_validator_input_set.py new file mode 100644 index 00000000..06ed2db1 --- /dev/null +++ b/tests/test_validator_input_set.py @@ -0,0 +1,293 @@ +"""openDox's validator's input set, held to plan 034's T057. + +T057 realizes #1144's 7.1, 7.1a, 7.1b and 7.2, with 7.1 as T007's batch G +amends it (R1Q11 (a) and R1Q12 (a), `openxFactory#656` comment `5850003126`): + + Narrow it to openDox's own kinds: its spec leg's four, which are 7.1's + three and T053's neutral snapshot schema ... Ship the four as package + data. A test checks each copy's digest against the spec-leg commit the + openDox root pins (R1Q12 (a)) ... A test asserts that `gate-intent` and + `ideation-possibles-register` are NOT in the set (7.1b). + +Its falsifier is *"the packaged-copy digest test and the 7.1b test"*. They are +the first two cases below: + +* `test_each_packaged_copy_is_the_spec_legs_file_at_the_pinned_commit` is the + digest test; +* `test_gate_intent_and_the_possibles_register_are_not_in_the_set` is 7.1b's. + +WHAT ELSE IT HOLDS. + +1. THE SET IS EXACTLY FOUR: the record, the validator's kinds and the files on + disk all name the same four copies, and none of the six schemas outside + the set is carried. +2. PRESENCE IS NOT IDENTITY: a copy that differs from its digest, is absent, + or has no digest recorded is refused before a byte of it is parsed, and a + validator is never built over it. A record that cannot hold every copy to a + digest is refused as a whole. +3. THE KINDS ARE THE COPIES': each kind the validator maps is the `kind` const + its entry declares, and `generator_seam.NEUTRAL_SNAPSHOT_KIND`, the + contract openDox's own generator declares (T052), is the packaged neutral + contract's `kind` (the holder's note to T057). + +A CREATED file: no carve-manifest row (RULED OQ-C). +""" + +from __future__ import annotations + +import hashlib +from pathlib import Path + +import pytest +import yaml + +from opendox import contracts +from opendox import default_generator +from opendox import generator_seam +from opendox import validator + +ROOT = Path(__file__).resolve().parents[1] +PACKAGE = ROOT / "src" / "opendox" / "contracts" + +#: openDox's own spec leg's four schemas (7.1, as batch G amends it). +THE_FOUR = ("ideation-workbench", "opendox-snapshot", "xfactory-workbench-chat-turn", + "xfactory-workbench-model-catalog") + +#: The commit the copies were taken at: openDox-spec#16's head (T053), the one +#: commit that carries all four. When T053's root step moves the openDox root's +#: spec pin to T053's landed commit, this moves with the record, in lockstep. +SPEC_COMMIT = "cd49eb253a431202b67de70b2c9ed941a72d0aa4" + +#: WHAT THE OPENDOX ROOT PINS, stated here apart from the record, so that a copy +#: and its recorded digest cannot move together unseen. The first three are +#: the digests the root's `contracts/manifest.yaml` records for them at its +#: spec pin, 8fe8c4c7 (the root's `main` at 663ac683). `SPEC_COMMIT` carries +#: those three files unchanged. The fourth is the digest T053's held root step +#: records for its new `opendox-snapshot` manifest entry (openDox-spec#16). +PINNED_BY_THE_ROOT = { + "ideation-workbench": + "d30438491119c20928fbe4e85088fc33682829eeb6558d87dafce651000faafc", + "opendox-snapshot": + "f9e3e111af1d4bd4c377c933027d81b582ae2b0a395b66f4e4621992454a584a", + "xfactory-workbench-chat-turn": + "350bfedc02696e7281a42c0bdc9a25059bf7af14d16d89d9f07018d3e691dc1d", + "xfactory-workbench-model-catalog": + "e563cc9fc6ede03dfd62537935d0ae0842617d7de46702aee6ad9026aa021635", +} + +#: The ten schemas of the family the consumer's script names +#: (`SCHEMA_FILENAMES`), by owner, as #1144's 7.1 tables them. +OPENXDOX_SPECS = ("ideation-dashboard-snapshot", "ideation-dashboard-snapshot-index", + "gate-action-record") +OPENXFACTORYS = ("ideation-possibles-register", "project-register", + "demotion-execution-receipt", "gate-intent") + + +# --------------------------------------------------------------------------- +# the falsifier +# --------------------------------------------------------------------------- + +def test_each_packaged_copy_is_the_spec_legs_file_at_the_pinned_commit() -> None: + """THE DIGEST TEST. Each copy on disk has the sha256 the record pins, the + record pins each copy at the spec leg's own path and at one spec-leg + commit, and each digest is the one the openDox root pins for that file.""" + record = contracts.record() + assert record.spec_leg == "opensoft/openDox-spec" + assert record.commit == SPEC_COMMIT, ( + f"the copies are recorded at {record.commit}, and this test holds them at " + f"{SPEC_COMMIT}. They move together, in one commit: copy the spec leg's " + "files at the commit the openDox root pins, and move both") + assert record.ids == THE_FOUR + for copy in record.copies: + on_disk = PACKAGE / "schemas" / f"{copy.id}.schema.yaml" + digest = hashlib.sha256(on_disk.read_bytes()).hexdigest() + assert copy.path == f"contracts/schemas/{copy.id}.schema.yaml" + assert copy.resource == f"schemas/{copy.id}.schema.yaml" + assert digest == copy.sha256, ( + f"{on_disk.relative_to(ROOT)} is {digest}, and the record pins " + f"{copy.sha256}. A copy is never edited in place") + assert copy.sha256 == PINNED_BY_THE_ROOT[copy.id], ( + f"the record pins {copy.id} at {copy.sha256}, and the openDox root " + f"pins {PINNED_BY_THE_ROOT[copy.id]}") + # the package reads the same bytes, through its own identity check + assert contracts.verified_bytes(copy.id) == on_disk.read_bytes() + + +def test_gate_intent_and_the_possibles_register_are_not_in_the_set() -> None: + """7.1b. `gate-intent` is an intent-plane schema that requirement 1 keeps + with openxFactory, and `ideation-possibles-register` is openxFactory's own + candidate register. Neither is a kind openDox validates, neither is a + packaged copy, neither is on disk, and no validator can be asked for + either.""" + for name in ("gate-intent", "ideation-possibles-register"): + assert name not in validator.KIND_ENTRIES + assert name not in {copy for copy, _pointer in validator.KIND_ENTRIES.values()} + assert name not in contracts.record().ids + assert not (PACKAGE / "schemas" / f"{name}.schema.yaml").exists() + with pytest.raises(validator.UnknownKind): + validator.validator_for(name) + with pytest.raises(contracts.CopyRefused): + contracts.verified_bytes(name) + + +# --------------------------------------------------------------------------- +# the set is exactly four +# --------------------------------------------------------------------------- + +def test_the_set_is_the_spec_legs_four_and_nothing_else() -> None: + """7.1: openDox validates its own spec leg's kinds. The record, the kinds' + entries and the files on disk name the same four copies. None of the + consumer's three and none of openxFactory's four is carried.""" + entries = {copy for copy, _pointer in validator.KIND_ENTRIES.values()} + on_disk = {path.name.removesuffix(".schema.yaml") + for path in (PACKAGE / "schemas").iterdir()} + assert set(THE_FOUR) == entries == on_disk == set(contracts.record().ids) + assert not (set(OPENXDOX_SPECS) | set(OPENXFACTORYS)) & on_disk + assert sorted(p.name for p in PACKAGE.iterdir() if p.name != "__pycache__") == [ + "__init__.py", "copies.yaml", "schemas"] + + +def test_each_kind_is_the_const_its_entry_declares() -> None: + """The validator's map of kinds is the copies' own: each kind's entry + declares that kind as its `kind` const. So a kind the map names and no + copy declares, or a copy's kind the map leaves out, fails here.""" + derived = {} + for copy_id in THE_FOUR: + document = contracts.load(copy_id) + envelopes = [document] if "oneOf" not in document else [ + validator._at_pointer(document, branch["$ref"][1:]) + for branch in document["oneOf"]] + for envelope in envelopes: + derived[envelope["properties"]["kind"]["const"]] = copy_id + assert {kind: copy for kind, (copy, _pointer) in validator.KIND_ENTRIES.items()} == derived + assert validator.KINDS == tuple(sorted(derived)) + for kind, (copy_id, pointer) in validator.KIND_ENTRIES.items(): + entry = validator._at_pointer(contracts.load(copy_id), pointer) + assert entry["properties"]["kind"]["const"] == kind + + +def test_the_neutral_snapshot_kind_is_the_packaged_contracts_kind() -> None: + """The contract openDox's own generator declares (T052's + `NEUTRAL_SNAPSHOT_KIND`) is the packaged neutral contract's `kind` const, + so what the generator writes is what this validator validates it + against.""" + snapshot_contract = contracts.load("opendox-snapshot") + kind = snapshot_contract["properties"]["kind"]["const"] + assert generator_seam.NEUTRAL_SNAPSHOT_KIND == kind == "opendox-snapshot" + assert default_generator.GENERATOR.contract == kind + assert validator.KIND_ENTRIES[kind] == ("opendox-snapshot", "") + assert validator.validator_for(kind).copy_id == "opendox-snapshot" + + +# --------------------------------------------------------------------------- +# presence is not identity +# --------------------------------------------------------------------------- + +def _serve(monkeypatch: pytest.MonkeyPatch, replaced: dict[str, bytes | None]) -> None: + """Answer the package's reads from `replaced` where it names the file + (None is an absent file), and from the package itself otherwise.""" + real = contracts._read_package_file + + def read(name: str) -> bytes: + if name in replaced: + if replaced[name] is None: + raise contracts.CopyRefused(f"opendox.contracts has no {name}") + return replaced[name] + return real(name) + + monkeypatch.setattr(contracts, "_read_package_file", read) + + +def _record_with(**changes) -> bytes: + data = yaml.safe_load((PACKAGE / "copies.yaml").read_text(encoding="utf-8")) + data.update(changes) + return yaml.safe_dump(data, sort_keys=False).encode() + + +def test_a_changed_copy_is_refused_before_a_byte_of_it_is_parsed( + monkeypatch: pytest.MonkeyPatch) -> None: + """One byte added to the neutral contract's copy is refused by the + identity check, naming both digests, before YAML is asked to read it; and + the validator reports itself unavailable rather than validating.""" + changed = (PACKAGE / "schemas" / "opendox-snapshot.schema.yaml").read_bytes() + b"\n" + _serve(monkeypatch, {"schemas/opendox-snapshot.schema.yaml": changed}) + parsed: list = [] + real_load = yaml.safe_load + + def spy(stream, *args, **kwargs): + parsed.append(stream) + return real_load(stream, *args, **kwargs) + + monkeypatch.setattr(yaml, "safe_load", spy) + with pytest.raises(contracts.CopyRefused) as refused: + contracts.load("opendox-snapshot") + assert PINNED_BY_THE_ROOT["opendox-snapshot"] in str(refused.value) + assert hashlib.sha256(changed).hexdigest() in str(refused.value) + assert parsed, "the record, which is read first, was not read through YAML" + assert changed not in parsed, ( + "the changed copy was parsed before its identity was proved") + + +def test_a_changed_copy_leaves_the_validator_unavailable( + monkeypatch: pytest.MonkeyPatch) -> None: + """The validator proves the copy on every call, so a copy that changes + after a validator was built and cached is refused on the next call, not + answered from the cache.""" + assert validator.validator_for("opendox-snapshot").is_valid( + yaml.safe_load((ROOT / "tests" / "fixtures" / "spec-examples" / + "opendox-snapshot-no-front-matter.example.yaml").read_text())) + real = (PACKAGE / "schemas" / "opendox-snapshot.schema.yaml").read_bytes() + _serve(monkeypatch, {"schemas/opendox-snapshot.schema.yaml": real.replace( + b'"minLength": 1', b'"minLength": 0', 1)}) + with pytest.raises(validator.ValidatorUnavailable) as unavailable: + validator.validator_for("opendox-snapshot") + assert "is not the file the record pins" in str(unavailable.value) + with pytest.raises(validator.ValidatorUnavailable): + validator.validators() + + +def test_an_absent_copy_is_refused(monkeypatch: pytest.MonkeyPatch) -> None: + _serve(monkeypatch, {"schemas/ideation-workbench.schema.yaml": None}) + with pytest.raises(contracts.CopyRefused): + contracts.verified_bytes("ideation-workbench") + with pytest.raises(validator.ValidatorUnavailable): + validator.validator_for("ideation-workbench") + + +@pytest.mark.parametrize("changes, says", [ + ({"copies": [{"id": "opendox-snapshot", + "path": "contracts/schemas/opendox-snapshot.schema.yaml", + "sha256": ""}]}, "not a 64-hex sha256"), + ({"copies": [{"id": "opendox-snapshot", + "path": "contracts/schemas/opendox-snapshot.schema.yaml"}]}, + "exactly id, path and sha256"), + ({"copies": [{"id": "opendox-snapshot", "path": "schemas/elsewhere.yaml", + "sha256": "0" * 64}]}, "not 'contracts/schemas/opendox-snapshot"), + ({"copies": [{"id": "opendox-snapshot", + "path": "contracts/schemas/opendox-snapshot.schema.yaml", + "sha256": "f" * 64}] * 2}, "recorded twice"), + ({"copies": []}, "not a non-empty list"), + ({"commit": "cd49eb25"}, "not a full 40-hex commit id"), + ({"spec_leg": "opensoft/openXdox-spec"}, "spec_leg is"), + ({"kind": "pinned_contract_manifest"}, "kind is"), + ({"schema_version": True}, "schema_version is"), + ({"unread": 1}, "its keys are"), +], ids=["empty digest", "no digest", "wrong path", "repeated id", "no copies", + "short commit", "another leg", "another kind", "boolean version", "unknown key"]) +def test_a_record_that_cannot_hold_every_copy_is_refused( + monkeypatch: pytest.MonkeyPatch, changes: dict, says: str) -> None: + """An empty or absent digest is drift and never a pass, and so is a record + whose shape leaves any copy unpinned.""" + _serve(monkeypatch, {contracts.RECORD_NAME: _record_with(**changes)}) + with pytest.raises(contracts.CopyRefused) as refused: + contracts.record() + assert says in str(refused.value) + with pytest.raises(validator.ValidatorUnavailable): + validator.validator_for("opendox-snapshot") + + +def test_the_record_as_shipped_is_accepted() -> None: + """The negative cases above change one field each of the shipped record, + so this is their control.""" + record = contracts.record() + assert (record.commit, record.ids) == (SPEC_COMMIT, THE_FOUR) From 97b314a20c2dd0def2d02ece5c84f84cd8144c92 Mon Sep 17 00:00:00 2001 From: Brett Heap <1513478+brettheap@users.noreply.github.com> Date: Mon, 28 Sep 2026 16:22:53 +0000 Subject: [PATCH 14/34] T057: each 7.1b assertion says which way a schema entered the set (plan 034) A copy of gate-intent planted under src/opendox/contracts/schemas/ failed the 7.1b test as "assert not True". Each of its four assertions now names what it found: a kind, a kind's copy, a pinned copy, or a carried file. Arc: neutral-product-standalone-operability Lane: openxfactory-4 (openXfactory-4-openDox_extraction) Co-Authored-By: Claude Opus 5.5 (1M context) --- tests/test_validator_input_set.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/tests/test_validator_input_set.py b/tests/test_validator_input_set.py index 06ed2db1..941cc7ea 100644 --- a/tests/test_validator_input_set.py +++ b/tests/test_validator_input_set.py @@ -120,10 +120,13 @@ def test_gate_intent_and_the_possibles_register_are_not_in_the_set() -> None: packaged copy, neither is on disk, and no validator can be asked for either.""" for name in ("gate-intent", "ideation-possibles-register"): - assert name not in validator.KIND_ENTRIES - assert name not in {copy for copy, _pointer in validator.KIND_ENTRIES.values()} - assert name not in contracts.record().ids - assert not (PACKAGE / "schemas" / f"{name}.schema.yaml").exists() + assert name not in validator.KIND_ENTRIES, f"{name} is a kind openDox validates" + assert name not in {copy for copy, _pointer in validator.KIND_ENTRIES.values()}, ( + f"a kind is validated against a copy of {name}") + assert name not in contracts.record().ids, f"the record pins a copy of {name}" + assert not (PACKAGE / "schemas" / f"{name}.schema.yaml").exists(), ( + f"a copy of {name} is carried under src/opendox/contracts/schemas/, " + "which 7.1b refuses: requirement 1 keeps it with openxFactory") with pytest.raises(validator.UnknownKind): validator.validator_for(name) with pytest.raises(contracts.CopyRefused): From 69f2040080d8bc1173a6b3bded56e91d19694629 Mon Sep 17 00:00:00 2001 From: Brett Heap <1513478+brettheap@users.noreply.github.com> Date: Mon, 28 Sep 2026 16:30:44 +0000 Subject: [PATCH 15/34] T057: hold the workbench kind over the manifests openDox writes (plan 034) CI's whole suite failed one case, because the two ideation-workbench examples copied from openDox-spec were tracked: tests/test_workbench.py::test_no_workbench_manifest_is_tracked_in_this_repo. The leg's committed-manifest guard, workbench.committed_manifests, refuses a workbench manifest tracked outside examples/, and it is right to. So those two examples leave the corpus. A local run had passed only because it ran before the files were committed. The kind is held instead over the manifests openDox's own workbench.Workbench writes: one of each seed kind, carrying every member route, an exclusion, every action and a notebook binding. An override with no recorded reason is refused as [required] at its member. Arc: neutral-product-standalone-operability Lane: openxfactory-4 (openXfactory-4-openDox_extraction) Co-Authored-By: Claude Opus 5.5 (1M context) --- ...on-workbench-adhoc-human-seen.example.yaml | 29 ---------- ...tion-workbench-cluster-seeded.example.yaml | 41 -------------- tests/test_validator.py | 56 ++++++++++++++++--- 3 files changed, 49 insertions(+), 77 deletions(-) delete mode 100644 tests/fixtures/spec-examples/ideation-workbench-adhoc-human-seen.example.yaml delete mode 100644 tests/fixtures/spec-examples/ideation-workbench-cluster-seeded.example.yaml diff --git a/tests/fixtures/spec-examples/ideation-workbench-adhoc-human-seen.example.yaml b/tests/fixtures/spec-examples/ideation-workbench-adhoc-human-seen.example.yaml deleted file mode 100644 index 87e1d6c5..00000000 --- a/tests/fixtures/spec-examples/ideation-workbench-adhoc-human-seen.example.yaml +++ /dev/null @@ -1,29 +0,0 @@ -# VALID ideation-workbench manifest — the named case "ad-hoc human-seen cluster -# submission" (add-ideation-dashboard task 2.4). A human hand-picked docs from the -# doc list (seed.kind=ad-hoc), then ran draft-organize to package the set as a -# human-seen cluster submission for organizer/cataloger review (tasks 3.5/3.10). -# - every manual-include member carries a recorded reason (human override = evidence); -# - no recipe block: an ad-hoc set need not be intensional (recipe only required -# when seed.kind=recipe); -# - action_history records the draft-organize that produced the submission packet. -schema_version: 1 -kind: ideation-workbench -repository: openxFactory -name: Human-seen cluster - gate console lineage -created: "2026-07-14T10:05:00Z" -updated: "2026-07-14T10:30:00Z" -members: - - document: doc-gate-console - via: manual-include - reason: Reader noticed the gate-console docs share a lineage the clusterer had not linked. - - document: doc-next-step-kickoff - via: manual-include - reason: Kickoff dispatch is the downstream half of the same human-seen theme. -seed: - kind: ad-hoc -action_history: - - action: draft-organize - at: "2026-07-14T10:30:00Z" - reference: staging/gate-console-lineage/PENDING-REVIEW-skeleton -notebook: - alias: xf-wb-gate-console-lineage diff --git a/tests/fixtures/spec-examples/ideation-workbench-cluster-seeded.example.yaml b/tests/fixtures/spec-examples/ideation-workbench-cluster-seeded.example.yaml deleted file mode 100644 index 7d3e9f85..00000000 --- a/tests/fixtures/spec-examples/ideation-workbench-cluster-seeded.example.yaml +++ /dev/null @@ -1,41 +0,0 @@ -# VALID ideation-workbench manifest, cluster-seeded with a saved recipe. -# - seed.kind=cluster-seeded carries cluster_id (schema allOf); -# - a manual-include member carries its recorded reason (never a silent edit); -# - an excluded entry carries its reason (negative evidence); -# - recipe.pinned is a SUBSET of recipe.checked (validator rule W1); -# - recipe.new_candidates is DISJOINT from members and excluded (validator rule W2); -# - the notebook alias matches xf-wb-*. -# This file lives under examples/ (static reference material), so the -# committed-manifest guard deliberately excludes it — a real saved manifest -# would live under gitignored ideation/workbench/. -schema_version: 1 -kind: ideation-workbench -repository: openxFactory -name: Dashboard readiness sweep -created: "2026-07-14T08:15:00Z" -updated: "2026-07-14T09:40:00Z" -members: - - {document: doc-keyword-lens, via: cluster-seed} - - {document: doc-cluster-canvas, via: cluster-seed} - - document: doc-ideation-dashboard-brainstorm - via: manual-include - reason: The originating brainstorm belongs in the readiness sweep even though it predates the cluster. -excluded: - - document: doc-lifecycle-notebook-projection - reason: Notebook-projection doc matched the recipe but is out of the dashboard readiness scope. -seed: - kind: cluster-seeded - cluster_id: cl-ideation-dashboard -recipe: - checked: [ideation-dashboard, keyword-lens, cluster-canvas] - pinned: [ideation-dashboard] - last_run: - source_revision: a1b2c3d4e5f6a7b8c9d0e1f2a3b4c5d6e7f8a9b0 - at: "2026-07-14T09:40:00Z" - new_candidates: [doc-workflow-visualization] -action_history: - - {action: readiness, at: "2026-07-14T09:00:00Z", reference: readiness/xf-wb-run-0007} - - {action: doc-health, at: "2026-07-14T09:20:00Z", reference: health/scoped/xf-wb-0007} - - {action: draft-organize, at: "2026-07-14T09:40:00Z", reference: staging/dashboard-readiness/DRAFT-skeleton} -notebook: - alias: xf-wb-dashboard-readiness diff --git a/tests/test_validator.py b/tests/test_validator.py index 5c0f99b5..4458c64b 100644 --- a/tests/test_validator.py +++ b/tests/test_validator.py @@ -6,10 +6,15 @@ THE CORPUS. `tests/fixtures/spec-examples/` is openDox-spec's own examples, copied byte for byte from openDox-spec#16 at `cd49eb25` -(`examples/ideation-dashboard/`, T053). They are every positive example of the -four kinds, and the neutral snapshot contract's 32 negatives, one per rule. -Each negative's `# expected_failure:` line names the rule it breaks, so the -corpus asks the validator for the rule itself, not for a wording it guesses at. +(`examples/ideation-dashboard/`, T053). They are the positive examples of the +neutral snapshot, chat-turn and model-catalog kinds, and the neutral snapshot +contract's 32 negatives, one per rule. Each negative's `# expected_failure:` +line names the rule it breaks, so the corpus asks the validator for the rule +itself, not for a wording it guesses at. openDox-spec's two `ideation-workbench` +examples are not carried: this leg's committed-manifest guard +(`workbench.committed_manifests`) refuses a workbench manifest tracked outside +`examples/`, and rightly, so that kind is held over the manifests openDox's +own workbench writes instead. WHAT IT HOLDS. @@ -19,9 +24,10 @@ and this module's reference rules name the same seven, and every subschema that can fail names a catalogued shape rule, so no refusal of the neutral kind is left without an identifier. -2. THE OTHER THREE KINDS, STRUCTURALLY. Every positive example of theirs - validates, and each chat-turn wire kind is judged against its own - envelope. +2. THE OTHER THREE KINDS, STRUCTURALLY. Every positive example of the + chat-turn and model-catalog kinds validates, each chat-turn wire kind is + judged against its own envelope, and every manifest openDox's own + workbench writes validates as an `ideation-workbench`. 3. THE EVALUATOR'S SEMANTICS, keyword by keyword, over small schemas of its own: JSON equality, the date-time format, the applicators, and where a violation is reported. @@ -215,6 +221,42 @@ def test_every_positive_example_of_the_other_kinds_validates(path: Path) -> None assert V.validate(instance) == [], V.report(V.validate(instance)) +def test_every_manifest_openDox_own_workbench_writes_validates() -> None: + """The `ideation-workbench` manifests this validator is asked about are the + ones openDox's own `workbench.Workbench` writes (T055 routes + `workbench.validate_manifest` here). One of each seed kind, carrying every + member route, an exclusion, every action and a notebook binding, + validates against the packaged copy. A human override with no recorded + reason, which the writer itself refuses, is refused by the contract too.""" + from opendox import workbench as wb + + now = "2026-09-27T12:00:00Z" + made = [ + wb.Workbench.create("fixture", "an ad-hoc set", now=now), + wb.Workbench.create("fixture", "a cluster set", seed=wb.SEED_CLUSTER, + cluster_id="compost", now=now), + wb.Workbench.create("fixture", "a recipe set", seed=wb.SEED_RECIPE, + recipe={"checked": ["compost", "soil"], "pinned": ["soil"]}, + now=now), + ] + assert {w.data["seed"]["kind"] for w in made} == set(wb.SEED_KINDS) + for w in made: + for via in sorted(wb.VIA_VALUES): + w.add_member(f"notes/{via}.md", via, now=now, + reason="a human chose it" if via == wb.VIA_MANUAL_INCLUDE else None) + w.exclude("notes/left-out.md", "not about the shed", now=now) + for action in sorted(wb.ACTION_VALUES): + w.record_action(action, now=now) + w.bind_notebook(now=now) + manifest = yaml.safe_load(w.render()) + assert V.validate(manifest) == [], V.report(V.validate(manifest)) + silent = yaml.safe_load(made[0].render()) + index = len(silent["members"]) + silent["members"].append({"document": "notes/silent.md", "via": wb.VIA_MANUAL_INCLUDE}) + assert V.report(V.validate(silent)) == [ + f"[required] /members/{index}: 'reason' is required"] + + def test_each_chat_turn_kind_is_judged_against_its_own_envelope() -> None: """A success envelope is valid as a success and not as a request: each wire kind is its own envelope, as openxFactory's `CHAT_TURN_DEFS` names From ca52182f3d0560809e37386e3f5827e785528575 Mon Sep 17 00:00:00 2001 From: Brett Heap <1513478+brettheap@users.noreply.github.com> Date: Mon, 28 Sep 2026 16:47:54 +0000 Subject: [PATCH 16/34] T057: ship the validator's copies as package data (plan 034) The holder has decided that the package-data line goes in this PR now. openDox-code's phase-1 work has landed, so no phase-1 writer edits pyproject.toml any more. 7.1 settles that the four copies travel as package data, "so `pip install openDox-code` puts them on disk beside the validator". Until now the table named only `web/**`, so a wheel carried opendox/contracts/__init__.py with no record and no copy. Every installed validator then refused, naming the absent record. The new key, "opendox.contracts", names copies.yaml and schemas/*.schema.yaml. It is a separate key, because test_gate_loop_contributed holds the bundle's `opendox = ["web/**"]` verbatim. test_the_package_data_ships_the_record_and_every_copy holds the table to the record: the patterns under opendox.contracts ship exactly the record and every copy it pins. The test fails without the line and passes with it. Arc: neutral-product-standalone-operability Lane: openxfactory-4 (openXfactory-4-openDox_extraction) Co-Authored-By: Claude Opus 5.5 (1M context) --- pyproject.toml | 12 ++++++++++++ tests/test_validator_input_set.py | 29 +++++++++++++++++++++++++++++ 2 files changed, 41 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index 10a82163..880ab584 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -238,6 +238,18 @@ where = ["src"] # what that census counts rather than whatever happens to be on disk. [tool.setuptools.package-data] opendox = ["web/**"] +# THE VALIDATOR'S PACKAGED COPIES (plan 034 T057; #1144 7.1, as T007's batch G +# amends it; R1Q12 (a), openxFactory#656 comment 5850003126). openDox's own +# validator reads its spec leg's four schemas from `opendox/contracts/`, and +# 7.1 settles that they travel as PACKAGE DATA, "so `pip install openDox-code` +# puts them on disk beside the validator". Without this line a wheel ships the +# package's `__init__.py` and no record and no copy, and every installed +# validator refuses, naming the absent file. The two patterns name the record +# and the schema copies and nothing else, so no other file under the directory +# ships as data. The bundle's line above stays as it is: +# `test_gate_loop_contributed` holds it verbatim, and +# `tests/test_validator_input_set.py` holds this one to the record. +"opendox.contracts" = ["copies.yaml", "schemas/*.schema.yaml"] [tool.pytest.ini_options] # The ROOTDIR ANCHOR. With no pytest ini table anywhere, pytest infers a diff --git a/tests/test_validator_input_set.py b/tests/test_validator_input_set.py index 941cc7ea..0b28d4e6 100644 --- a/tests/test_validator_input_set.py +++ b/tests/test_validator_input_set.py @@ -29,6 +29,9 @@ its entry declares, and `generator_seam.NEUTRAL_SNAPSHOT_KIND`, the contract openDox's own generator declares (T052), is the packaged neutral contract's `kind` (the holder's note to T057). +4. AN INSTALL CARRIES THEM: `pyproject.toml`'s package-data table ships, + under `opendox.contracts`, exactly the record and every copy the record + pins, and the bundle's own line beside it is unchanged. A CREATED file: no carve-manifest row (RULED OQ-C). """ @@ -294,3 +297,29 @@ def test_the_record_as_shipped_is_accepted() -> None: so this is their control.""" record = contracts.record() assert (record.commit, record.ids) == (SPEC_COMMIT, THE_FOUR) + + +# --------------------------------------------------------------------------- +# package data (7.1): an install carries the copies +# --------------------------------------------------------------------------- + +def test_the_package_data_ships_the_record_and_every_copy() -> None: + """7.1 settles that the copies travel as PACKAGE DATA, so an install has + them beside the validator. The package-data table names, under + `opendox.contracts`, exactly the record and every schema copy the record + pins, and the bundle's own line is unchanged. (A wheel built without it + carries `opendox/contracts/__init__.py` alone, and its validator refuses, + naming the absent record.)""" + import fnmatch + import tomllib + + table = tomllib.loads((ROOT / "pyproject.toml").read_text(encoding="utf-8")) + data = table["tool"]["setuptools"]["package-data"] + assert data["opendox"] == ["web/**"] + patterns = data["opendox.contracts"] + shipped = sorted( + relative for relative in (p.relative_to(PACKAGE).as_posix() + for p in PACKAGE.rglob("*") if p.is_file()) + if any(fnmatch.fnmatchcase(relative, pattern) for pattern in patterns)) + assert shipped == sorted([contracts.RECORD_NAME] + + [copy.resource for copy in contracts.record().copies]) From bc3470b63304056daddd4282302649310339db51 Mon Sep 17 00:00:00 2001 From: Brett Heap <1513478+brettheap@users.noreply.github.com> Date: Mon, 28 Sep 2026 18:00:45 +0000 Subject: [PATCH 17/34] T057: refuse a malformed copy as unavailable, never crash on it (plan 034) Copilot's review at ca52182 found a hole in the fail-closed build. A schema holding `type: {}` made set(names) raise TypeError, so a malformed packaged copy could crash the validator's construction when the module promises SchemaNotEvaluable, which is a ValidatorUnavailable. A probe of the same class found more: - `format: {}`, `allOf: 5`, a pattern with an unbounded repetition (OverflowError), and two non-text unknown keys (a mixed sort) also crashed the build. - Twenty-three other malformed values built, then crashed on an instance or judged it wrongly. For example, `uniqueItems: "yes"` read as true, a negative `maxLength` refused every string, and `maximum: nan` passed everything. The build now checks, and refuses as SchemaNotEvaluable: - every evaluated keyword's value against the shape draft 2020-12 gives it (`_SHAPES`, with a test that no evaluated keyword lacks one); - every reference's target and the kind's entry, walked like the document, since a reference can reach a node no walk of the subschemas passes; - an embedded resource (`$id` or `$schema` below the root), which would move where its references resolve; - a subschema that contains itself, which a YAML alias can build and no walk of ends; - a JSON-pointer index that is not a plain decimal, which Python's int() would read as another element (-1, 01). The four packaged copies use none of these forms, and all six kinds build as before. Arc: neutral-product-standalone-operability Lane: openxfactory-4 (openXfactory-4-openDox_extraction) Co-Authored-By: Claude Opus 5.5 (1M context) --- src/opendox/validator.py | 265 ++++++++++++++++++++++++++++++--------- tests/test_validator.py | 127 ++++++++++++++++++- 2 files changed, 333 insertions(+), 59 deletions(-) diff --git a/src/opendox/validator.py b/src/opendox/validator.py index a20c6d2d..00b0c43a 100644 --- a/src/opendox/validator.py +++ b/src/opendox/validator.py @@ -69,7 +69,13 @@ uses a keyword, a format, a dialect or a reference this module does not evaluate is REFUSED when its validator is built (`SchemaNotEvaluable`). It is never evaluated with that keyword left out, which would pass whatever the -keyword refuses. +keyword refuses. So is a copy that gives an evaluated keyword a value of +another shape (`_SHAPES`: `uniqueItems: "yes"`, `type: {}`, a negative +`maxLength`), an embedded resource (`$id` or `$schema` below the root, which +would move where its references resolve), or a subschema that contains itself. +The build walks every subschema and every reference's target, so nothing the +evaluator can reach escapes those checks, and a malformed copy is reported as +unavailable instead of crashing the build or misjudging an instance. RULE IDENTIFIERS. Every violation names the rule it broke. The neutral snapshot contract gives each rule an id, carried as `x-rule` by the subschema that @@ -114,6 +120,7 @@ class in `x-rules`. A snapshot validator is REFUSED when that catalog names a import calendar import hashlib +import math import re import threading from dataclasses import dataclass @@ -196,8 +203,9 @@ class ValidatorUnavailable(RuntimeError): class SchemaNotEvaluable(ValidatorUnavailable): """A packaged copy uses a keyword, format, dialect or reference this - module does not evaluate, or declares reference rules it does not - implement. It is refused when its validator is built.""" + module does not evaluate, gives a keyword a value of a shape it does not + evaluate, or declares reference rules it does not implement. It is + refused when its validator is built.""" class UnknownKind(ValueError): @@ -485,12 +493,108 @@ def _at_pointer(document: Any, pointer: str) -> Any: if isinstance(node, dict): node = node[token] elif isinstance(node, list): + # A JSON pointer's array index is a plain decimal: never "-1" and + # never "01", which Python's int() would read as other elements. + if not _INDEX.fullmatch(token): + raise KeyError(pointer) node = node[int(token)] else: raise KeyError(pointer) return node +_INDEX = re.compile(r"0|[1-9][0-9]*") + + +# --------------------------------------------------------------------------- +# the shapes the evaluator reads +# --------------------------------------------------------------------------- + +def _is_schema(value: Any) -> bool: + return isinstance(value, (dict, bool)) + + +def _is_count(value: Any) -> bool: + """A non-negative integer, as JSON counts one: `2.0` is 2, `true` is not 1.""" + return _is_type(value, "integer") and value >= 0 + + +def _is_bound(value: Any) -> bool: + return _is_number(value) and math.isfinite(value) + + +def _are_names(value: Any) -> bool: + """A list of distinct property names.""" + return (isinstance(value, list) and all(isinstance(name, str) for name in value) + and len(set(value)) == len(value)) + + +def _are_schemas(value: Any) -> bool: + return isinstance(value, list) and bool(value) and all(map(_is_schema, value)) + + +def _names_schemas(value: Any) -> bool: + return isinstance(value, dict) and all( + isinstance(name, str) and _is_schema(sub) for name, sub in value.items()) + + +def _names_names(value: Any) -> bool: + return isinstance(value, dict) and all( + isinstance(name, str) and _are_names(needed) for name, needed in value.items()) + + +def _names_types(value: Any) -> bool: + names = value if isinstance(value, list) else [value] + return (bool(names) and all(isinstance(name, str) and name in _TYPES for name in names) + and len(set(names)) == len(names)) + + +#: The shape each evaluated keyword's value must have, as draft 2020-12 gives +#: it, and how a refusal names that shape. The evaluator reads every value as +#: shaped here, so a copy whose keyword holds anything else is refused when its +#: validator is built. It is never evaluated, where it would crash on an +#: instance or judge it wrongly: a `uniqueItems: "yes"` read as true, or a +#: negative `maxLength` that no string meets. `format` and `$ref` are checked +#: beside it, with their own refusals. +_SHAPES: Mapping[str, tuple[Callable[[Any], bool], str]] = { + "$defs": (_names_schemas, "an object of schemas"), + "additionalProperties": (_is_schema, "a schema"), + "allOf": (_are_schemas, "a non-empty list of schemas"), + "anyOf": (_are_schemas, "a non-empty list of schemas"), + "contains": (_is_schema, "a schema"), + "dependentRequired": (_names_names, + "an object of lists of distinct property names"), + "enum": (lambda value: isinstance(value, list), "a list"), + "if": (_is_schema, "a schema"), + "items": (_is_schema, "a schema"), + "maxItems": (_is_count, "a non-negative integer"), + "maxLength": (_is_count, "a non-negative integer"), + "maxProperties": (_is_count, "a non-negative integer"), + "maximum": (_is_bound, "a finite number"), + "minItems": (_is_count, "a non-negative integer"), + "minLength": (_is_count, "a non-negative integer"), + "minProperties": (_is_count, "a non-negative integer"), + "minimum": (_is_bound, "a finite number"), + "not": (_is_schema, "a schema"), + "oneOf": (_are_schemas, "a non-empty list of schemas"), + "pattern": (lambda value: isinstance(value, str), "text"), + "properties": (_names_schemas, "an object of schemas"), + "propertyNames": (_is_schema, "a schema"), + "required": (_are_names, "a list of distinct property names"), + "then": (_is_schema, "a schema"), + "type": (_names_types, + f"one of {sorted(_TYPES)}, or a non-empty list of distinct ones"), + "uniqueItems": (lambda value: isinstance(value, bool), "a boolean"), + "x-rule": (lambda value: isinstance(value, str) and bool(value), + "a rule's identifier"), +} + + +class _ContainsItself(ValueError): + """A subschema is its own ancestor: a YAML alias can build one, and no walk + of it ends.""" + + class KindValidator: """The validator of one kind, built over one proved copy. @@ -507,11 +611,8 @@ def __init__(self, kind: str, copy_id: str, pointer: str, document: Any, self.digest = digest self._document = document self._patterns: dict[str, re.Pattern[str]] = {} - self._refuse_what_is_not_evaluated(document) - try: - self._entry = _at_pointer(document, pointer) - except (KeyError, IndexError, ValueError) as exc: - raise self._not_evaluable(f"it has no {pointer!r} for {kind}") from exc + self._refuse_what_is_not_evaluated(document, pointer) + self._entry = _at_pointer(document, pointer) # resolved, and a schema self._reference = self._reference_rules(document) # -- building ----------------------------------------------------------- @@ -521,45 +622,91 @@ def _not_evaluable(self, detail: str) -> SchemaNotEvaluable: f"openDox's validator cannot evaluate its packaged copy of " f"{self.copy_id} for {self.kind}: {detail}") - def _refuse_what_is_not_evaluated(self, document: Any) -> None: + def _refuse_what_is_not_evaluated(self, document: Any, pointer: str) -> None: + """Refuse the copy unless the evaluator would read all of it as it + stands: the whole document, the kind's entry, and every reference's + target. A reference can reach a node that no walk of the document's + subschemas passes (one inside an `enum`, say), and the evaluator would + read that node all the same.""" if not isinstance(document, dict): raise self._not_evaluable(f"it is a {type(document).__name__}, not a schema") if document.get("$schema") != DIALECT: raise self._not_evaluable( f"its dialect is {document.get('$schema')!r}, not {DIALECT!r}") - for at, node in _subschemas(document): - unknown = sorted(set(node) - KEYWORDS) - if unknown: - raise self._not_evaluable(f"{at or ''} uses {unknown}, which " - "this module does not evaluate") - if "type" in node: - names = node["type"] if isinstance(node["type"], list) else [node["type"]] - if not names or not set(names) <= _TYPES: - raise self._not_evaluable(f"{at} names the type(s) {names}") - if "format" in node and node["format"] not in FORMATS: + try: + entry = _at_pointer(document, pointer) + except (KeyError, IndexError, ValueError) as exc: + raise self._not_evaluable(f"it has no {pointer!r} for {self.kind}") from exc + if not _is_schema(entry): + raise self._not_evaluable(f"its {pointer!r} for {self.kind} is a " + f"{type(entry).__name__}, not a schema") + walked: set[int] = set() + pending: list[tuple[str, Any]] = [("", document), (pointer, entry)] + while pending: + start, subtree = pending.pop() + try: + for at, node in _subschemas(subtree, start): + if id(node) not in walked: + walked.add(id(node)) + pending.extend(self._refuse_node(document, at, node)) + except _ContainsItself as exc: + raise self._not_evaluable( + f"{exc.args[0] or ''} contains itself; this module evaluates a " + "schema that is a tree, and a copy repeats itself only by " + "reference") from None + + def _refuse_node(self, document: dict[str, Any], at: str, + node: dict[str, Any]) -> list[tuple[str, Any]]: + """Refuse `node` unless this module evaluates it as it stands, and + answer its reference's target, for the walk to check in its turn.""" + where = at or "" + unknown = sorted((key for key in node if key not in KEYWORDS), key=repr) + if unknown: + raise self._not_evaluable(f"{where} uses {unknown}, which " + "this module does not evaluate") + if at and ("$id" in node or "$schema" in node): + raise self._not_evaluable( + f"{where} is an embedded resource (it carries its own " + f"{'$id' if '$id' in node else '$schema'}), which would move where " + "its references resolve; this module resolves every reference " + "against the copy's root") + for keyword, value in node.items(): + shape = _SHAPES.get(keyword) + if shape is not None and not shape[0](value): + raise self._not_evaluable( + f"{where}'s {keyword} is {_brief(value)}, and this module " + f"evaluates {keyword} only as {shape[1]}") + if "format" in node and not (isinstance(node["format"], str) + and node["format"] in FORMATS): + raise self._not_evaluable( + f"{where} names the format {_brief(node['format'])}, which this module " + f"does not assert (it asserts {sorted(FORMATS)})") + if "pattern" in node: + try: + self._patterns[node["pattern"]] = re.compile(node["pattern"]) + except (re.error, OverflowError, RecursionError) as exc: raise self._not_evaluable( - f"{at} names the format {node['format']!r}, which this module " - f"does not assert (it asserts {sorted(FORMATS)})") - if "pattern" in node: - try: - self._patterns[node["pattern"]] = re.compile(node["pattern"]) - except (re.error, TypeError) as exc: - raise self._not_evaluable( - f"{at}'s pattern does not compile ({exc})") from exc - if "$ref" in node: - ref = node["$ref"] - if not isinstance(ref, str) or not (ref == "#" or ref.startswith("#/")): - raise self._not_evaluable( - f"{at} refers to {ref!r}; only a reference inside the copy " - "is evaluated") - try: - target = _at_pointer(document, ref[1:]) - except (KeyError, IndexError, ValueError) as exc: - raise self._not_evaluable(f"{at}'s reference {ref!r} names " - "nothing in the copy") from exc - if not isinstance(target, (dict, bool)): - raise self._not_evaluable(f"{at}'s reference {ref!r} names a " - f"{type(target).__name__}, not a schema") + f"{where}'s pattern does not compile ({exc})") from exc + return self._reference_target(document, where, node) if "$ref" in node else [] + + def _reference_target(self, document: dict[str, Any], where: str, + node: dict[str, Any]) -> list[tuple[str, Any]]: + """What `node`'s reference names, as (pointer, target), for the walk. + Refused unless the target is a schema inside the copy itself.""" + ref = node["$ref"] + if not isinstance(ref, str) or not (ref == "#" or ref.startswith("#/")): + raise self._not_evaluable( + f"{where} refers to {_brief(ref)}; only a reference inside the copy " + "is evaluated") + try: + target = _at_pointer(document, ref[1:]) + except (KeyError, IndexError, ValueError) as exc: + raise self._not_evaluable(f"{where}'s reference {ref!r} names " + "nothing in the copy") from exc + if not _is_schema(target): + raise self._not_evaluable(f"{where}'s reference {ref!r} names a " + f"{type(target).__name__}, not a schema") + return [(ref[1:], target)] def _reference_rules(self, document: dict[str, Any] ) -> tuple[Callable[[Any], Iterator[Violation]], ...]: @@ -595,16 +742,10 @@ def _valid(self, value: Any, schema: Any, path: tuple[str | int, ...]) -> bool: return next(self._evaluate(value, schema, path), None) is None def _pattern(self, pattern: str) -> re.Pattern[str]: - """The compiled pattern. Every pattern in the copy compiled when this - validator was built, so one is compiled here only if it is reached - through a reference the build's walk did not pass.""" - compiled = self._patterns.get(pattern) - if compiled is None: - try: - compiled = self._patterns[pattern] = re.compile(pattern) - except re.error as exc: - raise self._not_evaluable(f"a pattern does not compile ({exc})") from exc - return compiled + """The compiled pattern. The build walks every subschema the evaluator + can reach, references' targets included, and compiles each pattern it + meets, so this reads what the build compiled.""" + return self._patterns[pattern] def _evaluate(self, value: Any, schema: Any, path: tuple[str | int, ...]) -> Iterator[Violation]: @@ -727,21 +868,33 @@ def _object(self, value: dict[str, Any], schema: dict[str, Any], f"the property name {_brief(key)}: {found.detail}") -def _subschemas(node: Any, at: str = "") -> Iterator[tuple[str, dict[str, Any]]]: - """Every subschema of `node` that is an object, with its location.""" +def _subschemas(node: Any, at: str = "", _above: frozenset[int] = frozenset() + ) -> Iterator[tuple[str, dict[str, Any]]]: + """Every subschema of `node` that is an object, with its location as a JSON + pointer. A node is yielded before its subschemas are walked, so a caller + that refuses a malformed node stops the walk before it reads what the node + holds. Raises `_ContainsItself` at a subschema that is its own ancestor.""" if not isinstance(node, dict): return + if id(node) in _above: + raise _ContainsItself(at) yield at, node + above = _above | {id(node)} for key in ("properties", "$defs"): for name, sub in node.get(key, {}).items() if isinstance(node.get(key), dict) else (): - yield from _subschemas(sub, f"{at}/{key}/{name}") + yield from _subschemas(sub, f"{at}/{key}/{_escape(name)}", above) for key in ("additionalProperties", "items", "contains", "propertyNames", "not", "if", "then"): if key in node: - yield from _subschemas(node[key], f"{at}/{key}") + yield from _subschemas(node[key], f"{at}/{key}", above) for key in ("allOf", "anyOf", "oneOf"): for i, sub in enumerate(node.get(key, ())): - yield from _subschemas(sub, f"{at}/{key}/{i}") + yield from _subschemas(sub, f"{at}/{key}/{i}", above) + + +def _escape(token: Any) -> str: + """A JSON pointer's reference token for a key.""" + return str(token).replace("~", "~0").replace("/", "~1") # --------------------------------------------------------------------------- diff --git a/tests/test_validator.py b/tests/test_validator.py index 4458c64b..6d3f02d1 100644 --- a/tests/test_validator.py +++ b/tests/test_validator.py @@ -32,7 +32,9 @@ own: JSON equality, the date-time format, the applicators, and where a violation is reported. 4. FAIL CLOSED. A copy that uses what this module does not evaluate is refused - when its validator is built, never evaluated with a keyword left out. + when its validator is built, never evaluated with a keyword left out. So + is a keyword holding a value of another shape, wherever the evaluator could + reach it, and a copy's malformation is refused as unavailable, never a crash. 5. jsonschema's SHAPE, FOR THE doxBench SEAM. `serve_workbench`'s two readers of the seam, run over `validators()`, judge the chat turn as they judge it over openxFactory's jsonschema validators, so plan 034's T085 can register @@ -417,13 +419,26 @@ def test_a_boolean_schema() -> None: ({"$ref": "#/$defs/missing"}, "names nothing in the copy"), ({"$ref": "#/title", "title": "t"}, "not a schema"), ({"pattern": "("}, "does not compile"), - ({"type": "text"}, "names the type"), + ({"type": "text"}, "evaluates type only as one of"), ({"x-rules": [{"id": "r", "class": "reference", "says": "?"}]}, "implements []"), ({"x-rules": ["not a rule"]}, "each carry an id"), + # Each of these once crashed the build itself instead of refusing it. + ({"format": {}}, "does not assert"), + ({"pattern": "a{4294967296}"}, "does not compile"), + ({1: "a number as a key", "zz": "text"}, "does not evaluate"), + # An embedded resource would move where its references resolve. + ({"properties": {"a": {"$id": "https://example.test/a"}}}, "embedded resource"), + ({"properties": {"a": {"$schema": V.DIALECT}}}, "embedded resource"), + # A JSON pointer's index is a plain decimal, never Python's reading of it. + ({"$ref": "#/allOf/-1", "allOf": [{}]}, "names nothing in the copy"), + ({"$ref": "#/allOf/01", "allOf": [{}, {}]}, "names nothing in the copy"), ], ids=["patternProperties", "else", "a format", "a remote reference", "a dangling reference", "a reference to text", "a bad pattern", - "an unknown type", "an unimplemented reference rule", "a malformed catalog"]) + "an unknown type", "an unimplemented reference rule", "a malformed catalog", + "a format of another shape", "an unbounded repetition", "mixed keys", + "an embedded $id", "an embedded $schema", "a negative index", + "a zero-padded index"]) def test_what_is_not_evaluated_is_refused_when_the_validator_is_built( schema: dict, says: str) -> None: with pytest.raises(V.SchemaNotEvaluable) as refused: @@ -432,6 +447,112 @@ def test_what_is_not_evaluated_is_refused_when_the_validator_is_built( assert isinstance(refused.value, V.ValidatorUnavailable) +@pytest.mark.parametrize("schema, keyword", [ + ({"type": {}}, "type"), + ({"type": [["string"]]}, "type"), + ({"type": ["string", "string"]}, "type"), + ({"type": []}, "type"), + ({"enum": 5}, "enum"), + ({"required": 5}, "required"), + ({"required": [1]}, "required"), + ({"required": ["a", "a"]}, "required"), + ({"dependentRequired": []}, "dependentRequired"), + ({"dependentRequired": {"a": 5}}, "dependentRequired"), + ({"dependentRequired": {1: ["a"]}}, "dependentRequired"), + ({"minLength": "3"}, "minLength"), + ({"maxLength": -1}, "maxLength"), + ({"minItems": True}, "minItems"), + ({"maxItems": 1.5}, "maxItems"), + ({"minProperties": None}, "minProperties"), + ({"maxProperties": "2"}, "maxProperties"), + ({"minimum": "5"}, "minimum"), + ({"maximum": float("nan")}, "maximum"), + ({"uniqueItems": "yes"}, "uniqueItems"), + ({"pattern": b"^a"}, "pattern"), + ({"properties": []}, "properties"), + ({"properties": {"a": 5}}, "properties"), + ({"properties": {1: {}}}, "properties"), + ({"$defs": []}, "$defs"), + ({"items": [{}]}, "items"), + ({"additionalProperties": "no"}, "additionalProperties"), + ({"contains": []}, "contains"), + ({"propertyNames": 5}, "propertyNames"), + ({"not": []}, "not"), + ({"if": 5, "then": {}}, "if"), + ({"if": {}, "then": 5}, "then"), + ({"allOf": 5}, "allOf"), + ({"allOf": []}, "allOf"), + ({"anyOf": "ab"}, "anyOf"), + ({"oneOf": [5]}, "oneOf"), + ({"x-rule": {}}, "x-rule"), + ({"x-rule": ""}, "x-rule"), +], ids=lambda case: repr(case) if isinstance(case, dict) else case) +def test_a_keyword_holding_a_value_of_another_shape_is_refused_when_built( + schema: dict, keyword: str) -> None: + """The evaluator reads each keyword's value as draft 2020-12 shapes it. + Before `_SHAPES`, `type: {}` and `allOf: 5` crashed the build with a + TypeError. Every other case here built, and then crashed on an instance + or judged it wrongly: `uniqueItems: "yes"` read as true, and a negative + `maxLength` refused every string. Each is now refused as unavailable, at + the place it is written.""" + with pytest.raises(V.SchemaNotEvaluable) as refused: + _built({"properties": {"a": schema}}) + assert f"/properties/a's {keyword} is " in str(refused.value) + assert isinstance(refused.value, V.ValidatorUnavailable) + + +def test_every_keyword_this_module_evaluates_has_a_shape_or_its_own_check() -> None: + """A keyword added to `KEYWORDS` without a shape would be read unchecked. + `$ref` and `format` have their own refusals, `const` holds any value, and + the rest are annotations the evaluator never reads (`x-rules` is checked + as the snapshot contract's catalog).""" + own_check = {"$ref", "format", "const"} + annotations = {"$id", "$schema", "title", "description", "contract_schema_version", + "x-rules"} + assert set(V._SHAPES) == V.KEYWORDS - own_check - annotations + assert annotations | {"$defs", "x-rule"} == set(V._ANNOTATING) + + +def test_a_references_target_is_checked_where_no_walk_of_the_subschemas_reaches() -> None: + """A reference can name a node inside an `enum`, which no walk of the + subschemas passes, and the evaluator reads that node all the same. So + the build walks every reference's target. A malformed target is refused, + and a target's pattern is compiled when the validator is built.""" + malformed = {"$defs": {"x": {"enum": [{"uniqueItems": "yes"}]}}, + "properties": {"a": {"$ref": "#/$defs/x/enum/0"}}} + with pytest.raises(V.SchemaNotEvaluable) as refused: + _built(malformed) + assert "/$defs/x/enum/0's uniqueItems is 'yes'" in str(refused.value) + patterned = {"$defs": {"x": {"enum": [{"pattern": "^a"}]}}, + "properties": {"a": {"$ref": "#/$defs/x/enum/0"}}} + assert _found(patterned, {"a": "abc"}) == set() + assert _found(patterned, {"a": "b"}) == {("pattern", "/a")} + + +def test_the_kinds_entry_is_a_schema_and_is_checked_wherever_it_is() -> None: + """The kind's entry is where evaluation starts, so it is checked like + every subschema, even where no walk of the document reaches.""" + with pytest.raises(V.SchemaNotEvaluable) as refused: + _built({"$defs": {"x": "text"}}, pointer="/$defs/x") + assert "'/$defs/x' for a-test-kind is a str, not a schema" in str(refused.value) + with pytest.raises(V.SchemaNotEvaluable) as refused: + _built({}, pointer="/$defs/missing") + assert "it has no '/$defs/missing'" in str(refused.value) + with pytest.raises(V.SchemaNotEvaluable) as refused: + _built({"$defs": {"x": {"enum": [{"uniqueItems": "yes"}]}}}, + pointer="/$defs/x/enum/0") + assert "/$defs/x/enum/0's uniqueItems is 'yes'" in str(refused.value) + + +def test_a_subschema_that_contains_itself_is_refused() -> None: + """A YAML alias can build a mapping that holds itself, and no walk of it + ends. It is refused where it loops, and never recursed into.""" + looped = yaml.safe_load("&s {properties: {a: *s}}") + with pytest.raises(V.SchemaNotEvaluable) as refused: + _built({"properties": {"b": looped}}) + assert "/properties/b/properties/a contains itself" in str(refused.value) + + def test_another_dialect_is_refused() -> None: with pytest.raises(V.SchemaNotEvaluable) as refused: V.KindValidator("k", "c", "", {"$schema": "http://json-schema.org/draft-07/schema#"}, From 2b32cbf07ee05c6c37b0c01db1c362975b4a5f63 Mon Sep 17 00:00:00 2001 From: Brett Heap <1513478+brettheap@users.noreply.github.com> Date: Mon, 28 Sep 2026 18:26:53 +0000 Subject: [PATCH 18/34] T057: refuse a reference cycle that never moves into the instance (plan 034) Copilot's review at bc3470b found that a copy holding `$ref: "#"`, or two $defs that refer to each other, built, and then is_valid() raised RecursionError. Both are valid draft 2020-12 schemas that no evaluation ends; jsonschema 4.26.0 recurses on them until Python's limit too. The module promises that a copy it cannot evaluate is refused as SchemaNotEvaluable when its validator is built, never a crash in an evaluation. The build now follows every subschema a node applies at its own place in the instance: its reference's target, its allOf, anyOf and oneOf branches, its `not`, and its `if` and `then` when it has both, since the evaluator reads neither alone. A cycle among those is refused, naming its locations. The search is iterative, so it never recurses itself. A recursive schema that moves into the instance before it recurs is still evaluated, such as a tree whose children are items of the node. A copy nested deeper than Python's recursion limit lets the build's walk stop, and it is now refused too, never a RecursionError out of the build. The same review's two other findings, that `enum: []` and `required: []` should be refused, are answered on their threads with evidence and not taken. The draft 2020-12 metaschema gives `enum` no minItems and gives `required` `default: []`; the minItems of 1 was draft-04's. jsonschema treats both as this module does. Arc: neutral-product-standalone-operability Lane: openxfactory-4 (openXfactory-4-openDox_extraction) Co-Authored-By: Claude Opus 5.5 (1M context) --- src/opendox/validator.py | 97 +++++++++++++++++++++++++++++++++------- tests/test_validator.py | 46 +++++++++++++++++++ 2 files changed, 128 insertions(+), 15 deletions(-) diff --git a/src/opendox/validator.py b/src/opendox/validator.py index 00b0c43a..f3f78d31 100644 --- a/src/opendox/validator.py +++ b/src/opendox/validator.py @@ -72,10 +72,13 @@ keyword refuses. So is a copy that gives an evaluated keyword a value of another shape (`_SHAPES`: `uniqueItems: "yes"`, `type: {}`, a negative `maxLength`), an embedded resource (`$id` or `$schema` below the root, which -would move where its references resolve), or a subschema that contains itself. -The build walks every subschema and every reference's target, so nothing the -evaluator can reach escapes those checks, and a malformed copy is reported as -unavailable instead of crashing the build or misjudging an instance. +would move where its references resolve), a subschema that contains itself, +and a cycle of references that never moves into the instance (`$ref: "#"`), +which no evaluation ends. A recursive schema that moves into the instance +before it recurs (a tree's children, as items) is evaluated. The build walks +every subschema and every reference's target, so nothing the evaluator can +reach escapes those checks, and a malformed copy is reported as unavailable +instead of crashing the build or an evaluation, or misjudging an instance. RULE IDENTIFIERS. Every violation names the rule it broke. The neutral snapshot contract gives each rule an id, carried as `x-rule` by the subschema that @@ -640,20 +643,29 @@ def _refuse_what_is_not_evaluated(self, document: Any, pointer: str) -> None: if not _is_schema(entry): raise self._not_evaluable(f"its {pointer!r} for {self.kind} is a " f"{type(entry).__name__}, not a schema") - walked: set[int] = set() + nodes: dict[int, tuple[str, dict[str, Any]]] = {} pending: list[tuple[str, Any]] = [("", document), (pointer, entry)] - while pending: - start, subtree = pending.pop() - try: + try: + while pending: + start, subtree = pending.pop() for at, node in _subschemas(subtree, start): - if id(node) not in walked: - walked.add(id(node)) + if id(node) not in nodes: + nodes[id(node)] = (at, node) pending.extend(self._refuse_node(document, at, node)) - except _ContainsItself as exc: - raise self._not_evaluable( - f"{exc.args[0] or ''} contains itself; this module evaluates a " - "schema that is a tree, and a copy repeats itself only by " - "reference") from None + except _ContainsItself as exc: + raise self._not_evaluable( + f"{exc.args[0] or ''} contains itself; this module evaluates a " + "schema that is a tree, and a copy repeats itself only by " + "reference") from None + except RecursionError: + raise self._not_evaluable("it nests deeper than this module walks") from None + cycle = _cycle_in_place(nodes, document) + if cycle: + raise self._not_evaluable( + f"{' -> '.join(cycle)} is a cycle of subschemas that apply one another " + "at one place in the instance, so no evaluation of it ends; a " + "recursive schema moves into the instance (a property, an item) " + "before it recurs") def _refuse_node(self, document: dict[str, Any], at: str, node: dict[str, Any]) -> list[tuple[str, Any]]: @@ -897,6 +909,61 @@ def _escape(token: Any) -> str: return str(token).replace("~", "~0").replace("/", "~1") +def _in_place(node: dict[str, Any], document: dict[str, Any]) -> Iterator[Any]: + """The subschemas `node` applies at the same place in the instance as + itself: its reference's target, its `allOf`, `anyOf` and `oneOf` branches, + its `not`, and its `if` and `then` when it has both (the evaluator reads + neither alone). Every other applicator moves into the instance: to a + property, an item, or a property's name.""" + if "$ref" in node: + yield _at_pointer(document, node["$ref"][1:]) + for key in ("allOf", "anyOf", "oneOf"): + yield from node.get(key, ()) + if "not" in node: + yield node["not"] + if "if" in node and "then" in node: + yield node["if"] + yield node["then"] + + +def _in_place_ids(node: dict[str, Any], nodes: Mapping[int, Any], + document: dict[str, Any]) -> Iterator[int]: + return (id(sub) for sub in _in_place(node, document) if id(sub) in nodes) + + +def _cycle_in_place(nodes: Mapping[int, tuple[str, dict[str, Any]]], + document: dict[str, Any]) -> list[str]: + """The locations around a cycle of subschemas that apply one another at + one place in the instance, or [] when the copy has none. Evaluating such a + cycle never moves into the instance, so it never ends, whatever the + instance. (jsonschema recurses until Python's limit.)""" + done: set[int] = set() + for start in nodes: + loop = [] if start in done else _cycle_from(start, nodes, document, done) + if loop: + return [nodes[node_id][0] or "" for node_id in loop] + return [] + + +def _cycle_from(start: int, nodes: Mapping[int, tuple[str, dict[str, Any]]], + document: dict[str, Any], done: set[int]) -> list[int]: + """Depth first from `start`, without recursing: the first cycle met, as + the ids around it, or [] once every node reached is marked done.""" + trail = [start] + branches = [_in_place_ids(nodes[start][1], nodes, document)] + while branches: + step = next(branches[-1], None) + if step is None: + done.add(trail.pop()) + branches.pop() + elif step in trail: + return trail[trail.index(step):] + [step] + elif step not in done: + trail.append(step) + branches.append(_in_place_ids(nodes[step][1], nodes, document)) + return [] + + # --------------------------------------------------------------------------- # the validators, over copies proved on every call # --------------------------------------------------------------------------- diff --git a/tests/test_validator.py b/tests/test_validator.py index 6d3f02d1..3b543dc9 100644 --- a/tests/test_validator.py +++ b/tests/test_validator.py @@ -553,6 +553,52 @@ def test_a_subschema_that_contains_itself_is_refused() -> None: assert "/properties/b/properties/a contains itself" in str(refused.value) +@pytest.mark.parametrize("schema, cycle", [ + ({"$ref": "#"}, " -> "), + ({"$defs": {"a": {"$ref": "#/$defs/b"}, "b": {"$ref": "#/$defs/a"}}, + "$ref": "#/$defs/a"}, "/$defs/a -> /$defs/b -> /$defs/a"), + ({"allOf": [{"$ref": "#"}]}, " -> /allOf/0 -> "), + ({"anyOf": [{"type": "string"}, {"$ref": "#"}]}, " -> /anyOf/1 -> "), + ({"oneOf": [{"$ref": "#"}]}, " -> /oneOf/0 -> "), + ({"not": {"$ref": "#"}}, " -> /not -> "), + ({"if": {"$ref": "#"}, "then": {}}, " -> /if -> "), +], ids=["itself", "two $defs", "allOf", "anyOf", "oneOf", "not", "if"]) +def test_a_reference_cycle_that_never_moves_into_the_instance_is_refused( + schema: dict, cycle: str) -> None: + """Each of these is a valid draft 2020-12 schema that no evaluation ends: + jsonschema recurses on it until Python's limit, whatever the instance. + Here each is refused when its validator is built, naming the cycle.""" + with pytest.raises(V.SchemaNotEvaluable) as refused: + _built(schema) + assert f"{cycle} is a cycle of subschemas" in str(refused.value) + + +def test_a_recursive_schema_that_moves_into_the_instance_is_evaluated() -> None: + """A tree whose children are items of the node recurs through `items`, so + each step moves into the instance and every evaluation ends. And an `if` + without a `then` is never read, so a reference back from it is no cycle.""" + tree = {"$defs": {"node": {"type": "object", "required": ["name"], "properties": { + "name": {"type": "string"}, + "children": {"type": "array", "items": {"$ref": "#/$defs/node"}}}}}, + "$ref": "#/$defs/node"} + deep = {"name": "a", "children": [{"name": "b", "children": [{"name": "c"}]}]} + assert _found(tree, deep) == set() + deep["children"][0]["children"][0] = {} + assert _found(tree, deep) == {("required", "/children/0/children/0")} + assert _found({"if": {"$ref": "#"}, "type": "string"}, 5) == {("type", "")} + + +def test_a_copy_nested_deeper_than_the_walk_is_refused() -> None: + """Python's recursion limit bounds the walk. A copy nested past it is + refused as unavailable, never a RecursionError out of the build.""" + deep: dict[str, Any] = {} + for _ in range(5000): + deep = {"not": deep} + with pytest.raises(V.SchemaNotEvaluable) as refused: + _built(deep) + assert "nests deeper than this module walks" in str(refused.value) + + def test_another_dialect_is_refused() -> None: with pytest.raises(V.SchemaNotEvaluable) as refused: V.KindValidator("k", "c", "", {"$schema": "http://json-schema.org/draft-07/schema#"}, From 9d2cda1af9ab26b7f528d6fe1cdd05b01c190609 Mon Sep 17 00:00:00 2001 From: Brett Heap <1513478+brettheap@users.noreply.github.com> Date: Mon, 28 Sep 2026 18:40:39 +0000 Subject: [PATCH 19/34] T057: hold the record to the four, and read pointers as RFC 6901 does (plan 034) Copilot's review at 2b32cbf made two findings, and both are taken. The record accepted any well-formed copy id. An edited copies.yaml could add `gate-intent`, with its file beside the four, and verified_bytes() would serve it. So 7.1b's boundary held only in the tests' census. contracts.COPY_IDS now names openDox's four, and a record that names any other copy, or leaves one out, is refused, as a record naming another spec leg already is. A reference's pointer accepted an escape RFC 6901 does not define (`~2`), and read a percent-encoded fragment literally. A reference to `a%20b` named the key `a%20b`, where jsonschema decodes it and resolves `a b`, so the two would judge an instance differently. An invalid escape now makes no pointer, and a percent-encoded reference is refused as a fragment this module does not decode. The four copies use neither form. Arc: neutral-product-standalone-operability Lane: openxfactory-4 (openXfactory-4-openDox_extraction) Co-Authored-By: Claude Opus 5.5 (1M context) --- src/opendox/contracts/__init__.py | 13 +++++++++++++ src/opendox/validator.py | 11 +++++++++++ tests/test_validator.py | 13 ++++++++++++- tests/test_validator_input_set.py | 15 ++++++++++++++- 4 files changed, 50 insertions(+), 2 deletions(-) diff --git a/src/opendox/contracts/__init__.py b/src/opendox/contracts/__init__.py index d39ecbf6..35052d0d 100644 --- a/src/opendox/contracts/__init__.py +++ b/src/opendox/contracts/__init__.py @@ -52,6 +52,7 @@ from typing import Any __all__ = [ + "COPY_IDS", "COPY_KIND", "CopyRefused", "PackagedCopy", @@ -75,6 +76,14 @@ #: Where a copy sits under this package: `schemas/`. SCHEMA_DIR = "schemas" +#: THE INPUT SET, as a record must hold it: openDox's own spec leg's four +#: schemas (7.1, as T007's batch G amends it). A record that names any other +#: copy, or leaves one of these out, is refused, like a record naming another +#: leg. So no edit to the record lets a copy of `gate-intent` or of the +#: possibles register (7.1b) be served, even with its file beside the others. +COPY_IDS = frozenset({"ideation-workbench", "opendox-snapshot", + "xfactory-workbench-chat-turn", "xfactory-workbench-model-catalog"}) + _ID = re.compile(r"[a-z0-9]+(?:-[a-z0-9]+)*") _COMMIT = re.compile(r"[0-9a-f]{40}") _SHA256 = re.compile(r"[0-9a-f]{64}") @@ -186,6 +195,10 @@ def record() -> Record: if any(copy.id == copy_id for copy in copies): raise _refuse(f"{where}.id {copy_id!r} is recorded twice") copies.append(PackagedCopy(copy_id, path, digest)) + recorded = {copy.id for copy in copies} + if recorded != COPY_IDS: + raise _refuse(f"it records {sorted(recorded)}, not openDox's four, " + f"{sorted(COPY_IDS)} (7.1; 7.1b keeps every other schema out)") return Record(SPEC_LEG, commit, tuple(copies)) diff --git a/src/opendox/validator.py b/src/opendox/validator.py index f3f78d31..0fdc46fa 100644 --- a/src/opendox/validator.py +++ b/src/opendox/validator.py @@ -492,6 +492,9 @@ def _at_pointer(document: Any, pointer: str) -> Any: if not pointer.startswith("/"): raise KeyError(pointer) for token in pointer[1:].split("/"): + # RFC 6901 escapes only `~0` and `~1`; any other `~` makes no pointer. + if not _TOKEN.fullmatch(token): + raise KeyError(pointer) token = _unescape(token) if isinstance(node, dict): node = node[token] @@ -507,6 +510,7 @@ def _at_pointer(document: Any, pointer: str) -> Any: _INDEX = re.compile(r"0|[1-9][0-9]*") +_TOKEN = re.compile(r"(?:[^~]|~[01])*") # --------------------------------------------------------------------------- @@ -710,6 +714,13 @@ def _reference_target(self, document: dict[str, Any], where: str, raise self._not_evaluable( f"{where} refers to {_brief(ref)}; only a reference inside the copy " "is evaluated") + if "%" in ref: + # A reference's fragment is percent-encoded (RFC 3986), and this + # module does not decode it: read literally, `a%20b` would name + # another key than the `a b` jsonschema resolves. + raise self._not_evaluable( + f"{where} refers to {ref!r}, a percent-encoded fragment, which this " + "module does not decode") try: target = _at_pointer(document, ref[1:]) except (KeyError, IndexError, ValueError) as exc: diff --git a/tests/test_validator.py b/tests/test_validator.py index 3b543dc9..752f54a0 100644 --- a/tests/test_validator.py +++ b/tests/test_validator.py @@ -433,12 +433,15 @@ def test_a_boolean_schema() -> None: # A JSON pointer's index is a plain decimal, never Python's reading of it. ({"$ref": "#/allOf/-1", "allOf": [{}]}, "names nothing in the copy"), ({"$ref": "#/allOf/01", "allOf": [{}, {}]}, "names nothing in the copy"), + # RFC 6901 escapes only ~0 and ~1, and a fragment's %-encoding is not decoded. + ({"$ref": "#/$defs/~2", "$defs": {"~2": {}}}, "names nothing in the copy"), + ({"$ref": "#/$defs/a%20b", "$defs": {"a b": {}, "a%20b": {}}}, "percent-encoded"), ], ids=["patternProperties", "else", "a format", "a remote reference", "a dangling reference", "a reference to text", "a bad pattern", "an unknown type", "an unimplemented reference rule", "a malformed catalog", "a format of another shape", "an unbounded repetition", "mixed keys", "an embedded $id", "an embedded $schema", "a negative index", - "a zero-padded index"]) + "a zero-padded index", "an invalid escape", "a percent-encoded fragment"]) def test_what_is_not_evaluated_is_refused_when_the_validator_is_built( schema: dict, says: str) -> None: with pytest.raises(V.SchemaNotEvaluable) as refused: @@ -529,6 +532,14 @@ def test_a_references_target_is_checked_where_no_walk_of_the_subschemas_reaches( assert _found(patterned, {"a": "b"}) == {("pattern", "/a")} +def test_a_references_escapes_are_read_as_rfc_6901_reads_them() -> None: + """`~1` is `/` and `~0` is `~`, so each reference names the key it spells.""" + schema = {"properties": {"a": {"$ref": "#/$defs/x~1y"}, "b": {"$ref": "#/$defs/x~0y"}}, + "$defs": {"x/y": {"type": "string"}, "x~y": {"type": "integer"}}} + assert _found(schema, {"a": 1, "b": "t"}) == {("type", "/a"), ("type", "/b")} + assert _found(schema, {"a": "t", "b": 1}) == set() + + def test_the_kinds_entry_is_a_schema_and_is_checked_wherever_it_is() -> None: """The kind's entry is where evaluation starts, so it is checked like every subschema, even where no walk of the document reaches.""" diff --git a/tests/test_validator_input_set.py b/tests/test_validator_input_set.py index 0b28d4e6..6a4cbdbd 100644 --- a/tests/test_validator_input_set.py +++ b/tests/test_validator_input_set.py @@ -127,6 +127,7 @@ def test_gate_intent_and_the_possibles_register_are_not_in_the_set() -> None: assert name not in {copy for copy, _pointer in validator.KIND_ENTRIES.values()}, ( f"a kind is validated against a copy of {name}") assert name not in contracts.record().ids, f"the record pins a copy of {name}" + assert name not in contracts.COPY_IDS, f"a record may name a copy of {name}" assert not (PACKAGE / "schemas" / f"{name}.schema.yaml").exists(), ( f"a copy of {name} is carried under src/opendox/contracts/schemas/, " "which 7.1b refuses: requirement 1 keeps it with openxFactory") @@ -148,6 +149,7 @@ def test_the_set_is_the_spec_legs_four_and_nothing_else() -> None: on_disk = {path.name.removesuffix(".schema.yaml") for path in (PACKAGE / "schemas").iterdir()} assert set(THE_FOUR) == entries == on_disk == set(contracts.record().ids) + assert contracts.COPY_IDS == set(THE_FOUR) assert not (set(OPENXDOX_SPECS) | set(OPENXFACTORYS)) & on_disk assert sorted(p.name for p in PACKAGE.iterdir() if p.name != "__pycache__") == [ "__init__.py", "copies.yaml", "schemas"] @@ -260,6 +262,10 @@ def test_an_absent_copy_is_refused(monkeypatch: pytest.MonkeyPatch) -> None: validator.validator_for("ideation-workbench") +#: The shipped record's own entries, for the cases that add one or drop one. +_SHIPPED = yaml.safe_load((PACKAGE / "copies.yaml").read_text(encoding="utf-8"))["copies"] + + @pytest.mark.parametrize("changes, says", [ ({"copies": [{"id": "opendox-snapshot", "path": "contracts/schemas/opendox-snapshot.schema.yaml", @@ -278,8 +284,15 @@ def test_an_absent_copy_is_refused(monkeypatch: pytest.MonkeyPatch) -> None: ({"kind": "pinned_contract_manifest"}, "kind is"), ({"schema_version": True}, "schema_version is"), ({"unread": 1}, "its keys are"), + # 7.1b at run time: an edited record cannot let a fifth schema in, even a + # well-formed entry whose file sits beside the four, nor leave one out. + ({"copies": _SHIPPED + [{"id": "gate-intent", + "path": "contracts/schemas/gate-intent.schema.yaml", + "sha256": "0" * 64}]}, "not openDox's four"), + ({"copies": _SHIPPED[:3]}, "not openDox's four"), ], ids=["empty digest", "no digest", "wrong path", "repeated id", "no copies", - "short commit", "another leg", "another kind", "boolean version", "unknown key"]) + "short commit", "another leg", "another kind", "boolean version", "unknown key", + "a fifth copy", "three copies"]) def test_a_record_that_cannot_hold_every_copy_is_refused( monkeypatch: pytest.MonkeyPatch, changes: dict, says: str) -> None: """An empty or absent digest is drift and never a pass, and so is a record From 44745142f992dca6d446cd95b11432608fe2b4b0 Mon Sep 17 00:00:00 2001 From: Brett Heap <1513478+brettheap@users.noreply.github.com> Date: Mon, 28 Sep 2026 18:55:24 +0000 Subject: [PATCH 20/34] T057: order unlike keys by repr, and refuse YAML nested past the limit (plan 034) Copilot's review at 9d2cda1 found, in code this PR had not changed since, that the record's key refusal ran sorted() over a record's keys. So a copies.yaml with a key that is not text raised TypeError, where the module promises CopyRefused. The refusal now orders keys by their repr. An instance fuzz of the same class found a worse case in the evaluator: 115 of 3,000 odd instances crashed a validator. Under additionalProperties false, the report ran sorted() over an instance's extra keys, and YAML can make a key a number or null. Those keys are now ordered by repr too, and a validator judges such an instance rather than raising. Over two seeds of 3,000 instances, all six validators give 0 crashes. A record or copy whose YAML nests past Python's recursion limit escaped all three YAML reads as RecursionError. The record's read, contracts.load() and validator_for() now refuse it, as CopyRefused or ValidatorUnavailable. Arc: neutral-product-standalone-operability Lane: openxfactory-4 (openXfactory-4-openDox_extraction) Co-Authored-By: Claude Opus 5.5 (1M context) --- src/opendox/contracts/__init__.py | 11 +++++--- src/opendox/validator.py | 5 ++-- tests/test_validator.py | 13 +++++++++ tests/test_validator_input_set.py | 44 +++++++++++++++++++++++++++++++ 4 files changed, 67 insertions(+), 6 deletions(-) diff --git a/src/opendox/contracts/__init__.py b/src/opendox/contracts/__init__.py index 35052d0d..0410fee2 100644 --- a/src/opendox/contracts/__init__.py +++ b/src/opendox/contracts/__init__.py @@ -159,13 +159,16 @@ def record() -> Record: try: data = yaml.safe_load(_read_package_file(RECORD_NAME)) - except yaml.YAMLError as exc: - raise _refuse(f"it is not YAML ({exc.__class__.__name__})") from exc + except (yaml.YAMLError, RecursionError) as exc: + # RecursionError: YAML nested past Python's limit, which no read ends. + raise _refuse(f"it is not YAML this module can read " + f"({exc.__class__.__name__})") from exc if not isinstance(data, dict): raise _refuse(f"it is a {type(data).__name__}, not a mapping") expected = {"schema_version", "kind", "spec_leg", "commit", "copies"} if set(data) != expected: - raise _refuse(f"its keys are {sorted(data)}, not {sorted(expected)}") + # A YAML key need not be text, so the keys are ordered by their repr. + raise _refuse(f"its keys are {sorted(data, key=repr)}, not {sorted(expected)}") if data["schema_version"] != 1 or isinstance(data["schema_version"], bool): raise _refuse(f"schema_version is {data['schema_version']!r}, not 1") if data["kind"] != COPY_KIND: @@ -229,7 +232,7 @@ def load(copy_id: str) -> Any: data = verified_bytes(copy_id) try: return yaml.safe_load(data) - except yaml.YAMLError as exc: + except (yaml.YAMLError, RecursionError) as exc: raise CopyRefused( f"the packaged copy of {copy_id} matches its digest but is not " f"YAML ({exc.__class__.__name__})") from exc diff --git a/src/opendox/validator.py b/src/opendox/validator.py index 0fdc46fa..464ad1e5 100644 --- a/src/opendox/validator.py +++ b/src/opendox/validator.py @@ -879,7 +879,8 @@ def _object(self, value: dict[str, Any], schema: dict[str, Any], if extra_schema is False: if extra: yield broken("additionalProperties", - f"unexpected properties {_brief(sorted(extra))}") + f"unexpected properties " + f"{_brief(sorted(extra, key=repr))}") else: for key in extra: yield from self._evaluate(value[key], extra_schema, path + (key,)) @@ -1007,7 +1008,7 @@ def validator_for(kind: str) -> KindValidator: try: document = yaml.safe_load(data) - except yaml.YAMLError as exc: + except (yaml.YAMLError, RecursionError) as exc: raise ValidatorUnavailable( f"the packaged copy of {copy_id} matches its digest but is not YAML " f"({exc.__class__.__name__})") from exc diff --git a/tests/test_validator.py b/tests/test_validator.py index 752f54a0..7f3f3860 100644 --- a/tests/test_validator.py +++ b/tests/test_validator.py @@ -402,6 +402,19 @@ def test_a_reference_applies_beside_its_siblings_and_names_its_own_rule() -> Non ("inner", "type"), ("outer", "maximum")} +def test_an_instance_whose_keys_are_not_text_is_judged_never_crashed_on() -> None: + """YAML can give an instance a key that is not text: a number or null. It + is judged like any other key, and reporting it never orders unlike keys + against each other. An instance fuzz found that ordering raising TypeError + under `additionalProperties: false`.""" + odd = {1: "a number", None: "null", 2.5: "a float", "b": 2} + assert _found({"additionalProperties": False}, odd) == {("additionalProperties", "")} + judged = _built({"properties": {"b": {"type": "integer"}}, "additionalProperties": False, + "propertyNames": {"type": "string"}}).violations(odd) + assert {(v.keyword, v.where) for v in judged} == {("additionalProperties", ""), ("type", "")} + assert "unexpected properties [1, 2.5, None]" in V.report(judged)[0] + + def test_a_boolean_schema() -> None: assert not _found({"properties": {"a": True}}, {"a": object()}) assert _found({"properties": {"a": False}}, {"a": 1}) == {("false", "/a")} diff --git a/tests/test_validator_input_set.py b/tests/test_validator_input_set.py index 6a4cbdbd..8d878f86 100644 --- a/tests/test_validator_input_set.py +++ b/tests/test_validator_input_set.py @@ -305,6 +305,50 @@ def test_a_record_that_cannot_hold_every_copy_is_refused( validator.validator_for("opendox-snapshot") +def test_a_record_whose_keys_are_not_all_text_is_refused( + monkeypatch: pytest.MonkeyPatch) -> None: + """A YAML key need not be text. The refusal names such a key rather than + failing to order it against the others.""" + data = yaml.safe_load((PACKAGE / "copies.yaml").read_text(encoding="utf-8")) + data[1] = "a number as a key" + _serve(monkeypatch, {contracts.RECORD_NAME: yaml.safe_dump(data, sort_keys=False).encode()}) + with pytest.raises(contracts.CopyRefused) as refused: + contracts.record() + assert "its keys are ['commit', 'copies', 'kind', 'schema_version', 'spec_leg', 1]" in ( + str(refused.value)) + with pytest.raises(validator.ValidatorUnavailable): + validator.validator_for("opendox-snapshot") + + +#: YAML nested past Python's recursion limit, which no read of it ends. +_TOO_DEEP = b"[" * 5000 + b"]" * 5000 + + +def test_a_record_nested_past_the_limit_is_refused(monkeypatch: pytest.MonkeyPatch) -> None: + _serve(monkeypatch, {contracts.RECORD_NAME: _TOO_DEEP}) + with pytest.raises(contracts.CopyRefused) as refused: + contracts.record() + assert "not YAML this module can read (RecursionError)" in str(refused.value) + + +def test_a_copy_nested_past_the_limit_is_refused_under_its_own_digest( + monkeypatch: pytest.MonkeyPatch) -> None: + """Recorded under its own digest, so it passes the identity check, the copy + is still refused as unreadable, never a RecursionError to the caller.""" + entries = [dict(entry) for entry in _SHIPPED] + for entry in entries: + if entry["id"] == "opendox-snapshot": + entry["sha256"] = hashlib.sha256(_TOO_DEEP).hexdigest() + _serve(monkeypatch, {contracts.RECORD_NAME: _record_with(copies=entries), + "schemas/opendox-snapshot.schema.yaml": _TOO_DEEP}) + with pytest.raises(contracts.CopyRefused) as refused: + contracts.load("opendox-snapshot") + assert "(RecursionError)" in str(refused.value) + with pytest.raises(validator.ValidatorUnavailable) as unavailable: + validator.validator_for("opendox-snapshot") + assert "(RecursionError)" in str(unavailable.value) + + def test_the_record_as_shipped_is_accepted() -> None: """The negative cases above change one field each of the shipped record, so this is their control.""" From 80b5f9e86e8857b26101d3fa7c496bff39532f3e Mon Sep 17 00:00:00 2001 From: Brett Heap <1513478+brettheap@users.noreply.github.com> Date: Mon, 28 Sep 2026 19:15:53 +0000 Subject: [PATCH 21/34] T057: judge values of any size or depth without crashing (plan 034) Copilot's review at 4474514 made two findings, and both are taken. - A bound past a float's range, such as a 400-digit integer from YAML, crashed the build with OverflowError from math.isfinite(). Every int is now finite, and only floats are asked. - A `const` or `enum` value that contains itself, which a YAML alias builds, was accepted, and then recursed on evaluation. It is now refused when built, since no JSON value contains itself. A deeper probe found the same crash on the instance side. JSON equality was judged on nested tuples, which CPython builds and compares recursively, so an instance value 5000 deep crashed its judgement against any `const`, `enum` or `uniqueItems` with RecursionError. The canon is now text: a tag, a length and a payload per value, with a list or mapping as its children's canons (a mapping's sorted). It is built without recursing, and compared and hashed as text, and it keeps JSON equality: true is not 1, 1 is 1.0, key order is noise. Integers are written in hex, which has no 4300-digit limit. A violation's detail now always shows something. Where repr() fails, for a value nested past its limit or an int past 4300 digits, the detail shows a stand-in. PyYAML raises ValueError, not a YAMLError, for a literal it cannot construct, such as an integer past 4300 digits or an impossible date. All three YAML reads now refuse it. None of the four copies is recursive, so the evaluator's own depth is bounded by theirs. Arc: neutral-product-standalone-operability Lane: openxfactory-4 (openXfactory-4-openDox_extraction) Co-Authored-By: Claude Opus 5.5 (1M context) --- src/opendox/contracts/__init__.py | 6 +- src/opendox/validator.py | 146 ++++++++++++++++++++++++++---- tests/test_validator.py | 67 ++++++++++++++ tests/test_validator_input_set.py | 35 +++++-- 4 files changed, 226 insertions(+), 28 deletions(-) diff --git a/src/opendox/contracts/__init__.py b/src/opendox/contracts/__init__.py index 0410fee2..7a20bdfb 100644 --- a/src/opendox/contracts/__init__.py +++ b/src/opendox/contracts/__init__.py @@ -159,8 +159,10 @@ def record() -> Record: try: data = yaml.safe_load(_read_package_file(RECORD_NAME)) - except (yaml.YAMLError, RecursionError) as exc: + except (yaml.YAMLError, RecursionError, ValueError) as exc: # RecursionError: YAML nested past Python's limit, which no read ends. + # ValueError: a literal PyYAML cannot construct (an integer past + # Python's 4300 digits, or an impossible date). raise _refuse(f"it is not YAML this module can read " f"({exc.__class__.__name__})") from exc if not isinstance(data, dict): @@ -232,7 +234,7 @@ def load(copy_id: str) -> Any: data = verified_bytes(copy_id) try: return yaml.safe_load(data) - except (yaml.YAMLError, RecursionError) as exc: + except (yaml.YAMLError, RecursionError, ValueError) as exc: raise CopyRefused( f"the packaged copy of {copy_id} matches its digest but is not " f"YAML ({exc.__class__.__name__})") from exc diff --git a/src/opendox/validator.py b/src/opendox/validator.py index 464ad1e5..d12487a9 100644 --- a/src/opendox/validator.py +++ b/src/opendox/validator.py @@ -125,6 +125,7 @@ class in `x-rules`. A snapshot validator is REFUSED when that catalog names a import hashlib import math import re +import reprlib import threading from dataclasses import dataclass from typing import Any, Callable, Iterable, Iterator, Mapping @@ -260,8 +261,28 @@ def report(violations: Iterable[Violation]) -> list[str]: # JSON values # --------------------------------------------------------------------------- +#: A repr that stops at a few levels, for a value too deep for `repr()`. +_SHALLOW = reprlib.Repr() +_SHALLOW.maxlevel = 3 + + +def _shown(value: Any) -> str: + """`repr(value)`, or a stand-in where it cannot be made: a value nested + past Python's limit shows its first levels, and an int past 4300 digits + (which only Python, never YAML or JSON, can hand in) shows its size.""" + try: + return repr(value) + except (RecursionError, ValueError): + try: + return _SHALLOW.repr(value) + except (RecursionError, ValueError): + if isinstance(value, int): + return f"" + return f"" + + def _brief(value: Any) -> str: - text = repr(value) + text = _shown(value) return text if len(text) <= _BRIEF else text[:_BRIEF - 3] + "..." @@ -287,25 +308,104 @@ def _is_number(value: Any) -> bool: return isinstance(value, (int, float)) and not isinstance(value, bool) -def _canon(value: Any) -> Any: - """JSON equality: `true` is not `1`, `1` is `1.0`, and key order is noise.""" +def _token(tag: str, text: str) -> str: + return f"{tag}{len(text)}:{text}" + + +def _canon_scalar(value: Any) -> str | None: + """A value that is not a list or a mapping, canonical; None for one that is.""" if isinstance(value, bool): - return ("boolean", value) - if isinstance(value, (int, float)): - return ("number", value) + return _token("b", "1" if value else "0") + if isinstance(value, int) or (isinstance(value, float) and value.is_integer()): + # 1 is 1.0. Hex, because decimal text of an int stops at 4300 digits. + return _token("n", format(int(value), "x")) + if isinstance(value, float): + return _token("f", value.hex()) if isinstance(value, str): - return ("string", value) + return _token("s", value) if value is None: - return ("null",) - if isinstance(value, list): - return ("array", tuple(_canon(v) for v in value)) - if isinstance(value, dict): - # A key is text in JSON. A YAML instance can carry another scalar as a - # key, so each key is tagged, and a mixed mapping still sorts. - return ("object", tuple(sorted( - ((("text", k) if isinstance(k, str) else ("other", repr(k))), _canon(v)) - for k, v in value.items()))) - return ("other", repr(value)) # never equal to a JSON value + return _token("z", "") + if isinstance(value, (list, dict)): + return None + # Never equal to a JSON value. Where no repr can be made, the object is its + # own identity, equal to itself alone. + try: + return _token("o", repr(value)) + except (RecursionError, ValueError): + return _token("o", f"<{type(value).__name__} {id(value)}>") + + +#: What a list or mapping that contains itself canonicalizes to. A YAML alias +#: can build one, and no JSON value is one. No token begins with "!". +_ITSELF = "!itself" + + +def _canon(value: Any) -> str: + """JSON equality: `true` is not `1`, `1` is `1.0`, and key order is noise. + + The canon is text: every value is a tag, a length and its payload, and a + list or mapping is its children's canons, the mapping's sorted, so two + canons are equal exactly when their values are. It is built without + recursing, and compared and hashed as text, so a value of any depth is + judged without exhausting Python's stack (nested tuples compare + recursively, and would). A list or mapping inside itself ends as + `_ITSELF`.""" + scalar = _canon_scalar(value) + if scalar is not None: + return scalar + done: list[str] = [] # finished canons, in the order met + open_ids: set[int] = set() # the containers on the current path + work: list[tuple[bool, Any]] = [(False, value)] + while work: + closing, item = work.pop() + if closing: + open_ids.discard(id(item)) + done.append(_close(item, done)) + continue + scalar = _canon_scalar(item) + if scalar is not None: + done.append(scalar) + elif id(item) in open_ids: + done.append(_ITSELF) + else: + open_ids.add(id(item)) + work.append((True, item)) + work.extend((False, child) for child in reversed( + list(item.values()) if isinstance(item, dict) else item)) + return done[0] + + +def _close(container: list[Any] | dict[Any, Any], done: list[str]) -> str: + """The canon of `container`, from its children's canons at the end of + `done`, which it takes off.""" + count = len(container) + children = done[len(done) - count:] + del done[len(done) - count:] + if isinstance(container, list): + return f"a{count}:" + "".join(children) + # A key is text in JSON. A YAML instance can carry another scalar as a key, + # and a key is a scalar, so each key is its own canon: a text key is never + # the number it spells. + return f"d{count}:" + "".join(sorted( + _canon_scalar(key) + child for key, child in zip(container, children))) + + +def _holds_itself(value: Any) -> bool: + """Whether a list or mapping inside `value` contains itself.""" + path: set[int] = set() + work: list[tuple[bool, Any]] = [(False, value)] + while work: + leaving, node = work.pop() + if leaving: + path.discard(id(node)) + elif isinstance(node, (list, dict)): + if id(node) in path: + return True + path.add(id(node)) + work.append((True, node)) + work.extend((False, child) + for child in (node.values() if isinstance(node, dict) else node)) + return False _DATE_TIME = re.compile( @@ -527,7 +627,10 @@ def _is_count(value: Any) -> bool: def _is_bound(value: Any) -> bool: - return _is_number(value) and math.isfinite(value) + """A number that is not infinite or NaN. Every int is finite, and + `math.isfinite()` cannot convert one past a float's range, so it is asked + only about floats.""" + return _is_number(value) and (isinstance(value, int) or math.isfinite(value)) def _are_names(value: Any) -> bool: @@ -686,6 +789,11 @@ def _refuse_node(self, document: dict[str, Any], at: str, f"{'$id' if '$id' in node else '$schema'}), which would move where " "its references resolve; this module resolves every reference " "against the copy's root") + for keyword in ("const", "enum"): + if keyword in node and _holds_itself(node[keyword]): + raise self._not_evaluable( + f"{where}'s {keyword} holds a value that contains itself, " + "which no JSON value does") for keyword, value in node.items(): shape = _SHAPES.get(keyword) if shape is not None and not shape[0](value): @@ -1008,7 +1116,7 @@ def validator_for(kind: str) -> KindValidator: try: document = yaml.safe_load(data) - except (yaml.YAMLError, RecursionError) as exc: + except (yaml.YAMLError, RecursionError, ValueError) as exc: raise ValidatorUnavailable( f"the packaged copy of {copy_id} matches its digest but is not YAML " f"({exc.__class__.__name__})") from exc diff --git a/tests/test_validator.py b/tests/test_validator.py index 7f3f3860..70f5f536 100644 --- a/tests/test_validator.py +++ b/tests/test_validator.py @@ -415,6 +415,73 @@ def test_an_instance_whose_keys_are_not_text_is_judged_never_crashed_on() -> Non assert "unexpected properties [1, 2.5, None]" in V.report(judged)[0] +def _deep(levels: int) -> list[Any]: + """A list nested `levels` deep, past Python's recursion limit at 5000.""" + value: list[Any] = [] + for _ in range(levels): + value = [value] + return value + + +def test_an_integer_bound_of_any_size_is_evaluated() -> None: + """YAML gives an integer of up to 4300 digits, and a JSON number has no + bound. `math.isfinite()` could not convert one past a float's range, so + such a bound crashed the build with OverflowError.""" + huge = int("9" * 400) + assert _found({"minimum": huge}, 5) == {("minimum", "")} + assert _found({"maximum": -huge}, 5) == {("maximum", "")} + assert _found({"maxLength": huge, "const": huge}, huge) == set() + + +def test_a_const_or_enum_that_contains_itself_is_refused() -> None: + """A YAML alias can build a list that contains itself. No JSON value does, + so a copy whose `const` or `enum` holds one is refused when built.""" + looped = yaml.safe_load("&c [*c]") + for schema in ({"const": looped}, {"enum": [1, looped]}): + with pytest.raises(V.SchemaNotEvaluable) as refused: + _built({"properties": {"a": schema}}) + assert "contains itself, which no JSON value does" in str(refused.value) + + +def test_values_of_any_depth_are_compared_without_recursing() -> None: + """JSON equality is judged on a flat canon, built without recursing. So + neither a deep schema value nor a deep instance exhausts Python's stack + (nested tuples compare recursively, and did), and a violation's detail + stays brief.""" + assert _found({"const": _deep(5000)}, _deep(5000)) == set() + assert _found({"const": [1]}, _deep(5000)) == {("const", "")} + assert _found({"enum": [_deep(5000)]}, _deep(4999)) == {("enum", "")} + assert _found({"uniqueItems": True}, [_deep(5000), _deep(5000)]) == {("uniqueItems", "")} + assert _found({"const": [[1]]}, yaml.safe_load("&c [*c]")) == {("const", "")} + assert len(_built({"const": [1]}).violations(_deep(5000))[0].detail) < 200 + + +def test_a_violations_detail_always_shows_the_value() -> None: + """repr() fails for a value nested deep enough (20000 levels here), and + for an int past 4300 digits, which Python can hand in though YAML and JSON + cannot. Either way the detail shows a stand-in, and the violation is + reported.""" + [deep] = _built({"type": "string"}).violations(_deep(20000)) + assert deep.detail == "[[[[...]]]] is not of type string" + [huge] = _built({"type": "string"}).violations(10 ** 5000) + assert huge.detail == f" is not of type string" + [odd] = _built({"const": 1}).violations({10 ** 5000}) + assert odd.detail == " is not 1" + + +def test_the_canon_keeps_json_equality() -> None: + """`true` is not `1`, `1` is `1.0` and `-0.0` is `0`, key order is noise, + and a text key is not the number it spells.""" + canon = V._canon + assert canon(True) != canon(1) and canon(False) != canon(0) and canon(None) != canon(0) + assert canon(1) == canon(1.0) and canon(-0.0) == canon(0) and canon(0.5) != canon(1) + assert canon({"a": 1, "b": [2, 3]}) == canon({"b": [2.0, 3], "a": 1.0}) + assert canon({"1": "x"}) != canon({1: "x"}) and canon([1, 2]) != canon([2, 1]) + assert canon("a") != canon(["a"]) and canon([]) != canon({}) and canon("") != canon(None) + # An int past 4300 digits has no decimal text, and still has a canon. + assert canon(10 ** 5000) == canon(10 ** 5000) != canon(10 ** 5000 + 1) + + def test_a_boolean_schema() -> None: assert not _found({"properties": {"a": True}}, {"a": object()}) assert _found({"properties": {"a": False}}, {"a": 1}) == {("false", "/a")} diff --git a/tests/test_validator_input_set.py b/tests/test_validator_input_set.py index 8d878f86..78e45477 100644 --- a/tests/test_validator_input_set.py +++ b/tests/test_validator_input_set.py @@ -331,22 +331,43 @@ def test_a_record_nested_past_the_limit_is_refused(monkeypatch: pytest.MonkeyPat assert "not YAML this module can read (RecursionError)" in str(refused.value) -def test_a_copy_nested_past_the_limit_is_refused_under_its_own_digest( - monkeypatch: pytest.MonkeyPatch) -> None: +@pytest.mark.parametrize("copy, raised", [ + (_TOO_DEEP, "RecursionError"), + (b"kind: 2026-02-30\n", "ValueError"), +], ids=["nested past the limit", "an impossible date"]) +def test_an_unreadable_copy_is_refused_under_its_own_digest( + monkeypatch: pytest.MonkeyPatch, copy: bytes, raised: str) -> None: """Recorded under its own digest, so it passes the identity check, the copy - is still refused as unreadable, never a RecursionError to the caller.""" + is still refused as unreadable, never a RecursionError or a ValueError to + the caller.""" entries = [dict(entry) for entry in _SHIPPED] for entry in entries: if entry["id"] == "opendox-snapshot": - entry["sha256"] = hashlib.sha256(_TOO_DEEP).hexdigest() + entry["sha256"] = hashlib.sha256(copy).hexdigest() _serve(monkeypatch, {contracts.RECORD_NAME: _record_with(copies=entries), - "schemas/opendox-snapshot.schema.yaml": _TOO_DEEP}) + "schemas/opendox-snapshot.schema.yaml": copy}) with pytest.raises(contracts.CopyRefused) as refused: contracts.load("opendox-snapshot") - assert "(RecursionError)" in str(refused.value) + assert f"({raised})" in str(refused.value) with pytest.raises(validator.ValidatorUnavailable) as unavailable: validator.validator_for("opendox-snapshot") - assert "(RecursionError)" in str(unavailable.value) + assert f"({raised})" in str(unavailable.value) + + +@pytest.mark.parametrize("record", [ + b"schema_version: " + b"9" * 5000 + b"\n", + b"schema_version: 1\nkind: 2026-02-30\n", +], ids=["an integer past 4300 digits", "an impossible date"]) +def test_a_record_pyyaml_cannot_construct_is_refused( + monkeypatch: pytest.MonkeyPatch, record: bytes) -> None: + """PyYAML raises ValueError, not a YAMLError, for a literal it cannot + construct. The record is refused, never a ValueError to the caller.""" + _serve(monkeypatch, {contracts.RECORD_NAME: record}) + with pytest.raises(contracts.CopyRefused) as refused: + contracts.record() + assert "not YAML this module can read (ValueError)" in str(refused.value) + with pytest.raises(validator.ValidatorUnavailable): + validator.validator_for("opendox-snapshot") def test_the_record_as_shipped_is_accepted() -> None: From 521de95c2ed8f2c7e12714379fc9c7e79bc61d52 Mon Sep 17 00:00:00 2001 From: Brett Heap <1513478+brettheap@users.noreply.github.com> Date: Tue, 29 Sep 2026 16:19:00 +0000 Subject: [PATCH 22/34] T050: the docstring says the required check collects this suite Since T036 (#52), `validate` runs the whole suite (`python -m pytest -q` over the configured testpaths) instead of an explicit list of files. So the paragraph that said this suite was "not yet wired" into that list, and ran by node id, has been false since T036 landed. It now says that the required check collects this suite like every other. The text is #57's correction of the same paragraph, byte for byte. Copilot raised it on #57 (r4117299033). It is taken here because the paragraph is already false at main, so it must not land with #53. When #57 takes this head, that hunk merges clean. The opening paragraph ("T052 and T054 ... have not landed") stays. It is true until T054 lands, and #57 corrects it. Arc: neutral-product-standalone-operability Lane: openxfactory-4 (openXfactory-4-openDox_extraction) Co-Authored-By: Claude Opus 5.5 (1M context) --- tests/test_plain_documents_fixture.py | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/tests/test_plain_documents_fixture.py b/tests/test_plain_documents_fixture.py index 6a0f23de..02a92586 100644 --- a/tests/test_plain_documents_fixture.py +++ b/tests/test_plain_documents_fixture.py @@ -35,13 +35,10 @@ will require regardless of station — `title` and `summary` — per T050's task line and the answer's own example. -NOT YET WIRED INTO `.github/workflows/validate.yml`'s explicit pytest list: -the phase-2 draft-ahead scope keeps this PR out of that file (conftest.py, -pyproject.toml, validate.yml and README.md are the phase-1 chain's), so this -suite runs by node id today, exactly as other narrowed-out suites in this -tree have (`tests/test_display_facet_leaves.py`'s own S7-residue history). -It joins the enumerated list whichever later task next touches it — most -likely F5.3/T056, or T049's own close. +COLLECTED BY THE REQUIRED CHECK. Since T036, `validate` runs the whole suite +(`python -m pytest -q` over the configured testpaths) instead of an explicit +list of files, so this suite is collected like every other and needs no entry +anywhere. A CREATED file: no row in openxFactory's `docs/opendox-carve-manifest.yaml` (RULED OQ-C: the manifest declares what LEAVES openxFactory, never what a From 7028f6136560ab1617bbd37c815806cb48f63e85 Mon Sep 17 00:00:00 2001 From: Brett Heap <1513478+brettheap@users.noreply.github.com> Date: Tue, 29 Sep 2026 16:27:11 +0000 Subject: [PATCH 23/34] T050: the unrelated source shares no topic with the pair, not just no phrase Copilot at 521de95c (r4135803935) is right. The case showed only that the source outside the rain-barrel pair lacks the literal phrase. A later fixture edit could give the sources another topic in common and still pass, which defeats the guarantee the case exists for. The case now compares the two things a topic that sources share can be derived from when nobody declares one (R1Q13 (a) with (c)): the words of each document's name, and the other documents each one names. - Each rain-barrel note's name carries the pair's topic. - The source outside the pair shares no name word with either note. Only the article and conjunction in FUNCTION_WORDS are dropped. - No document on either side names one on the other, by title or by file name. It needs no projection, because T054 is not in this tree. T054's projection test proves the grouping itself over this fixture. The new case goes red on three plants, and the old one passes all three: - a shared name word ("garden" in both titles), which is Copilot's scenario; - the unrelated note naming a pair member by its file name; - a pair member naming the unrelated note by its title. The real fixture passes, with 20 cases, so the triple does not move. Arc: neutral-product-standalone-operability Lane: openxfactory-4 (openXfactory-4-openDox_extraction) Co-Authored-By: Claude Opus 5.5 (1M context) --- tests/test_plain_documents_fixture.py | 89 ++++++++++++++++++++++++--- 1 file changed, 79 insertions(+), 10 deletions(-) diff --git a/tests/test_plain_documents_fixture.py b/tests/test_plain_documents_fixture.py index 02a92586..e615caf4 100644 --- a/tests/test_plain_documents_fixture.py +++ b/tests/test_plain_documents_fixture.py @@ -199,17 +199,86 @@ def test_at_least_two_sources_share_a_topic() -> None: ) +#: The two words this fixture's titles use that carry no topic under any +#: reading of a name: an article and a conjunction. They are dropped before +#: two names are compared, and nothing else is. +FUNCTION_WORDS = frozenset({"the", "and"}) + +_LETTER_RUN = re.compile(r"[^\W\d_]+") + + +def _tokens(text: str) -> list[str]: + """The runs of letters in `text`, case-folded, in order.""" + return [run.casefold() for run in _LETTER_RUN.findall(text)] + + +def _name_words(header: dict[str, str]) -> set[str]: + """The words of a document's name: its `title:` (every document here + declares one, `test_required_fields_present`), as runs of three or more + letters, without `FUNCTION_WORDS`.""" + return {word for word in _tokens(header.get("title", "")) + if len(word) >= 3} - FUNCTION_WORDS + + +def _names(text: str, path: Path, header: dict[str, str]) -> bool: + """Whether `text` names the document at `path`: whether it holds that + document's title, or its file name, as a run of whole words.""" + words = _tokens(text) + for name in (header.get("title", ""), path.stem): + run = _tokens(name) + if run and any(words[i:i + len(run)] == run + for i in range(len(words) - len(run) + 1)): + return True + return False + + def test_at_least_one_source_shares_no_topic() -> None: - """Keeps the assertion above honest: a fixture where every source shared - one word would not exercise topic-based selection at all.""" - singleton = [ - path for path, (header, body) in _documents().items() - if "stage" not in header - and SHARED_SOURCE_TOPIC not in " ".join( - (header.get("title", ""), header.get("summary", ""), body) - ).lower() - ] - assert singleton, "every source shares the same topic phrase" + """At least one source shares no topic with the rain-barrel pair. + Without it, a fixture where every source shared one topic would not + exercise topic-based grouping at all. + + NO TOPIC, not just not the phrase. The phrase check alone would pass a + fixture edit that gave the sources some other topic in common, so this + compares the two things a topic that sources share can be derived from + when nobody declares one (R1Q13 (a) with (c)): + - the words of each document's name; + - the other documents each one names. + + So each rain-barrel note's name carries the pair's topic. A source + outside the pair shares no name word with either note, and it and the + notes never name each other, in either direction. Words elsewhere in + the body are not compared. That the projection then forms exactly this + group is proved over this fixture by T054's projection test. + """ + sources = {path: parts for path, parts in _documents().items() + if "stage" not in parts[0]} + pair = [path for path, (header, body) in sources.items() + if SHARED_SOURCE_TOPIC in " ".join( + (header.get("title", ""), header.get("summary", ""), body) + ).lower()] + others = [path for path in sources if path not in pair] + assert others, "every source shares the same topic phrase" + + topic = set(SHARED_SOURCE_TOPIC.split()) + for member in pair: + assert topic <= _name_words(sources[member][0]), ( + f"{member.name}'s name does not carry {SHARED_SOURCE_TOPIC!r}") + + for other in others: + other_header = sources[other][0] + for member in pair: + member_header = sources[member][0] + shared = _name_words(other_header) & _name_words(member_header) + assert not shared, ( + f"{other.name} and {member.name} share the name word(s) " + f"{sorted(shared)}, so a topic rule could group them") + for writer, named, named_header in ( + (other, member, member_header), + (member, other, other_header)): + assert not _names(writer.read_text(encoding="utf-8"), + named, named_header), ( + f"{writer.name} names {named.name}, so its topics " + "could reach across the pair") if __name__ == "__main__": From 4ba410ff91200d288b78043fb694c3c372e1df4b Mon Sep 17 00:00:00 2001 From: Brett Heap <1513478+brettheap@users.noreply.github.com> Date: Tue, 29 Sep 2026 16:33:21 +0000 Subject: [PATCH 24/34] T050: compare each source's whole derived topic set; pin the fixture's shape Copilot at 7028f613 (r4135883800) is right. Comparing name words and direct naming misses a topic two sources share through a third document: if both name the same other document, both inherit its name words. The case now derives each source's whole topic set: the words of its name, plus the name words of every other fixture document it names, whatever that document's station. Then it compares whole sets. That catches a common name word, one source naming the other, and both naming the same third document. Only "the" and "and" are dropped, so the comparison is stricter than a topic rule with a longer stop list, never looser. The review's overview also asked that the intended source declaration and count be enforced. The module docstring states both, so the cases now hold them: - no document declares `stage: source`, because a source here is a document that declares nothing (R1Q13 (a) with (c)); - there are three sources, two of which share "rain barrel". Six plants each turn the cases red: - a shared name word; - the unrelated note naming a pair member; - a pair member naming the unrelated note; - both naming grouping-compost-corner.md (the review's scenario); - `stage: source` on the unrelated note; - a fourth source. 7028f613's cases pass the third-document plant and the fourth source. Still 20 cases, all passing, so the triple does not move. Arc: neutral-product-standalone-operability Lane: openxfactory-4 (openXfactory-4-openDox_extraction) Co-Authored-By: Claude Opus 5.5 (1M context) --- tests/test_plain_documents_fixture.py | 93 ++++++++++++++++++--------- 1 file changed, 61 insertions(+), 32 deletions(-) diff --git a/tests/test_plain_documents_fixture.py b/tests/test_plain_documents_fixture.py index e615caf4..e00446e8 100644 --- a/tests/test_plain_documents_fixture.py +++ b/tests/test_plain_documents_fixture.py @@ -158,7 +158,12 @@ def test_required_fields_present(path: Path) -> None: def test_spread_across_the_six_stations() -> None: """One document per explicit station, and every undeclared document is - a source (R1Q13 (a) with (c)).""" + a source (R1Q13 (a) with (c)). + + No document declares `stage: source`. This fixture's sources are the + documents that declare nothing, which is the case R1Q13 (a) with (c) + names ("a document that declares nothing is a source"), so the fixture + exercises that reading and not a declared one.""" by_role: dict[str, list[Path]] = {role: [] for role in STAGE_ROLES} for path, (header, _) in _documents().items(): stage = header.get("stage") @@ -168,6 +173,10 @@ def test_spread_across_the_six_stations() -> None: assert stage in STAGE_ROLES, ( f"{path.name} declares stage {stage!r}, not one of {STAGE_ROLES}" ) + assert stage != "source", ( + f"{path.name} declares `stage: source`; a source in this fixture " + "declares nothing, so that it exercises that reading" + ) by_role[stage].append(path) empty = [role for role, paths in by_role.items() if not paths] @@ -201,7 +210,9 @@ def test_at_least_two_sources_share_a_topic() -> None: #: The two words this fixture's titles use that carry no topic under any #: reading of a name: an article and a conjunction. They are dropped before -#: two names are compared, and nothing else is. +#: two names are compared, and nothing else is. So the comparison is +#: stricter than a topic rule that drops more words: it may call a word a +#: shared topic where such a rule would not, but it never misses one. FUNCTION_WORDS = frozenset({"the", "and"}) _LETTER_RUN = re.compile(r"[^\W\d_]+") @@ -232,53 +243,71 @@ def _names(text: str, path: Path, header: dict[str, str]) -> bool: return False +def _derived_topics(path: Path, + documents: dict[Path, tuple[dict[str, str], str]]) -> set[str]: + """The topics a document that declares none is read to carry: the words + of its own name, and the name words of every other document in the + fixture that it names, whatever that document's station.""" + header, _ = documents[path] + topics = set(_name_words(header)) + text = path.read_text(encoding="utf-8") + for named, (named_header, _) in documents.items(): + if named != path and _names(text, named, named_header): + topics |= _name_words(named_header) + return topics + + def test_at_least_one_source_shares_no_topic() -> None: """At least one source shares no topic with the rain-barrel pair. Without it, a fixture where every source shared one topic would not exercise topic-based grouping at all. NO TOPIC, not just not the phrase. The phrase check alone would pass a - fixture edit that gave the sources some other topic in common, so this - compares the two things a topic that sources share can be derived from - when nobody declares one (R1Q13 (a) with (c)): - - the words of each document's name; - - the other documents each one names. - - So each rain-barrel note's name carries the pair's topic. A source - outside the pair shares no name word with either note, and it and the - notes never name each other, in either direction. Words elsewhere in - the body are not compared. That the projection then forms exactly this - group is proved over this fixture by T054's projection test. + fixture edit that gave the sources some other topic in common. So this + compares each source's whole derived topic set: what a topic that + sources share can come from when nobody declares one (R1Q13 (a) with + (c)). That set is: + - the words of the source's name; + - the name words of every other document it names. + Comparing whole sets catches every way two sources come to share a + topic: a common name word, one naming the other, or both naming the same + third document. + + THE SHAPE is the one the module docstring states: three sources, two of + which share "rain barrel". Each note in the pair carries the pair's + topic. The third source's topics meet neither note's. Words elsewhere in + a body are not topics and are not compared. That the projection then + forms exactly this group is proved over this fixture by T054's + projection test. """ - sources = {path: parts for path, parts in _documents().items() - if "stage" not in parts[0]} - pair = [path for path, (header, body) in sources.items() + documents = _documents() + sources = [path for path, (header, _) in documents.items() + if "stage" not in header] + pair = [path for path in sources if SHARED_SOURCE_TOPIC in " ".join( - (header.get("title", ""), header.get("summary", ""), body) + (documents[path][0].get("title", ""), + documents[path][0].get("summary", ""), + documents[path][1]) ).lower()] others = [path for path in sources if path not in pair] - assert others, "every source shares the same topic phrase" + assert (len(pair), len(others)) == (2, 1), ( + f"want two sources sharing {SHARED_SOURCE_TOPIC!r} and one outside " + f"them; the pair is {[p.name for p in pair]} and the rest " + f"{[p.name for p in others]}" + ) + derived = {path: _derived_topics(path, documents) for path in sources} topic = set(SHARED_SOURCE_TOPIC.split()) for member in pair: - assert topic <= _name_words(sources[member][0]), ( - f"{member.name}'s name does not carry {SHARED_SOURCE_TOPIC!r}") - + assert topic <= derived[member], ( + f"{member.name} does not carry {SHARED_SOURCE_TOPIC!r}: " + f"{sorted(derived[member])}") for other in others: - other_header = sources[other][0] for member in pair: - member_header = sources[member][0] - shared = _name_words(other_header) & _name_words(member_header) + shared = derived[other] & derived[member] assert not shared, ( - f"{other.name} and {member.name} share the name word(s) " + f"{other.name} and {member.name} share the topic(s) " f"{sorted(shared)}, so a topic rule could group them") - for writer, named, named_header in ( - (other, member, member_header), - (member, other, other_header)): - assert not _names(writer.read_text(encoding="utf-8"), - named, named_header), ( - f"{writer.name} names {named.name}, so its topics " - "could reach across the pair") if __name__ == "__main__": From 8e7da4a23c66590a2cba1d786b959af2ad39b28f Mon Sep 17 00:00:00 2001 From: Brett Heap <1513478+brettheap@users.noreply.github.com> Date: Tue, 29 Sep 2026 17:01:42 +0000 Subject: [PATCH 25/34] T054: openDox's own scaffold carries the small neutral field set The holder's decision of 2026-09-28: render_scaffold()'s output read as missing title and summary under NEUTRAL_FIELDS. The H1 comes first, so leading_header() stops at the blank line after it, and its field is the capitalized `Summary:`. So standalone agent_capture refused a document `create` had written, and the projection lost the supplied summary (Copilot r4126022820, reproduced on #59). The scaffold now LEADS with the neutral fields the registered corpus obliges: - SCAFFOLD_LEAD_FIELDS = ("title", "summary"), filled from the create's own input. The title drops the family suffix, which stays on the H1. - render_scaffold(lead_fields=...) writes them first, as `name: value` lines, then a blank line, then the governed block unchanged. It refuses a field it has no value for. With lead_fields=() (the default) its output is byte-identical to before. - create_scaffold() passes scaffold_lead_fields(): the fields of that tuple that required_header_fields() answers. Why it follows the corpus rather than always leading: openxFactory's gated create (tests/ideation-dashboard/test_gate_routes.py) and openXdox's authoring suite (tests/test_authoring_agent.py) pin the governed layout with the H1 on the first line. openxFactory's adapter obliges only the governed block's own fields, which it reads over a 15-line window. So there the answer is (), and the created bytes do not change. It asks the corpus the way agent_capture does, so with no home registered create_scaffold now refuses, naming the seam (4.2). Every create path runs after an entry point registers one. The owed half of openDox-code#53's review is taken here too: T054's falsifier case now also asserts that notes-toolshed-inventory.md lands in no group. The pre-change case passes a projection that keeps its stopwords; this one does not. Tests, in tests/test_neutral_projection.py (25 -> 29 cases): - the scaffold under openDox's default leads with both fields, missing_required_headers() is empty, agent_capture accepts the text, and the projection keeps its title and summary; - under a corpus that obliges neither, the bytes equal render_scaffold()'s default, H1 first; - the lead block is SCAFFOLD_LEAD_FIELDS' order, drops the suffix, refuses an unknown field, and covers NEUTRAL_FIELDS; - with nothing registered, create_scaffold refuses and writes nothing. Evidence: - mutations 8/8 killed; - the whole suite as CI runs it: selected=2598 passed=2587 skipped=11; - F4.1's scan: 19 = 19. Arc: neutral-product-standalone-operability Lane: openxfactory-4 (openXfactory-4-openDox_extraction) Co-Authored-By: Claude Opus 5.5 (1M context) --- src/opendox/authoring.py | 91 ++++++++++++++++-- src/opendox/runtime/local_git_adapter.py | 8 +- tests/test_neutral_projection.py | 115 +++++++++++++++++++++++ 3 files changed, 204 insertions(+), 10 deletions(-) diff --git a/src/opendox/authoring.py b/src/opendox/authoring.py index f4a151e1..ca0303be 100644 --- a/src/opendox/authoring.py +++ b/src/opendox/authoring.py @@ -9,7 +9,10 @@ family's ` — Brainstorm` suffix, `Status:`/`Kind:`/`Summary:`/`Topics:`/ `Repository context:`/`Captured:` pre-filled, plus a `## Possible feats` seed section per the openxFactory `ideation/README.md` "Ideation Header - Format") into the CHOSEN ideation area and writes it through + Format"). Where the registered corpus obliges openDox's small neutral + field set, the skeleton leads with a `title:`/`summary:` block above + the H1 (`scaffold_lead_fields`, plan 034 T054). It renders that skeleton + into the CHOSEN ideation area and writes it through `boundary.OutputBoundary.create_document` — create-only, so scaffolding over an existing path refuses as a SOURCE_EDIT (never silently overwritten). `edit_target`/`edit_command` back "select-to-edit": they @@ -105,6 +108,21 @@ # directory" while only the TRAVERSAL half was checked (PR #49 wave-2 critic). IDEATION_PREFIX = "ideation/" +#: THE FIELDS A SCAFFOLD CAN LEAD WITH (plan 034 T054; the holder's decision of +#: 2026-09-28): openDox's small neutral field set, `local_git_adapter +#: .NEUTRAL_FIELDS` (RULED R1Q13 (a)), each filled from the create's own input. +#: openDox's own default adapter reads a document's fields from its LEADING +#: `name: value` block (`local_git_adapter.leading_header`), which ends at the +#: first blank line. The governed block below the H1 carries neither name: its +#: `Summary:` is a different field, and it has no title field at all. So a +#: scaffold that must satisfy that adapter leads with them. +#: +#: They are written only where the registered corpus obliges them +#: (`scaffold_lead_fields`). Where it obliges neither, the governed layout is +#: unchanged: openxFactory's gated create and openXdox's authoring suite pin it +#: with the H1 on the first line. +SCAFFOLD_LEAD_FIELDS: tuple[str, ...] = ("title", "summary") + def _utcnow() -> str: return datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") @@ -118,7 +136,7 @@ def render_scaffold( *, title: str, summary: str, topics: Sequence[str], repository_context: str, kind: str = DEFAULT_KIND, status: str = DEFAULT_STATUS, possible_feats: Sequence[str] = (), source: str | None = None, - now: str | None = None, + now: str | None = None, lead_fields: Sequence[str] = (), ) -> str: """Render a header-compliant skeleton for a NEW ideation document: the H1 with the family's ` — Brainstorm` suffix (so `grep '— Brainstorm$'` finds @@ -132,15 +150,40 @@ def render_scaffold( the keyword-lens recipe at a named `source_revision`). It is additive: the six required fields keep their declared order and their positions, so every existing header check still passes and a scaffold with no source is - byte-identical to what this function rendered before.""" + byte-identical to what this function rendered before. + + `lead_fields` (plan 034 T054) names the fields of `SCAFFOLD_LEAD_FIELDS` + that the scaffold LEADS with. + - Each is written first, as a `name: value` line, in that tuple's order, + and a blank line then separates the block from the H1. + - `title` is the title as given, without the family suffix. `summary` is + the summary as given. + - That block is where openDox's own default adapter reads its small + neutral field set, so a created document satisfies it. + `create_scaffold` passes `scaffold_lead_fields()`. + - A name outside `SCAFFOLD_LEAD_FIELDS` is refused (`ValueError`), since + the scaffold has no value for it. + - With none, which is the default, the scaffold is byte-identical to what + this function rendered before, H1 first.""" + unknown = [name for name in lead_fields if name not in SCAFFOLD_LEAD_FIELDS] + if unknown: + raise ValueError( + f"a scaffold can lead only with {SCAFFOLD_LEAD_FIELDS}, and it has " + f"no value for {unknown}") now = now or _utcnow() - heading = title if title.endswith(BRAINSTORM_SUFFIX) else f"{title}{BRAINSTORM_SUFFIX}" + suffixed = title.endswith(BRAINSTORM_SUFFIX) + heading = title if suffixed else f"{title}{BRAINSTORM_SUFFIX}" + lead_values = {"title": title[:-len(BRAINSTORM_SUFFIX)] if suffixed else title, + "summary": summary} + lead = "".join(f"{name}: {lead_values[name]}\n" + for name in SCAFFOLD_LEAD_FIELDS if name in lead_fields) topics_line = ", ".join(t.strip() for t in topics if t and t.strip()) feats = [f for f in possible_feats if f and f.strip()] or [DEFAULT_POSSIBLE_FEAT] feat_lines = "\n".join(f"- {f}" for f in feats) source_line = f"Source: {source.strip()}\n" if source and source.strip() else "" return ( - f"# {heading}\n\n" + (f"{lead}\n" if lead else "") + + f"# {heading}\n\n" f"Status: {status}\n" f"Kind: {kind}\n" f"Summary: {summary}\n" @@ -248,12 +291,25 @@ def create_scaffold( `staged` from an `ideation/staging/` area; the ruling REVERSED that and the `status_for_area` helper it needed is gone.) `source` records the optional provenance citation. Both are additive: omitting them reproduces the - previous behaviour exactly.""" + previous behaviour exactly. + + THE SCAFFOLD LEADS WITH WHAT THE REGISTERED CORPUS OBLIGES (plan 034 T054; + the holder's decision of 2026-09-28). The fields come from + `scaffold_lead_fields()`, and they are the small neutral field set, + `title` and `summary`, wherever the corpus obliges them, as openDox's own + default adapter does. Before this, openDox's own scaffold read there as + missing both: its H1 comes first, and its `Summary:` is capitalized. So + standalone `agent_capture` refused a document `create` had written, and + the neutral projection lost its summary. Under a corpus that obliges + neither, openxFactory's among them, the scaffold is byte-identical to what + this function wrote before. Asking the corpus means a bare process, with + no home registered, refuses here as `agent_capture` does.""" rel = scaffold_relpath(area, title) text = render_scaffold(title=title, summary=summary, topics=topics, repository_context=repository_context, kind=kind, status=status or DEFAULT_STATUS, - possible_feats=possible_feats, source=source, now=now) + possible_feats=possible_feats, source=source, now=now, + lead_fields=scaffold_lead_fields()) return boundary.create_document(rel, text) @@ -446,6 +502,27 @@ def required_header_fields() -> tuple[str, ...]: return _classify_proposal("").required_fields +def scaffold_lead_fields() -> tuple[str, ...]: + """The fields a scaffold for THIS corpus leads with: those of + `SCAFFOLD_LEAD_FIELDS` that the registered corpus obliges + (`required_header_fields()`), in `SCAFFOLD_LEAD_FIELDS`' order. + + - Under openDox's own default adapter, `WorkingTreeCorpus`, that is both. + So a document `create` writes carries its small neutral field set where + that adapter reads it. `agent_capture` then accepts it, and the neutral + projection copies its title and summary (plan 034 T054). + - Under an adapter that obliges neither, the answer is `()`, and the + scaffold keeps the governed layout, H1 first. openxFactory's adapter is + one such: its fields are the governed block's own `Status:`, `Kind:`, + `Summary:` and the rest. + + It asks the corpus the way `agent_capture` does. So with nothing + registered it refuses as that does (4.2), rather than guessing a layout. + """ + obliged = set(required_header_fields()) + return tuple(name for name in SCAFFOLD_LEAD_FIELDS if name in obliged) + + def missing_required_headers(text: str) -> list[str]: """Required ideation headers absent — OR value-empty — in `text`'s header window. diff --git a/src/opendox/runtime/local_git_adapter.py b/src/opendox/runtime/local_git_adapter.py index de37dd3b..a2ad764d 100644 --- a/src/opendox/runtime/local_git_adapter.py +++ b/src/opendox/runtime/local_git_adapter.py @@ -167,9 +167,11 @@ #: standalone default adapter, `WorkingTreeCorpus`, obliges of a document it #: recognizes, so `authoring.required_header_fields()` answers it. They are #: the `title` and `summary` of T053's neutral snapshot, which openDox's own -#: projection copies from the same header (`leading_header`). A document -#: without them is still listed and read, as a source: a missing field is -#: reported, never a reason to refuse or drop the document. +#: projection copies from the same header (`leading_header`). openDox's own +#: `create` writes them there wherever the corpus obliges them +#: (`authoring.scaffold_lead_fields`). A document without them is still +#: listed and read, as a source: a missing field is reported, never a reason +#: to refuse or drop the document. NEUTRAL_FIELDS: tuple[str, ...] = ("title", "summary") #: The corpus's ONE verdict of its own, and it is a fact about git rather than diff --git a/tests/test_neutral_projection.py b/tests/test_neutral_projection.py index fb74cc77..16bf8546 100644 --- a/tests/test_neutral_projection.py +++ b/tests/test_neutral_projection.py @@ -40,6 +40,10 @@ 5. THE FIELD SET: openDox's default adapter obliges `title` and `summary` (R1Q13 (a)), so `authoring.required_header_fields()` answers them through the entry point, and a document without them is still read. + (5a) THE SCAFFOLD. Following the holder's decision, what openDox's own + `create` writes carries that field set where the adapter reads it, so + `agent_capture` accepts it and the projection keeps its title and summary. + Where the corpus obliges neither field, the governed layout is unchanged. 6. THE VALUES: `SNAPSHOT_VALUES`' defaults, in Python and in `display.js`, are the neutral schema's values, so openDox's views match openDox's snapshot. And the wheel's grouping tile counts a group's edges where the group carries @@ -79,6 +83,7 @@ ) from opendox import generator_seam as gs from opendox import neutral_projection as projection +from opendox.boundary import HUMAN, OutputBoundary from opendox.runtime import local_git_adapter as lga ROOT = Path(__file__).resolve().parents[1] @@ -567,6 +572,15 @@ def test_the_projection_over_the_plain_documents_fixture_is_a_neutral_snapshot( assert derived, snapshot["clusters"] assert any(set(c["topics"]) >= {"rain", "barrel"} for c in derived), derived + # AND THE SOURCE OUTSIDE THAT PAIR LANDS IN NO GROUP. T050's own fixture + # test pins what the topics are derived from, the names and what each + # document names (openDox-code#53, Copilot r4135803935 and r4135883800). + # This pins the projection's own reading of them. + grouped = {e["document"] for c in snapshot["clusters"] + for e in c["document_edges"]} + assert "notes-toolshed-inventory.md" in sources + assert "notes-toolshed-inventory.md" not in grouped, snapshot["clusters"] + # --------------------------------------------------------------------------- # the falsifier, part 2: the topic rule over repository (b), no front matter @@ -868,6 +882,107 @@ def test_the_header_reader_keeps_an_empty_value_and_stops_at_a_blank_line() -> N "a": "2", "b": ""} +# --------------------------------------------------------------------------- +# 5a — openDox's own scaffold carries the small neutral field set +# --------------------------------------------------------------------------- + +#: One create's input, with its date pinned, so two scaffolds compare as bytes. +_CREATE: dict[str, Any] = { + "title": "Shed roof", "summary": "Where the water gets in after a storm.", + "topics": ["shed"], "repository_context": "garden", + "now": "2026-09-29T12:00:00Z"} + + +def test_openDoxs_own_scaffold_carries_the_small_neutral_field_set( + tmp_path: Path) -> None: + """The holder's decision of 2026-09-28 on this PR: what `create` writes + must satisfy `NEUTRAL_FIELDS` under openDox's own default adapter. + + It read there as missing both fields, because its H1 comes first and its + `Summary:` is capitalized. So standalone `agent_capture` refused what + `create` had written, and the projection lost the supplied summary + (Copilot r4126022820, reproduced on openDox-code#59). The scaffold now + leads with `title:` and `summary:`, and its governed block follows it + unchanged.""" + root = _repository(tmp_path, files={"notes-first.md": "title: F\nsummary: S\n"}) + assert authoring.scaffold_lead_fields() == lga.NEUTRAL_FIELDS + written = authoring.create_scaffold(OutputBoundary(root, actor=HUMAN), + **_CREATE) + text = written.read_text(encoding="utf-8") + + assert lga.leading_header(text) == {"title": "Shed roof", + "summary": _CREATE["summary"]} + assert authoring.missing_required_headers(text) == [] + assert text == (f"title: Shed roof\nsummary: {_CREATE['summary']}\n\n" + + authoring.render_scaffold(**_CREATE)) + + # AN AGENT'S CAPTURE OF THE SAME TEXT is not refused as header-incomplete. + captured = authoring.agent_capture( + authoring.agent_boundary(root), + path="ideation/brainstorm/shed-roof-again.md", text=text) + assert captured.read_text(encoding="utf-8") == text + + # AND THE PROJECTION KEEPS the title and summary the create was given. + documents = _by_path(_generate(root)) + for path in (written, captured): + document = documents[path.relative_to(root).as_posix()] + assert (document["title"], document["summary"]) == ( + "Shed roof", _CREATE["summary"]) + + +def test_a_scaffold_keeps_the_governed_layout_where_the_corpus_obliges_neither( + tmp_path: Path) -> None: + """Under a corpus that obliges neither neutral field, the scaffold is + byte-identical to `render_scaffold`'s default, H1 first. openxFactory's + adapter is one such, since its fields are the governed block's own. + openxFactory's gated create (`tests/ideation-dashboard/test_gate_routes.py`) + and openXdox's authoring suite (`tests/test_authoring_agent.py`) pin that + layout.""" + governed = ("Status", "Kind", "Summary", "Topics", "Repository context", + "Captured") + corpus_adapter.register_home(lambda root: ( + lga.WorkingTreeCorpus(required_fields=governed), + corpus_adapter.CorpusRef(name="home", location=str(root)))) + assert authoring.required_header_fields() == governed + assert authoring.scaffold_lead_fields() == () + root = _repository(tmp_path, files={"notes-first.md": "title: F\nsummary: S\n"}) + written = authoring.create_scaffold(OutputBoundary(root, actor=HUMAN), + **_CREATE) + text = written.read_text(encoding="utf-8") + assert text == authoring.render_scaffold(**_CREATE) + assert text.startswith("# Shed roof — Brainstorm\n") + + +def test_the_lead_block_is_the_neutral_fields_in_order_then_a_blank_line() -> None: + """The lead is `SCAFFOLD_LEAD_FIELDS`' order whatever order it is asked + in. The neutral title is the title without the family suffix, which + stays on the H1. A field the scaffold has no value for is refused. And + every field openDox's own default obliges is one a scaffold can lead + with, so a created document always satisfies it.""" + suffixed = {**_CREATE, "title": "Shed roof — Brainstorm"} + plain = authoring.render_scaffold(**suffixed) + assert plain.startswith("# Shed roof — Brainstorm\n") + assert authoring.render_scaffold(**suffixed, lead_fields=("summary", "title")) == ( + f"title: Shed roof\nsummary: {_CREATE['summary']}\n\n" + plain) + assert authoring.render_scaffold(**suffixed, lead_fields=("summary",)) == ( + f"summary: {_CREATE['summary']}\n\n" + plain) + with pytest.raises(ValueError, match=r"no value for \['author'\]"): + authoring.render_scaffold(**suffixed, lead_fields=("title", "author")) + assert set(lga.NEUTRAL_FIELDS) <= set(authoring.SCAFFOLD_LEAD_FIELDS) + + +def test_a_scaffold_with_no_corpus_registered_refuses_and_writes_nothing( + tmp_path: Path) -> None: + """`create_scaffold` asks the corpus which fields to lead with, as + `agent_capture` asks it which to require. So with no home registered it + refuses as that does (4.2), naming the seam, and it writes nothing.""" + corpus_adapter._home_factory = corpus_adapter._UNSET + with pytest.raises(corpus_adapter.CorpusRefused) as refused: + authoring.create_scaffold(OutputBoundary(tmp_path, actor=HUMAN), **_CREATE) + assert refused.value.refusal.kind == corpus_adapter.ADAPTER_NOT_REGISTERED + assert list(tmp_path.rglob("*")) == [] + + # --------------------------------------------------------------------------- # 6 — the product's views match the product's snapshot # --------------------------------------------------------------------------- From bf51a30a5868b03076ca44d4849601a68e06b9aa Mon Sep 17 00:00:00 2001 From: Brett Heap <1513478+brettheap@users.noreply.github.com> Date: Tue, 29 Sep 2026 17:32:15 +0000 Subject: [PATCH 26/34] T057: an instance deeper than the walk is judged, not crashed on Copilot at 27bcefc0 (r4136329332) is right. A recursive schema that moves into the instance (a tree whose children are items of the node) recurs once per level of the instance. So a deep enough tree raised RecursionError out of the validator, although the build accepts the schema. That breaks the module's promise that a value of any size or depth is judged and never crashed on. It was measured before this change: a 200-level tree already raised, at the default limit of 1000, and so did 5000. KindValidator.iter_errors() now catches RecursionError from the walk, once the walk's frames have unwound, and yields one violation: `[evaluation-depth] : ...`. Its rule, DEPTH_RULE, is exported. So the instance is never valid, which fails closed. What the walk found before the limit stands, and the reference rules still run after it. violations(), is_valid() and validate() all go through it. The evaluator stays recursive, because rewriting it on an explicit stack would risk the 0-disagreement cross-check with jsonschema. A tree the walk reaches is judged as before. The regression case is test_an_instance_deeper_than_the_walk_is_judged_not_crashed_on: - a 50-level tree is valid; - a 5000-level one yields exactly the DEPTH_RULE violation, at the root, and is not valid; - with a node near the top broken, both that `required` violation and the depth violation are named. The case raises RecursionError against the pre-change validator. A mutant that swallows the error silently fails it too, since the instance would then pass. The whole suite: selected=2812 passed=2801 skipped=11, which is +1. F4.1: 19. Arc: neutral-product-standalone-operability Lane: openxfactory-4 (openXfactory-4-openDox_extraction) Co-Authored-By: Claude Opus 5.5 (1M context) --- src/opendox/validator.py | 30 ++++++++++++++++++++++++++++-- tests/test_validator.py | 31 +++++++++++++++++++++++++++++++ 2 files changed, 59 insertions(+), 2 deletions(-) diff --git a/src/opendox/validator.py b/src/opendox/validator.py index d12487a9..6ba885e6 100644 --- a/src/opendox/validator.py +++ b/src/opendox/validator.py @@ -75,7 +75,11 @@ would move where its references resolve), a subschema that contains itself, and a cycle of references that never moves into the instance (`$ref: "#"`), which no evaluation ends. A recursive schema that moves into the instance -before it recurs (a tree's children, as items) is evaluated. The build walks +before it recurs (a tree's children, as items) is evaluated. It recurs once per +level of the instance, so an instance nested past what Python's recursion limit +lets the walk reach cannot be walked to its end. Such an instance is judged, +never crashed on: it breaks `DEPTH_RULE`, at the root, so it is never valid, +and whatever the walk found before the limit stands. The build walks every subschema and every reference's target, so nothing the evaluator can reach escapes those checks, and a malformed copy is reported as unavailable instead of crashing the build or an evaluation, or misjudging an instance. @@ -133,6 +137,7 @@ class in `x-rules`. A snapshot validator is REFUSED when that catalog names a from opendox import contracts __all__ = [ + "DEPTH_RULE", "DIALECT", "FORMATS", "KEYWORDS", @@ -153,6 +158,14 @@ class in `x-rules`. A snapshot validator is REFUSED when that catalog names a #: The one dialect the four copies declare, and the one this module evaluates. DIALECT = "https://json-schema.org/draft/2020-12/schema" +#: The rule an instance breaks when it nests deeper than the evaluation can +#: walk. It is this evaluator's own limit, and no contract's rule. A recursive +#: schema that moves into the instance recurs once per level, so a deep enough +#: instance reaches Python's recursion limit. It is judged, not crashed on +#: (`KindValidator.iter_errors`): one violation at the root, so it is never +#: valid, which is failing closed. +DEPTH_RULE = "evaluation-depth" + #: THE INPUT SET (7.1, as batch G amends it): each instance kind openDox #: validates, and where its schema is, as (packaged copy id, JSON pointer into #: that copy). "" is the copy's whole document. The chat-turn copy holds three @@ -859,7 +872,20 @@ def _reference_rules(self, document: dict[str, Any] # -- evaluating --------------------------------------------------------- def iter_errors(self, instance: Any) -> Iterator[Violation]: - yield from self._evaluate(instance, self._entry, ()) + try: + yield from self._evaluate(instance, self._entry, ()) + except RecursionError: + # JUDGED, NOT CRASHED ON (Copilot at openDox-code#58 27bcefc0, + # r4136329332). A recursive schema that moves into the instance + # recurs once per level, so an instance nested past what Python's + # recursion limit lets the walk reach raised out of the validator. + # The walk's frames have unwound by the time this is yielded. + # What it found before the limit stands, and one violation names + # the limit, so the instance is never valid. + yield Violation(DEPTH_RULE, (), "depth", + "the instance nests deeper than this validator's " + "evaluation can walk (Python's recursion limit), so " + "it is not judged valid") for check in self._reference: yield from check(instance) diff --git a/tests/test_validator.py b/tests/test_validator.py index 70f5f536..55e59ca8 100644 --- a/tests/test_validator.py +++ b/tests/test_validator.py @@ -679,6 +679,37 @@ def test_a_recursive_schema_that_moves_into_the_instance_is_evaluated() -> None: assert _found({"if": {"$ref": "#"}, "type": "string"}, 5) == {("type", "")} +def test_an_instance_deeper_than_the_walk_is_judged_not_crashed_on() -> None: + """Copilot at 27bcefc0 (r4136329332). The tree above recurs once per level + of the instance, so a deep enough tree raised `RecursionError` out of the + validator. A 200-level tree already did, at the default limit of 1000. It + is now judged: one `DEPTH_RULE` violation at the root, so it is never + valid. What the walk found before the limit stands, and a tree the walk + does reach is judged as before.""" + tree = {"$defs": {"node": {"type": "object", "required": ["name"], "properties": { + "name": {"type": "string"}, + "children": {"type": "array", "items": {"$ref": "#/$defs/node"}}}}}, + "$ref": "#/$defs/node"} + + def tree_of(levels: int) -> dict[str, Any]: + node: dict[str, Any] = {"name": "leaf"} + for _ in range(levels): + node = {"name": "n", "children": [node]} + return node + + built = _built(tree) + assert built.violations(tree_of(50)) == [] + deep = tree_of(5000) + [found] = built.violations(deep) + assert (found.rule, found.where, found.keyword) == (V.DEPTH_RULE, "", "depth") + assert found.line().startswith("[evaluation-depth] : ") + assert not built.is_valid(deep) + # A broken node near the top is still named, beside the limit. + del deep["children"][0]["name"] + assert {(v.rule, v.where) for v in built.violations(deep)} == { + ("required", "/children/0"), (V.DEPTH_RULE, "")} + + def test_a_copy_nested_deeper_than_the_walk_is_refused() -> None: """Python's recursion limit bounds the walk. A copy nested past it is refused as unavailable, never a RecursionError out of the build.""" From 6b68e32a4b2358edf31e74b17a8c1cd19417736d Mon Sep 17 00:00:00 2001 From: Brett Heap <1513478+brettheap@users.noreply.github.com> Date: Tue, 29 Sep 2026 17:52:01 +0000 Subject: [PATCH 27/34] T057: a violation's detail shows a bound of any size Copilot at bf51a30a lists this as "previously missed", with no thread. It is right. _is_bound() admits an integer of any size, so a schema with `minimum: 10**5000` builds. But the details of minimum, maximum, minLength, maxLength, minItems, maxItems, minProperties and maxProperties interpolated the bound directly. So validating 0 against that schema raised ValueError, Python's 4300-digit limit on int-to-text, while the violation was being written. That breaks the promise that an evaluation never crashes. Reproduced for minimum, maximum (at -10**5000), minLength, minItems and minProperties. The three max counts cannot fire at such a size. Each bound is now shown through _brief(), as the instance's value already was, so a huge one reads ``. An ordinary bound reads exactly as before, since _brief(5) is "5". The regression case is test_a_violations_detail_shows_a_bound_of_any_size, five cases, which all fail against bf51a30a's validator, plus an ordinary-bound check. The whole suite: selected=2817 passed=2806 skipped=11, which is +5. Arc: neutral-product-standalone-operability Lane: openxfactory-4 (openXfactory-4-openDox_extraction) Co-Authored-By: Claude Opus 5.5 (1M context) --- src/opendox/validator.py | 16 ++++++++-------- tests/test_validator.py | 18 ++++++++++++++++++ 2 files changed, 26 insertions(+), 8 deletions(-) diff --git a/src/opendox/validator.py b/src/opendox/validator.py index 6ba885e6..76658cdc 100644 --- a/src/opendox/validator.py +++ b/src/opendox/validator.py @@ -932,9 +932,9 @@ def broken(keyword: str, detail: str) -> Violation: yield from self._string(value, schema, broken) if _is_number(value): if "minimum" in schema and value < schema["minimum"]: - yield broken("minimum", f"{_brief(value)} is less than {schema['minimum']}") + yield broken("minimum", f"{_brief(value)} is less than {_brief(schema['minimum'])}") if "maximum" in schema and value > schema["maximum"]: - yield broken("maximum", f"{_brief(value)} is more than {schema['maximum']}") + yield broken("maximum", f"{_brief(value)} is more than {_brief(schema['maximum'])}") if isinstance(value, list): yield from self._array(value, schema, path, broken) if isinstance(value, dict): @@ -960,9 +960,9 @@ def broken(keyword: str, detail: str) -> Violation: def _string(self, value: str, schema: dict[str, Any], broken: Callable[[str, str], Violation]) -> Iterator[Violation]: if len(value) < schema.get("minLength", 0): - yield broken("minLength", f"{_brief(value)} is shorter than {schema['minLength']}") + yield broken("minLength", f"{_brief(value)} is shorter than {_brief(schema['minLength'])}") if "maxLength" in schema and len(value) > schema["maxLength"]: - yield broken("maxLength", f"{_brief(value)} is longer than {schema['maxLength']}") + yield broken("maxLength", f"{_brief(value)} is longer than {_brief(schema['maxLength'])}") if "pattern" in schema and not self._pattern(schema["pattern"]).search(value): yield broken("pattern", f"{_brief(value)} does not match the rule's pattern") if "format" in schema and not FORMATS[schema["format"]](value): @@ -972,9 +972,9 @@ def _array(self, value: list[Any], schema: dict[str, Any], path: tuple[str | int, ...], broken: Callable[[str, str], Violation]) -> Iterator[Violation]: if len(value) < schema.get("minItems", 0): - yield broken("minItems", f"{len(value)} items, fewer than {schema['minItems']}") + yield broken("minItems", f"{len(value)} items, fewer than {_brief(schema['minItems'])}") if "maxItems" in schema and len(value) > schema["maxItems"]: - yield broken("maxItems", f"{len(value)} items, more than {schema['maxItems']}") + yield broken("maxItems", f"{len(value)} items, more than {_brief(schema['maxItems'])}") if schema.get("uniqueItems") and len({_canon(v) for v in value}) != len(value): yield broken("uniqueItems", f"{_brief(value)} repeats an item") if "items" in schema: @@ -993,10 +993,10 @@ def _object(self, value: dict[str, Any], schema: dict[str, Any], yield broken("required", f"{key!r} is required") if len(value) < schema.get("minProperties", 0): yield broken("minProperties", - f"{len(value)} properties, fewer than {schema['minProperties']}") + f"{len(value)} properties, fewer than {_brief(schema['minProperties'])}") if "maxProperties" in schema and len(value) > schema["maxProperties"]: yield broken("maxProperties", - f"{len(value)} properties, more than {schema['maxProperties']}") + f"{len(value)} properties, more than {_brief(schema['maxProperties'])}") for key, needed in schema.get("dependentRequired", {}).items(): if key in value: for other in needed: diff --git a/tests/test_validator.py b/tests/test_validator.py index 55e59ca8..4525f883 100644 --- a/tests/test_validator.py +++ b/tests/test_validator.py @@ -469,6 +469,24 @@ def test_a_violations_detail_always_shows_the_value() -> None: assert odd.detail == " is not 1" +@pytest.mark.parametrize("schema, instance", [ + ({"minimum": 10 ** 5000}, 0), ({"maximum": -(10 ** 5000)}, 0), + ({"minLength": 10 ** 5000}, "a"), ({"minItems": 10 ** 5000}, []), + ({"minProperties": 10 ** 5000}, {})], + ids=["minimum", "maximum", "minLength", "minItems", "minProperties"]) +def test_a_violations_detail_shows_a_bound_of_any_size(schema, instance) -> None: + """Copilot at bf51a30a, a finding its review lists as previously missed. + A bound builds at any size, but the detail interpolated it directly, so a + bound past 4300 digits raised `ValueError` while its violation was being + written. The bound is now shown the way the value is.""" + [found] = _built(schema).violations(instance) + assert found.keyword == next(iter(schema)) + assert f"" in found.detail + # An ordinary bound reads exactly as it did. + [plain] = _built({"minimum": 5}).violations(1) + assert plain.detail == "1 is less than 5" + + def test_the_canon_keeps_json_equality() -> None: """`true` is not `1`, `1` is `1.0` and `-0.0` is `0`, key order is noise, and a text key is not the number it spells.""" From e94ab323f0c5e3f37112b90dc80f5e1710869279 Mon Sep 17 00:00:00 2001 From: Brett Heap <1513478+brettheap@users.noreply.github.com> Date: Tue, 29 Sep 2026 18:50:51 +0000 Subject: [PATCH 28/34] T057: the record and SPEC_COMMIT move to the spec commit the root pins, f7ee3c76 T053 has landed: openDox-spec#16 squashed as f7ee3c76. The openDox root now pins it (opensoft/openDox#14, dox-v1.1): its spec gitlink and contracts/spec-pin.yaml name f7ee3c76, and its contracts/manifest.yaml records the four files at that commit. T057's falsifier holds each copy to "the spec-leg commit the openDox root pins", so the record's commit and the test's SPEC_COMMIT move from #16's head, cd49eb25, to f7ee3c76 together, on the holder's word. No digest moves. - f7ee3c76 has cd49eb25's tree (7b00d19e). - The four copies are byte for byte the files at f7ee3c76. - The four digests are the ones the root's manifest records there. The comments in copies.yaml and test_validator_input_set.py now say what the root pins, and test_validator.py's corpus note names the landed commit. Checked: - Moving SPEC_COMMIT back alone fails 2 of the falsifier's 28 cases, so the two still move together or not at all. - The whole suite: selected=2817 passed=2806 skipped=11, unchanged. Arc: neutral-product-standalone-operability Lane: openxfactory-4 (openXfactory-4-openDox_extraction) Co-Authored-By: Claude Opus 5.5 (1M context) --- src/opendox/contracts/copies.yaml | 21 +++++++++++---------- tests/test_validator.py | 3 ++- tests/test_validator_input_set.py | 19 ++++++++++--------- 3 files changed, 23 insertions(+), 20 deletions(-) diff --git a/src/opendox/contracts/copies.yaml b/src/opendox/contracts/copies.yaml index d84a9e79..cff23db1 100644 --- a/src/opendox/contracts/copies.yaml +++ b/src/opendox/contracts/copies.yaml @@ -11,16 +11,17 @@ # WHERE THE VALUES CAME FROM, read with `git show : | sha256sum` # in opensoft/openDox-spec. # -# * `commit` is the head of openDox-spec#16 (T053), which is still open. It is -# the one commit that carries all four files. -# * Three of the four files are unchanged there from the blobs the openDox root -# pins today (its `contracts/spec-pin.yaml`, 8fe8c4c7). Their digests are the -# ones the root's `contracts/manifest.yaml` records for them. -# * The fourth is T053's `opendox-snapshot`. Its digest is the one T053's root -# step records. +# * `commit` is the spec-leg commit the openDox root pins: T053 as landed, +# openDox-spec#16's squash f7ee3c76, which the root's `contracts/spec-pin.yaml` +# and `spec` gitlink name since opensoft/openDox#14 (dox-v1.1). It is the one +# commit that carries all four files. +# * Each digest is the one the root's `contracts/manifest.yaml` records for that +# file at that commit. Three of the four files are unchanged from the root's +# previous spec pin, 8fe8c4c7. The fourth is T053's `opendox-snapshot`. # -# When T053 lands and the root's spec pin moves to its commit, `commit` moves -# here in lockstep. A digest moves only when its file changes. +# `commit` moved here from #16's head, cd49eb25, in lockstep with the root's +# spec pin. f7ee3c76 has cd49eb25's tree, so no digest moved. A digest moves +# only when its file changes. # # NEVER EDIT A COPY OR A DIGEST IN PLACE. A copy changes in the spec leg. It # arrives here when the spec leg's file is copied at the pinned commit and its @@ -28,7 +29,7 @@ schema_version: 1 kind: packaged-contract-copies spec_leg: opensoft/openDox-spec -commit: "cd49eb253a431202b67de70b2c9ed941a72d0aa4" +commit: "f7ee3c763b3af4581daf1cd54406e5111e9358e6" copies: - id: ideation-workbench path: contracts/schemas/ideation-workbench.schema.yaml diff --git a/tests/test_validator.py b/tests/test_validator.py index 4525f883..c28540b4 100644 --- a/tests/test_validator.py +++ b/tests/test_validator.py @@ -6,7 +6,8 @@ THE CORPUS. `tests/fixtures/spec-examples/` is openDox-spec's own examples, copied byte for byte from openDox-spec#16 at `cd49eb25` -(`examples/ideation-dashboard/`, T053). They are the positive examples of the +(`examples/ideation-dashboard/`, T053). T053 landed as `f7ee3c76`, with the +same tree, so the copies are the landed files. They are the positive examples of the neutral snapshot, chat-turn and model-catalog kinds, and the neutral snapshot contract's 32 negatives, one per rule. Each negative's `# expected_failure:` line names the rule it breaks, so the corpus asks the validator for the rule diff --git a/tests/test_validator_input_set.py b/tests/test_validator_input_set.py index 78e45477..bd7097d6 100644 --- a/tests/test_validator_input_set.py +++ b/tests/test_validator_input_set.py @@ -56,17 +56,18 @@ THE_FOUR = ("ideation-workbench", "opendox-snapshot", "xfactory-workbench-chat-turn", "xfactory-workbench-model-catalog") -#: The commit the copies were taken at: openDox-spec#16's head (T053), the one -#: commit that carries all four. When T053's root step moves the openDox root's -#: spec pin to T053's landed commit, this moves with the record, in lockstep. -SPEC_COMMIT = "cd49eb253a431202b67de70b2c9ed941a72d0aa4" +#: The spec-leg commit the openDox root pins, which the copies are held to: T053 +#: as landed, openDox-spec#16's squash (opensoft/openDox#14 moved the root's spec +#: pin to it). It is the one commit that carries all four. It moved from #16's +#: head, cd49eb25, with the record, in lockstep. The two commits have one tree. +SPEC_COMMIT = "f7ee3c763b3af4581daf1cd54406e5111e9358e6" #: WHAT THE OPENDOX ROOT PINS, stated here apart from the record, so that a copy -#: and its recorded digest cannot move together unseen. The first three are -#: the digests the root's `contracts/manifest.yaml` records for them at its -#: spec pin, 8fe8c4c7 (the root's `main` at 663ac683). `SPEC_COMMIT` carries -#: those three files unchanged. The fourth is the digest T053's held root step -#: records for its new `opendox-snapshot` manifest entry (openDox-spec#16). +#: and its recorded digest cannot move together unseen. All four are the +#: digests the root's `contracts/manifest.yaml` records for them at its spec +#: pin, `SPEC_COMMIT` (the root's `main` at 52005213, opensoft/openDox#14). The +#: first three are unchanged from the root's previous spec pin, 8fe8c4c7. The +#: fourth is T053's new `opendox-snapshot` entry. PINNED_BY_THE_ROOT = { "ideation-workbench": "d30438491119c20928fbe4e85088fc33682829eeb6558d87dafce651000faafc", From 03e06ccd56cf88fd4a787eac4e87fc8ee537e9f8 Mon Sep 17 00:00:00 2001 From: Brett Heap <1513478+brettheap@users.noreply.github.com> Date: Tue, 29 Sep 2026 23:42:04 +0000 Subject: [PATCH 29/34] T054: a scaffold's leading title and summary are one header line each Copilot at 1a603677 (r4139383472) is right. The lead block interpolated title and summary as given, so summary="ok\nstage: completion" wrote a line that leading_header() reads as the document's own stage:. A create could then land a document in a station the caller never asked for. render_scaffold() now refuses a lead value that carries a control character (Unicode Cc: \n, \r, \t, \x85 and the rest) or a line or paragraph separator (Zl, Zp), which covers every break str.splitlines ends a line at. It raises ValueError before anything is rendered, so create_scaffold writes nothing. With no lead block (a corpus that obliges neither field) nothing new is refused, and the governed layout is unchanged. The regression case is test_a_lead_value_that_is_not_one_header_line_is_refused, over title and summary and ten breakers (20 cases). All 20 fail against 1a603677's authoring.py. The whole suite: selected=2618 passed=2607 skipped=11, which is +20. Arc: neutral-product-standalone-operability Lane: openxfactory-4 (openXfactory-4-openDox_extraction) Co-Authored-By: Claude Opus 5.5 (1M context) --- src/opendox/authoring.py | 23 +++++++++++++++++++++++ tests/test_neutral_projection.py | 26 ++++++++++++++++++++++++++ 2 files changed, 49 insertions(+) diff --git a/src/opendox/authoring.py b/src/opendox/authoring.py index ca0303be..33d7517a 100644 --- a/src/opendox/authoring.py +++ b/src/opendox/authoring.py @@ -55,6 +55,7 @@ import shlex import tempfile +import unicodedata from datetime import datetime, timezone from pathlib import Path from typing import Sequence @@ -163,6 +164,12 @@ def render_scaffold( `create_scaffold` passes `scaffold_lead_fields()`. - A name outside `SCAFFOLD_LEAD_FIELDS` is refused (`ValueError`), since the scaffold has no value for it. + - A lead value that is not ONE header line is refused (`ValueError`) + before anything is rendered: a line break, any other control + character, or a line or paragraph separator. Written into the block, `summary="ok\nstage: + completion"` would be read as the document's own `stage:` line, so a + create could land a document in a station the caller never asked for + (Copilot at openDox-code#57 1a603677, r4139383472). - With none, which is the default, the scaffold is byte-identical to what this function rendered before, H1 first.""" unknown = [name for name in lead_fields if name not in SCAFFOLD_LEAD_FIELDS] @@ -175,6 +182,12 @@ def render_scaffold( heading = title if suffixed else f"{title}{BRAINSTORM_SUFFIX}" lead_values = {"title": title[:-len(BRAINSTORM_SUFFIX)] if suffixed else title, "summary": summary} + for name in SCAFFOLD_LEAD_FIELDS: + if name in lead_fields and not _one_header_line(lead_values[name]): + raise ValueError( + f"a scaffold's leading {name}: must be one header line, and " + f"{lead_values[name]!r} carries a line break or another " + "control character") lead = "".join(f"{name}: {lead_values[name]}\n" for name in SCAFFOLD_LEAD_FIELDS if name in lead_fields) topics_line = ", ".join(t.strip() for t in topics if t and t.strip()) @@ -198,6 +211,16 @@ def render_scaffold( ) +def _one_header_line(value: str) -> bool: + """Whether `value` can stand as one `name: value` header line: no control + character (Unicode `Cc`, which holds `\n`, `\r`, `\t`, `\x85` and the + other ASCII breaks) and no line or paragraph separator (`Zl`, `Zp`). That + is every character `str.splitlines` ends a line at, and every other + control a header reader could stumble on.""" + return not any(unicodedata.category(char) in ("Cc", "Zl", "Zp") + for char in value) + + def normalized_area(area: str) -> str: """`area` with the one trailing slash `scaffold_relpath` assumes.""" return area if area.endswith("/") else area + "/" diff --git a/tests/test_neutral_projection.py b/tests/test_neutral_projection.py index 16bf8546..ff68e318 100644 --- a/tests/test_neutral_projection.py +++ b/tests/test_neutral_projection.py @@ -971,6 +971,32 @@ def test_the_lead_block_is_the_neutral_fields_in_order_then_a_blank_line() -> No assert set(lga.NEUTRAL_FIELDS) <= set(authoring.SCAFFOLD_LEAD_FIELDS) +@pytest.mark.parametrize("field", ["title", "summary"]) +@pytest.mark.parametrize("breaker", ["\n", "\r", "\r\n", "\x0b", "\x1e", "\x85", + "
", "
", "\t", "\x00"]) +def test_a_lead_value_that_is_not_one_header_line_is_refused( + field: str, breaker: str, tmp_path: Path) -> None: + """A leading `title:` or `summary:` is ONE header line. A value that + breaks it would put a line of the caller's choosing into the block the + default adapter reads: `summary="ok\\nstage: completion"` read as the + document's own `stage:` (Copilot at openDox-code#57 1a603677, + r4139383472). So it is refused before anything is rendered or written. + A value with no lead block to break keeps the governed layout as before, + and an ordinary value with inner spaces and dashes still leads.""" + value = f"ok{breaker}stage: completion" + asked = {**_CREATE, field: value} + with pytest.raises(ValueError, match=f"leading {field}: must be one header line"): + authoring.render_scaffold(**asked, lead_fields=("title", "summary")) + assert authoring.render_scaffold(**asked).startswith("# "), ( + "with no lead block, nothing new is refused") + with pytest.raises(ValueError): + authoring.create_scaffold(OutputBoundary(tmp_path, actor=HUMAN), **asked) + assert not any(tmp_path.rglob("*.md")), "a refused create writes nothing" + fine = {**_CREATE, field: "A shed roof — pitched, not flat"} + assert authoring.render_scaffold(**fine, lead_fields=(field,)).startswith( + f"{field}: A shed roof — pitched, not flat\n") + + def test_a_scaffold_with_no_corpus_registered_refuses_and_writes_nothing( tmp_path: Path) -> None: """`create_scaffold` asks the corpus which fields to lead with, as From 50b0d42b7c2ea89e73511dd65bc39fd9254ffa32 Mon Sep 17 00:00:00 2001 From: Brett Heap <1513478+brettheap@users.noreply.github.com> Date: Wed, 30 Sep 2026 00:11:29 +0000 Subject: [PATCH 30/34] T054: one emptiness rule in both vocabularies; the commit date uses the corpus's git Copilot at 03e06ccd raised two threads, and both are taken. r4139523226 (local_git_adapter.py): the suffix vocabulary counts a field given with nothing after its colon as missing, but the kind_field path counted it present by its key alone. _classify_by_header now uses the same test, `not header.get(field)`, so a required field means the same whichever vocabulary a corpus is built with. r4139523258 (default_generator.py): _commit_date ran a bare `git` from PATH, while the home corpus may run another. LocalGitCorpus now exposes its `executable`, and generate() reads the commit date with the home corpus's own git where it is a LocalGitCorpus, and with `git` otherwise. Tests, in tests/test_neutral_projection.py: - test_an_empty_field_is_missing_whichever_vocabulary_is_in_force runs both vocabularies over one repository. It fails against 03e06ccd's adapter. - test_the_commit_date_is_read_by_the_home_corpus_s_own_git registers a home corpus whose git is a logging wrapper. It fails against 03e06ccd's generator. The whole suite: selected=2620 passed=2609 skipped=11, which is +2. F4.1's scan is unchanged, at 20. Arc: neutral-product-standalone-operability Lane: openxfactory-4 (openXfactory-4-openDox_extraction) Co-Authored-By: Claude Opus 5.5 (1M context) --- src/opendox/default_generator.py | 17 ++++++--- src/opendox/runtime/local_git_adapter.py | 12 ++++++- tests/test_neutral_projection.py | 44 ++++++++++++++++++++++++ 3 files changed, 67 insertions(+), 6 deletions(-) diff --git a/src/opendox/default_generator.py b/src/opendox/default_generator.py index 1c814763..d99cb698 100644 --- a/src/opendox/default_generator.py +++ b/src/opendox/default_generator.py @@ -70,7 +70,7 @@ from typing import Any from opendox import corpus_adapter, generator_seam, neutral_projection -from opendox.runtime.local_git_adapter import GitCommandFailed, GitRunner +from opendox.runtime.local_git_adapter import GitCommandFailed, GitRunner, LocalGitCorpus __all__ = ["GENERATOR", "generate"] @@ -86,13 +86,17 @@ r"(?:Z|[+-][0-9]{2}:[0-9]{2})") -def _commit_date(location: str, revision: str) -> str | None: +def _commit_date(location: str, revision: str, *, + executable: str = "git") -> str | None: """The committer date of `revision` in the checkout at `location`, or - None where `git` cannot answer one.""" + None where `git` cannot answer one. `executable` is the git to run: the + home corpus's own where it is a `LocalGitCorpus`, so the date is read by + the git that resolved the revision (Copilot at openDox-code#57 03e06ccd, + r4139523258).""" if not _OBJECT_ID.fullmatch(revision): return None try: - stamp = GitRunner(Path(location)).out( + stamp = GitRunner(Path(location), executable).out( "show", "-s", "--format=%cI", f"{revision}^{{commit}}", "--") except (GitCommandFailed, OSError): return None @@ -109,7 +113,10 @@ def generate(repo_root: Path, repository: str, *, corpus = adapter.resolve(ref) anchor = source_revision if source_revision is not None else corpus.revision if generated_at is None and anchor is not None: - generated_at = _commit_date(corpus.location, anchor) + generated_at = _commit_date( + corpus.location, anchor, + executable=(adapter.executable if isinstance(adapter, LocalGitCorpus) + else "git")) projection = neutral_projection.project( adapter, corpus, repository, source_revision=anchor, generated_at=generated_at) diff --git a/src/opendox/runtime/local_git_adapter.py b/src/opendox/runtime/local_git_adapter.py index a2ad764d..83836400 100644 --- a/src/opendox/runtime/local_git_adapter.py +++ b/src/opendox/runtime/local_git_adapter.py @@ -1615,6 +1615,13 @@ def __init__(self, *, executable: str = "git", self._kind_field = kind_field self._required_fields = tuple(required_fields) + @property + def executable(self) -> str: + """The `git` this corpus runs, for a caller that reads the same + checkout with its own `GitRunner` (`default_generator`'s commit date), + so both run one git.""" + return self._executable + # -- resolve ---------------------------------------------------------- def resolve(self, ref: CorpusRef) -> ResolvedCorpus: @@ -1999,10 +2006,13 @@ def _classify_by_header(self, corpus: ResolvedCorpus, unclassifiable=( f"{document.key!r} carries no {self._kind_field!r} header; " "it is still listed and still readable")) + # THE SAME EMPTINESS RULE AS THE SUFFIX PATH (Copilot at + # openDox-code#57 03e06ccd, r4139523226): a field given with nothing + # after its colon is no field, whichever vocabulary is in force. return Classification( id=document, kind=kind, required_fields=self._required_fields, missing_fields=tuple(field for field in self._required_fields - if field not in header)) + if not header.get(field))) def _header_of(self, corpus: ResolvedCorpus, document: DocumentId) -> dict[str, str]: diff --git a/tests/test_neutral_projection.py b/tests/test_neutral_projection.py index ff68e318..e4073e71 100644 --- a/tests/test_neutral_projection.py +++ b/tests/test_neutral_projection.py @@ -868,6 +868,50 @@ def test_the_default_adapter_obliges_the_small_neutral_field_set(tmp_path: Path) assert _by_path(_generate(root))["none.md"]["stage"] == "source" +def test_an_empty_field_is_missing_whichever_vocabulary_is_in_force( + tmp_path: Path) -> None: + """The suffix vocabulary and the `kind_field` one read a field given with + nothing after its colon the same way: as no field. The `kind_field` path + counted it present by its key alone (Copilot at openDox-code#57 + 03e06ccd, r4139523226).""" + root = _repository(tmp_path, files={ + "both.md": "kind: note\ntitle: T\nsummary: S\n", + "empty.md": "kind: note\ntitle:\nsummary: S\n", + "none.md": "kind: note\n"}) + ref = corpus_adapter.CorpusRef(name="home", location=str(root)) + by_suffix = lga.LocalGitCorpus(required_fields=lga.NEUTRAL_FIELDS) + by_header = lga.LocalGitCorpus(kind_field="kind", + required_fields=lga.NEUTRAL_FIELDS) + for adapter in (by_suffix, by_header): + corpus = adapter.resolve(ref) + missing = {d.key: adapter.classify(corpus, d).missing_fields + for d in adapter.list_documents(corpus)} + assert missing == {"both.md": (), "empty.md": ("title",), + "none.md": ("title", "summary")}, adapter._kind_field + + +def test_the_commit_date_is_read_by_the_home_corpus_s_own_git(tmp_path: Path) -> None: + """`generated_at` is read with the git the home corpus runs, not a bare + `git` from PATH, so the date and the revision it stamps come from one + git (Copilot at openDox-code#57 03e06ccd, r4139523258).""" + from opendox import default_generator + + root = _repository(tmp_path, files={"a.md": "title: A\nsummary: S\n"}) + real = shutil.which("git") + log = tmp_path / "calls.log" + wrapper = tmp_path / "logging-git" + wrapper.write_text(f'#!/bin/sh\necho "$@" >> "{log}"\nexec "{real}" "$@"\n', + encoding="utf-8") + wrapper.chmod(0o755) + corpus_adapter.register_home(lambda location: ( + lga.WorkingTreeCorpus(executable=str(wrapper)), + corpus_adapter.CorpusRef(name="home", location=str(location)))) + snapshot = default_generator.generate(root, "garden") + assert snapshot["generation"].get("generated_at"), "a date was read" + assert "--format=%cI" in log.read_text(encoding="utf-8"), ( + "the commit date was read by the home corpus's own git") + + def test_required_header_fields_answers_the_neutral_set_through_the_entry_point() -> None: from opendox import cli From d7954fc604dc5fd46ef7fc0c995220a59a3a5a23 Mon Sep 17 00:00:00 2001 From: Brett Heap <1513478+brettheap@users.noreply.github.com> Date: Wed, 30 Sep 2026 00:22:56 +0000 Subject: [PATCH 31/34] T054: the projection reads keys as paths only where the adapter declares so Copilot at 50b0d42b (r4139607560) is right. DocumentId.key is opaque to the CorpusAdapter interface: an implementation may make it an object id or a row key. The neutral projection read every key as a repository path and every document's leading block as its header, so a valid non-git home adapter would have its keys written into the snapshot as paths. The projection now names the requirement. An adapter declares `document_keys_are_paths = True` (neutral_projection.PATH_KEYS) for its keys to be read as repository paths and its documents' leading `Name: value` block as their header. LocalGitCorpus declares it, so WorkingTreeCorpus and the conformance corpus do too. project() refuses any other adapter with ProjectionRefused, naming the declaration, before it reads a document. The same review's r4139607585 does not reproduce: a path reaches the invalid-stage notice only after _unwritable() has passed it, and that refuses any control character (U+0000 to U+001F, U+007F to U+009F). Such a path is named by its repr in the "left out" notice and never reaches the stage check. The regression case is test_an_adapter_whose_keys_are_not_declared_paths_is_refused. It fails against 50b0d42b's projection. The whole suite: selected=2621 passed=2610 skipped=11, which is +1. F4.1's scan is unchanged. Arc: neutral-product-standalone-operability Lane: openxfactory-4 (openXfactory-4-openDox_extraction) Co-Authored-By: Claude Opus 5.5 (1M context) --- src/opendox/neutral_projection.py | 18 +++++++++++++ src/opendox/runtime/local_git_adapter.py | 7 +++++ tests/test_neutral_projection.py | 33 ++++++++++++++++++++++++ 3 files changed, 58 insertions(+) diff --git a/src/opendox/neutral_projection.py b/src/opendox/neutral_projection.py index d0addd82..1d9e1ed3 100644 --- a/src/opendox/neutral_projection.py +++ b/src/opendox/neutral_projection.py @@ -94,6 +94,7 @@ __all__ = [ "GENERATOR_VERSION", "Notice", + "PATH_KEYS", "Projection", "ProjectionRefused", "SCHEMA_VERSION", @@ -162,6 +163,13 @@ _CONTROL = re.compile("[\u0000-\u001f\u007f-\u009f]") +#: What an adapter declares, as a class or instance attribute set to True, +#: for this projection to read its `DocumentId.key` as a repository path and +#: its documents' leading `Name: value` block as their header. `LocalGitCorpus` +#: declares it. An adapter that does not is refused, not guessed at. +PATH_KEYS = "document_keys_are_paths" + + class ProjectionRefused(generator_seam.GeneratorSeamError): """The projection cannot write a snapshot the neutral contract admits. @@ -420,6 +428,16 @@ def project(adapter: CorpusAdapter, corpus: ResolvedCorpus, repository: str, the projection refuses, since the contract requires an anchor. `generated_at`, where given, is recorded verbatim, and it is otherwise absent. `repository` is recorded as given.""" + if getattr(adapter, PATH_KEYS, False) is not True: + raise ProjectionRefused( + f"the neutral snapshot records each document by its repository " + f"path, and {type(adapter).__name__} does not declare " + f"`{PATH_KEYS} = True`. A `DocumentId.key` is opaque to the " + "CorpusAdapter interface: it may be an object id or a row key, and " + "read as a path it would fabricate one. Register a home corpus " + "whose keys are repository paths and whose documents open with a " + "`Name: value` header, and have it declare so (Copilot at " + "openDox-code#57 50b0d42b, r4139607560).") revision = source_revision if source_revision is not None else corpus.revision if revision is None: raise ProjectionRefused( diff --git a/src/opendox/runtime/local_git_adapter.py b/src/opendox/runtime/local_git_adapter.py index 83836400..54472eff 100644 --- a/src/opendox/runtime/local_git_adapter.py +++ b/src/opendox/runtime/local_git_adapter.py @@ -1606,6 +1606,13 @@ class LocalGitCorpus: #: (Copilot review of openDox-code#26). A parameter that cannot change an #: outcome is removed rather than wired up, because wiring it up would #: reintroduce the assumption `_served_ref` exists to refuse. + #: THIS ADAPTER'S KEYS ARE REPOSITORY PATHS, and its documents open with a + #: leading `Name: value` header. `DocumentId.key` is opaque to the + #: `CorpusAdapter` interface, so a consumer that needs a path, openDox's + #: neutral projection, reads keys as paths only where the adapter says so + #: (`neutral_projection.PATH_KEYS`). + document_keys_are_paths = True + def __init__(self, *, executable: str = "git", write_path: str | None = WRITE_PATH, kind_field: str | None = None, diff --git a/tests/test_neutral_projection.py b/tests/test_neutral_projection.py index e4073e71..84ce7093 100644 --- a/tests/test_neutral_projection.py +++ b/tests/test_neutral_projection.py @@ -835,6 +835,39 @@ def test_the_anchors_come_from_the_source_revision_and_the_bytes_repeat( assert "generated_at" not in unknown +def test_an_adapter_whose_keys_are_not_declared_paths_is_refused(tmp_path: Path) -> None: + """`DocumentId.key` is opaque to the `CorpusAdapter` interface, so the + projection reads keys as repository paths only for an adapter that + declares `PATH_KEYS`. One that does not is refused, naming the + declaration, rather than having an object id or a row key written as a + path (Copilot at openDox-code#57 50b0d42b, r4139607560).""" + root = _repository(tmp_path, files={"a.md": "title: A\nsummary: S\n"}) + assert projection.PATH_KEYS == "document_keys_are_paths" + assert lga.LocalGitCorpus.document_keys_are_paths is True + + class _Opaque: + """Delegates everything to a path adapter, and declares nothing.""" + def __init__(self, inner): + self._inner = inner + + def __getattr__(self, name): + if name == projection.PATH_KEYS: + raise AttributeError(name) + return getattr(self._inner, name) + + inner = lga.WorkingTreeCorpus() + corpus = inner.resolve(corpus_adapter.CorpusRef(name="home", location=str(root))) + with pytest.raises(projection.ProjectionRefused) as caught: + projection.project(_Opaque(inner), corpus, "garden") + assert "document_keys_are_paths = True" in str(caught.value) + assert isinstance(caught.value, gs.GeneratorSeamError) + declared = _Opaque(inner) + declared.document_keys_are_paths = True + paths = [d["path"] for d in + projection.project(declared, corpus, "garden").snapshot["documents"]] + assert paths == ["a.md"] + + def test_a_repository_with_no_commit_needs_a_pinned_revision(tmp_path: Path) -> None: root = _repository(tmp_path, files={"a.md": "# A\n"}, commit=False) with pytest.raises(projection.ProjectionRefused) as caught: From 3351f6a72139f0455556b7f7b9e12a9a8c097322 Mon Sep 17 00:00:00 2001 From: Brett Heap <1513478+brettheap@users.noreply.github.com> Date: Wed, 30 Sep 2026 00:46:40 +0000 Subject: [PATCH 32/34] T057: a const or enum that holds a value JSON has not is refused at build Copilot at cb40b977 (r4139739110) is right. const and enum were checked only for a value that contains itself, and YAML also builds sets, bytes, dates, mappings with keys that are not text, and non-finite numbers. So `const: !!set {a: null}` built and accepted an equal set, against this module's fail-closed claim for a malformed copy. _first_non_json() now walks the value, without recursing, after the cycle test. A value that is not text, a finite number, true, false, null, or a list or text-keyed mapping of those makes the build refuse with SchemaNotEvaluable, naming what it found. The four packaged copies hold none, and all six kinds build as before. The same review's r4139739167 does not reproduce. It asks for the six-stations example's keyword_index `water` count to be 2, but four documents carry `water` (notes/rain-barrels.md, notes/watering.md, plans/rain-harvest.md, submissions/rain-harvest-build.md), and the validator finds no violation in the example. The file is also byte for byte openDox-spec's at f7ee3c76, so it is not this leg's to edit. The regression case is test_a_const_or_enum_that_holds_a_value_json_has_not_is_refused, over a set, bytes, a date, a non-text key, inf and nan (6 cases). All 6 fail against cb40b977's validator. The whole suite: selected=2846 passed=2835 skipped=11, which is +6. Arc: neutral-product-standalone-operability Lane: openxfactory-4 (openXfactory-4-openDox_extraction) Co-Authored-By: Claude Opus 5.5 (1M context) --- src/opendox/validator.py | 36 ++++++++++++++++++++++++++++++++++++ tests/test_validator.py | 23 +++++++++++++++++++++++ 2 files changed, 59 insertions(+) diff --git a/src/opendox/validator.py b/src/opendox/validator.py index 76658cdc..c2f24e42 100644 --- a/src/opendox/validator.py +++ b/src/opendox/validator.py @@ -421,6 +421,32 @@ def _holds_itself(value: Any) -> bool: return False +def _first_non_json(value: Any) -> str | None: + """What in `value` is not a JSON value, or None where all of it is: text, + a whole or finite number, true, false, null, and lists and mappings of + them whose keys are text. Walked without recursing; the caller has + already refused a value that contains itself.""" + work: list[Any] = [value] + while work: + node = work.pop() + if node is None or isinstance(node, (bool, str)): + continue + if _is_number(node): + if not _is_bound(node): + return f"the non-finite number {node!r}" + continue + if isinstance(node, list): + work.extend(node) + elif isinstance(node, dict): + for key, child in node.items(): + if not isinstance(key, str): + return f"a mapping key that is not text ({_brief(key)})" + work.append(child) + else: + return f"a {type(node).__name__} ({_brief(node)})" + return None + + _DATE_TIME = re.compile( r"([0-9]{4})-(0[1-9]|1[0-2])-([0-9]{2})T(?:[01][0-9]|2[0-3]):[0-5][0-9]:[0-5][0-9]" r"(?:\.[0-9]+)?(?:Z|[+-](?:[01][0-9]|2[0-3]):[0-5][0-9])") @@ -807,6 +833,16 @@ def _refuse_node(self, document: dict[str, Any], at: str, raise self._not_evaluable( f"{where}'s {keyword} holds a value that contains itself, " "which no JSON value does") + # AND ONLY JSON VALUES (Copilot at openDox-code#58 cb40b977, + # r4139739110). YAML builds sets, bytes and dates, which no JSON + # value is, and a `const: !!set {a: null}` would otherwise build + # and accept an equal set. Checked after the cycle test, so the + # walk ends. + stray = _first_non_json(node[keyword]) if keyword in node else None + if stray is not None: + raise self._not_evaluable( + f"{where}'s {keyword} holds {stray}, which is not a JSON " + "value") for keyword, value in node.items(): shape = _SHAPES.get(keyword) if shape is not None and not shape[0](value): diff --git a/tests/test_validator.py b/tests/test_validator.py index c28540b4..52143c6f 100644 --- a/tests/test_validator.py +++ b/tests/test_validator.py @@ -444,6 +444,29 @@ def test_a_const_or_enum_that_contains_itself_is_refused() -> None: assert "contains itself, which no JSON value does" in str(refused.value) +@pytest.mark.parametrize("text,what", [ + ("!!set {a: null}", "a set"), + ("!!binary aGk=", "a bytes"), + ("2026-09-30", "a date"), + ("[1, {2: x}]", "a mapping key that is not text"), + ("{a: [.inf]}", "the non-finite number inf"), + ("[.nan]", "the non-finite number nan"), +], ids=["set", "binary", "date", "int-key", "inf", "nan"]) +def test_a_const_or_enum_that_holds_a_value_json_has_not_is_refused(text, what) -> None: + """YAML builds values JSON has not: sets, bytes, dates, non-text keys and + non-finite numbers. A `const` or `enum` holding one is refused when built, + anywhere inside the value (Copilot at openDox-code#58 cb40b977, + r4139739110). Before, `const: !!set {a: null}` built and accepted an + equal set. A JSON value, nested, still builds.""" + value = yaml.safe_load(text) + for schema in ({"const": value}, {"enum": ["ok", value]}): + with pytest.raises(V.SchemaNotEvaluable) as refused: + _built({"properties": {"a": schema}}) + assert what in str(refused.value) and "is not a JSON value" in str(refused.value) + nested = yaml.safe_load("{a: [1, 2.5, true, null, {b: text}]}") + assert _found({"const": nested}, nested) == set() + + def test_values_of_any_depth_are_compared_without_recursing() -> None: """JSON equality is judged on a flat canon, built without recursing. So neither a deep schema value nor a deep instance exhausts Python's stack From 2b8ad2458510b1608674649d6d90a1f0b61212b5 Mon Sep 17 00:00:00 2001 From: Brett Heap <1513478+brettheap@users.noreply.github.com> Date: Wed, 30 Sep 2026 00:55:38 +0000 Subject: [PATCH 33/34] T057: a packaged file that is present but unreadable is refused, not raised The T058 writer measured this, from Copilot on openDox-code#68 (r4139734412). contracts._read_package_file turned only FileNotFoundError, IsADirectoryError and NotADirectoryError into CopyRefused. A record or copy that is present and cannot be read raised PermissionError straight out of validator_for(): with copies.yaml at mode 000, `generate --strict` exited 1 with a traceback. Every OSError there is now a CopyRefused. A missing file keeps its message. Any other names the file as present and unreadable, with the error's class and reason. So the validator reports itself unavailable, as it does for a missing file. The regression case is test_a_present_file_that_cannot_be_read_is_refused_not_raised, over the record and the neutral contract's copy, each at mode 000 in a copied package (skipped as root, who reads it anyway). Both cases fail against 3351f6a7's reader. The whole suite: selected=2848 passed=2837 skipped=11, which is +2. Arc: neutral-product-standalone-operability Lane: openxfactory-4 (openXfactory-4-openDox_extraction) Co-Authored-By: Claude Opus 5.5 (1M context) --- src/opendox/contracts/__init__.py | 11 +++++++++++ tests/test_validator_input_set.py | 28 ++++++++++++++++++++++++++++ 2 files changed, 39 insertions(+) diff --git a/src/opendox/contracts/__init__.py b/src/opendox/contracts/__init__.py index 7a20bdfb..823e7a30 100644 --- a/src/opendox/contracts/__init__.py +++ b/src/opendox/contracts/__init__.py @@ -140,12 +140,23 @@ def _refuse(detail: str) -> CopyRefused: def _read_package_file(name: str) -> bytes: + """`name`'s bytes from the package, or `CopyRefused`. + + EVERY `OSError` IS A REFUSAL (the T058 writer's measurement, from Copilot + at openDox-code#68, r4139734412). A file that is absent is refused as + missing. One that is present and cannot be read, such as a record at mode + 000, raised `PermissionError` straight out of `validator_for()`, so + `generate --strict` ended in a traceback where it owes a refusal.""" try: return resources.files(__name__).joinpath(name).read_bytes() except (FileNotFoundError, IsADirectoryError, NotADirectoryError) as exc: raise CopyRefused( f"opendox.contracts has no {name}: the package was built or " f"installed without it ({type(exc).__name__})") from exc + except OSError as exc: + raise CopyRefused( + f"opendox.contracts has {name}, and it cannot be read " + f"({type(exc).__name__}: {exc.strerror or exc})") from exc def record() -> Record: diff --git a/tests/test_validator_input_set.py b/tests/test_validator_input_set.py index bd7097d6..763bae43 100644 --- a/tests/test_validator_input_set.py +++ b/tests/test_validator_input_set.py @@ -39,6 +39,9 @@ from __future__ import annotations import hashlib +import os +import shutil +import types from pathlib import Path import pytest @@ -207,6 +210,31 @@ def read(name: str) -> bytes: monkeypatch.setattr(contracts, "_read_package_file", read) +@pytest.mark.skipif(hasattr(os, "geteuid") and os.geteuid() == 0, + reason="root reads a file at mode 000") +@pytest.mark.parametrize("name", ["copies.yaml", "schemas/opendox-snapshot.schema.yaml"]) +def test_a_present_file_that_cannot_be_read_is_refused_not_raised( + name: str, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + """A packaged record or copy that is present but unreadable is refused as + `CopyRefused`, so the validator reports itself unavailable, as for a + missing one. It raised `PermissionError` out of `validator_for()`, and + `generate --strict` ended in a traceback (the T058 writer's measurement, + from Copilot at openDox-code#68, r4139734412).""" + package = tmp_path / "contracts" + shutil.copytree(PACKAGE, package) + (package / name).chmod(0) + monkeypatch.setattr(contracts, "resources", + types.SimpleNamespace(files=lambda _name: package)) + try: + with pytest.raises(contracts.CopyRefused) as refused: + contracts.load("opendox-snapshot") + assert "cannot be read (PermissionError" in str(refused.value) + with pytest.raises(validator.ValidatorUnavailable): + validator.validator_for("opendox-snapshot") + finally: + (package / name).chmod(0o644) + + def _record_with(**changes) -> bytes: data = yaml.safe_load((PACKAGE / "copies.yaml").read_text(encoding="utf-8")) data.update(changes) From 753ffa197d778a1b9fdbd874647291ed62ea1c37 Mon Sep 17 00:00:00 2001 From: Brett Heap <1513478+brettheap@users.noreply.github.com> Date: Wed, 30 Sep 2026 01:05:18 +0000 Subject: [PATCH 34/34] T057: an instance key of any size is named in a pointer and a report, not crashed on Copilot at 2b8ad245 raised three threads. Two of them are real, measured with an instance key of 10**5000. An int past 4300 digits has no decimal text: - r4139823704: Violation.where ran str() over each path part, so a violation beneath such a key raised ValueError. - r4139823779: the unexpected-properties report ordered the keys with repr and showed the list with _brief, so that report raised too. Now: - A pointer part that str() cannot give is shown as _shown shows it, ``. - Extra keys are ordered by _shown and shown item by item (_brief_items), so one key too large to show is named by its size and the rest still read as themselves. - The schema side's unknown-keyword ordering uses _shown too. - An ordinary key, pointer or report reads exactly as before. r4139823750 does not reproduce. _canon_scalar answers None only for a list or a mapping, and neither can be a key. A tuple key canonicalizes as an opaque value, never equal to a JSON one. Measured: - enum over {('non-text',): 1} gives one enum violation; - uniqueItems over two such mappings gives one uniqueItems violation; - neither crashes. Tests, in tests/test_validator.py: - test_an_instance_key_of_any_size_is_named_not_crashed_on fails against 2b8ad245's validator. - test_a_non_text_key_that_is_not_a_scalar_is_judged_as_itself is the guard for the claim that does not reproduce. The whole suite: selected=2850 passed=2839 skipped=11, which is +2. Arc: neutral-product-standalone-operability Lane: openxfactory-4 (openXfactory-4-openDox_extraction) Co-Authored-By: Claude Opus 5.5 (1M context) --- src/opendox/validator.py | 32 ++++++++++++++++++++++++++++---- tests/test_validator.py | 28 ++++++++++++++++++++++++++++ 2 files changed, 56 insertions(+), 4 deletions(-) diff --git a/src/opendox/validator.py b/src/opendox/validator.py index c2f24e42..16c97c56 100644 --- a/src/opendox/validator.py +++ b/src/opendox/validator.py @@ -243,8 +243,12 @@ class Violation: @property def where(self) -> str: - """`path` as a JSON pointer ("" is the instance itself).""" - return "".join("/" + str(part).replace("~", "~0").replace("/", "~1") + """`path` as a JSON pointer ("" is the instance itself). A part that + is not text or an index is shown as `_shown` shows it, so a key of any + size is named and never crashes the pointer (an int past 4300 digits + has no decimal text; Copilot at openDox-code#58 2b8ad245, + r4139823704).""" + return "".join("/" + _part_text(part).replace("~", "~0").replace("/", "~1") for part in self.path) # jsonschema's names for the same fields, for the doxBench seam's readers. @@ -294,11 +298,31 @@ def _shown(value: Any) -> str: return f"" +def _part_text(part: Any) -> str: + """A path part as pointer text: text as it is, anything else as `str()` + gives it, or as `_shown` does where `str()` cannot.""" + if isinstance(part, str): + return part + try: + return str(part) + except (RecursionError, ValueError): + return _shown(part) + + def _brief(value: Any) -> str: text = _shown(value) return text if len(text) <= _BRIEF else text[:_BRIEF - 3] + "..." +def _brief_items(values: list[Any]) -> str: + """A list shown item by item, each as `_shown` shows it, and cut as + `_brief` cuts. So one key too large to show is named by its size, and the + others still read as themselves (Copilot at openDox-code#58 2b8ad245, + r4139823779). An ordinary list reads exactly as `_brief` shows it.""" + text = "[" + ", ".join(_shown(value) for value in values) + "]" + return text if len(text) <= _BRIEF else text[:_BRIEF - 3] + "..." + + def _is_type(value: Any, name: str) -> bool: if name == "object": return isinstance(value, dict) @@ -818,7 +842,7 @@ def _refuse_node(self, document: dict[str, Any], at: str, """Refuse `node` unless this module evaluates it as it stands, and answer its reference's target, for the walk to check in its turn.""" where = at or "" - unknown = sorted((key for key in node if key not in KEYWORDS), key=repr) + unknown = sorted((key for key in node if key not in KEYWORDS), key=_shown) if unknown: raise self._not_evaluable(f"{where} uses {unknown}, which " "this module does not evaluate") @@ -1050,7 +1074,7 @@ def _object(self, value: dict[str, Any], schema: dict[str, Any], if extra: yield broken("additionalProperties", f"unexpected properties " - f"{_brief(sorted(extra, key=repr))}") + f"{_brief_items(sorted(extra, key=_shown))}") else: for key in extra: yield from self._evaluate(value[key], extra_schema, path + (key,)) diff --git a/tests/test_validator.py b/tests/test_validator.py index 52143c6f..75bc826f 100644 --- a/tests/test_validator.py +++ b/tests/test_validator.py @@ -424,6 +424,34 @@ def _deep(levels: int) -> list[Any]: return value +def test_an_instance_key_of_any_size_is_named_not_crashed_on() -> None: + """A mapping key that is an int past 4300 digits has no decimal text, so + naming it in a violation's pointer, or ordering it in the report of + unexpected properties, raised ValueError (Copilot at openDox-code#58 + 2b8ad245, r4139823704 and r4139823779). Both now show its size. An + ordinary key reads exactly as before.""" + huge = 10 ** 5000 + shown = f"" + below = _built({"additionalProperties": {"type": "string"}}).violations({huge: 1}) + assert [(v.keyword, v.where) for v in below] == [("type", f"/{shown}")] + extra = _built({"additionalProperties": False}).violations({huge: 1, "b": 2}) + assert [v.keyword for v in extra] == ["additionalProperties"] + assert shown in V.report(extra)[0] and "'b'" in V.report(extra)[0] + assert _found({"additionalProperties": {"type": "string"}}, {7: 1, "a/b": 2}) == { + ("type", "/7"), ("type", "/a~1b")} + + +def test_a_non_text_key_that_is_not_a_scalar_is_judged_as_itself() -> None: + """A tuple key is canonicalized as an opaque value, never equal to a JSON + one, so `const`, `enum` and `uniqueItems` judge it without crashing + (Copilot at openDox-code#58 2b8ad245, r4139823750, which does not + reproduce: this is the guard that it stays so).""" + odd = {("non-text",): 1} + assert _found({"enum": [{"a": 1}]}, odd) == {("enum", "")} + assert _found({"const": {"non-text": 1}}, odd) == {("const", "")} + assert _found({"uniqueItems": True}, [odd, {("non-text",): 1}]) == {("uniqueItems", "")} + + def test_an_integer_bound_of_any_size_is_evaluated() -> None: """YAML gives an integer of up to 4300 digits, and a JSON number has no bound. `math.isfinite()` could not convert one past a float's range, so