From e406a006f7334c86c1ff84b3256c0a32d982ead4 Mon Sep 17 00:00:00 2001 From: Brett Heap <1513478+brettheap@users.noreply.github.com> Date: Mon, 28 Sep 2026 18:50:14 +0000 Subject: [PATCH 01/88] =?UTF-8?q?T071:=2013.2=20and=2013.3=20=E2=80=94=20l?= =?UTF-8?q?oad=5Fsettings=20refuses=20a=20non-PostgreSQL=20DSN=20and=20a?= =?UTF-8?q?=20collapsed=20DSN=20pair=20(plan=20034)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Realizes #1144 13.2 and 13.3, falsifier F13.1's `load_settings` block. - 13.2: `_refuse_non_postgresql_dsn` refuses either DSN (`OPENDOX_DATABASE_URL` or `OPENDOX_MIGRATION_DATABASE_URL`) whose URI scheme is not `postgresql://` or `postgres://`, naming the setting and the dialect kept. The keyword/value conninfo form (`host=h dbname=d …`) names no dialect at all and is unaffected — that syntax is libpq's own grammar, and no other driver reads it. - 13.3: `OPENDOX_MIGRATION_DATABASE_URL` stops being optional in `load_settings` (the `Setting` row's `required` flag, `_require` in place of `_optional`, and `RuntimeSettings.migration_database_url`'s type). A new `_refuse_the_same_dsn_in_both_settings` refuses the two DSNs being the exact same STRING, naming `OPENDOX_MIGRATION_DATABASE_URL`, once they are already known to agree on where they land (`_refuse_two_dsns_that_select_different_schemas`, unchanged, now called first): two DIFFERENT secrets for one role still pass, as the existing "single-role install" case documents. - Explicitly NOT in this task: 13.4-13.6 (`OPENDOX_INSTALL_MODE`, T070). Nothing here reads or names that setting, and `load_settings`'s only new required input is the migration DSN itself. Every existing call site that built an environment without `OPENDOX_MIGRATION_DATABASE_URL` needed one once it became required: `tests_runtime/conftest.py` gains a `migration_dsn` fixture (a `postgres_dsn` distinguished by a URI fragment, invisible to every DSN reader this module has); `test_api_endpoints.py`, `test_migrations_apply.py`, `test_runtime_cli.py` and `test_runtime_surface.py` thread it or a literal peer through. `test_two_dsns_that_select_different_schemas_are_refused`'s "a migration DSN that is simply absent" case is rewritten from accepted to refused, which is the behavior 13.3 changes. Two new tests (`test_a_non_postgresql_dsn_is_refused_naming_the_dialect_kept`, `test_the_same_dsn_in_both_settings_is_refused_naming_the_migration_one`) cover the two new refusals directly. Measured locally against this change (own Postgres container, bridge IP — this sandbox's host-mapped loopback ports are unreachable): `python -m pytest -q` reports 2469 passed, 11 skipped, 1 failed — the one failure is `tests/test_model_provider_broker.py::test_the_broker_child_inherits_no_ credential_shaped_environment`, already red against unmodified `main` (2d116415) in the same environment (an `LC_CTYPE` ambient in this sandbox, unrelated to runtime/config.py). Against `main`'s own reading (2479 selected / 2468 passed / 11 skipped, matching this repo's last recorded CI triple), this change is +2/+2/+0 for the two new tests — `validate.yml`'s `Pin the triple` floors (`MIN_SELECTED=2476`, `MIN_PASSED=2465`, `EXPECT_SKIPPED=11`) permit the rise unchanged. Arc: neutral-product-standalone-operability Lane: openxfactory-4 (openXfactory-4-openDox_extraction) Co-Authored-By: Claude Opus 5.5 (1M context) --- src/opendox/runtime/config.py | 104 ++++++++++++++++++--- tests_runtime/conftest.py | 31 ++++++- tests_runtime/test_api_endpoints.py | 18 +++- tests_runtime/test_migrations_apply.py | 7 ++ tests_runtime/test_runtime_cli.py | 124 ++++++++++++++++++++++++- tests_runtime/test_runtime_surface.py | 2 + 6 files changed, 264 insertions(+), 22 deletions(-) diff --git a/src/opendox/runtime/config.py b/src/opendox/runtime/config.py index 5a832e12..046a46ab 100644 --- a/src/opendox/runtime/config.py +++ b/src/opendox/runtime/config.py @@ -81,7 +81,7 @@ class Setting: "it must not be the identity migrations run as", ), Setting( - PREFIX + "MIGRATION_DATABASE_URL", None, False, True, + PREFIX + "MIGRATION_DATABASE_URL", None, True, True, "the PRIVILEGED DSN ordered-SQL migrations are applied with, used by " "`opendox runtime migrate` alone and never by the served application", ), @@ -180,10 +180,14 @@ class RuntimeSettings: """The resolved configuration of one runtime process. Construct with :func:`load_settings`; the fields are in `SETTINGS` order. + + `migration_database_url` is never `None` from either loader (plan 034, + 13.3): `load_settings` requires it exactly as it requires `database_url`, + and `load_migration_settings` already refused to load without one. """ database_url: str - migration_database_url: str | None + migration_database_url: str oidc_issuer: str oidc_audience: str oidc_jwks_url: str | None @@ -1288,6 +1292,63 @@ def effective_schema(dsn: str) -> str | None: return user_named_by(dsn) +#: THE ONLY DIALECT THIS RUNTIME KEEPS (plan 034, 13.2). `psycopg` is the one +#: driver `runtime` depends on and it speaks PostgreSQL alone, but a DSN is a +#: string and nothing stopped an operator writing `sqlite:///…` into either +#: setting and discovering the mismatch however far the code got before the +#: driver refused it. RULING Q1 keeps this database DOCUMENT-FREE, which is +#: why a second dialect is refused HERE rather than supported: it would double +#: every migration and every schema test forever, for a database that holds no +#: document. `postgres://` is accepted beside `postgresql://` because libpq +#: treats the two as one scheme. +POSTGRESQL_SCHEMES = frozenset({"postgresql", "postgres"}) + + +def _refuse_non_postgresql_dsn(name: str, dsn: str) -> None: + """`name`'s DSN is refused unless it selects a PostgreSQL scheme. + + A DSN in the keyword/value form (`host=h dbname=d …`) names NO DIALECT AT + ALL — that syntax is libpq's own conninfo grammar, and no other driver + reads it — so only the URI form is checked: `urlsplit` reports an EMPTY + scheme for the keyword/value form (there is no `://` to split on), and an + empty scheme is read as "says nothing" here, exactly as `schema_selected_by` + reads a DSN that names no schema as `None` rather than as a refusal. + """ + scheme = urllib.parse.urlsplit(dsn).scheme + if scheme and scheme not in POSTGRESQL_SCHEMES: + raise ConfigurationError( + f"{name} names the {scheme!r} dialect. PostgreSQL " + "(`postgresql://` or `postgres://`) is the only dialect this " + "runtime keeps: a second one would double every migration and " + "every schema test forever, for a database that holds no " + "document (RULING Q1)") + + +def _refuse_the_same_dsn_in_both_settings(served: str, migration: str) -> None: + """One credential pasted into both settings is refused (plan 034, 13.3). + + `OPENDOX_DATABASE_URL` is the least-privileged identity the API serves + with; `OPENDOX_MIGRATION_DATABASE_URL` is the privileged one ordered-SQL + migrations run as — the whole point of keeping two settings. A + single-user install is not a reason to collapse them into one: this is + the two settings simply BEING each other, which is different from + `_refuse_two_dsns_that_select_different_schemas` below, where they + DISAGREE about where they land. It is different too from the accepted + "single-role install" (`test_a_dsn_that_names_no_database_still_reaches_ + one`): two DSNs for the same ROLE with two DIFFERENT secrets are two + credentials, not one pasted twice, and this checks the value actually + given, not the identity it happens to resolve to. + """ + if served == migration: + raise ConfigurationError( + f"{PREFIX}MIGRATION_DATABASE_URL is the same value as " + f"{PREFIX}DATABASE_URL. The identity migrations run as must not " + "also be the identity the API serves with; give the migration " + "credential its own DSN, even where both reach the same " + "database (the values are not repeated: a DSN carries a " + "password)") + + def _refuse_two_dsns_that_select_different_schemas( served: str, migration: str | None) -> None: """Both DSNs must land in one schema, or neither answer means anything. @@ -1400,21 +1461,42 @@ def load_settings(env: Mapping[str, str] | None = None) -> RuntimeSettings: accept `HS256` would verify a token signed with the public key anybody can fetch from the broker's JWKS, and discovering that on the first request means it is already serving. + + BOTH DSNs ARE REQUIRED (plan 034, 13.3): `OPENDOX_MIGRATION_DATABASE_URL` + used to default to `None`, so the collapsed, single-role shape this + refuses (below) was reachable only by accident, through a caller who + happened to set both to the same value — an install that left the + migration credential unset was never asked the question at all. Naming it + explicitly, even at the same database a served DSN already names, is what + keeps the two identities two DECISIONS rather than one remembered twice. """ env = os.environ if env is None else env algorithms = _algorithms(env) - # AND THE TWO DSNs LAND IN ONE SCHEMA. See - # `_refuse_two_dsns_that_select_different_schemas`: this is the half of - # that invariant a string can answer, and it is asked here because this is - # the one loader that holds BOTH values. - _refuse_two_dsns_that_select_different_schemas( - _require(env, _by_name(PREFIX + "DATABASE_URL")), - _optional(env, _by_name(PREFIX + "MIGRATION_DATABASE_URL"))) + served = _require(env, _by_name(PREFIX + "DATABASE_URL")) + migration = _require(env, _by_name(PREFIX + "MIGRATION_DATABASE_URL")) + # THE DIALECT FIRST: a scheme this module cannot parse as PostgreSQL is not + # yet a DSN worth comparing at all. + _refuse_non_postgresql_dsn(PREFIX + "DATABASE_URL", served) + _refuse_non_postgresql_dsn(PREFIX + "MIGRATION_DATABASE_URL", migration) + # THEN WHETHER THEY DISAGREE. See `_refuse_two_dsns_that_select_different_ + # schemas`: this is the half of that invariant a string can answer, and it + # is asked here because this is the one loader that holds BOTH values. A + # DSN compared against ITSELF can never disagree, so this step passes + # silently on exactly the pair the next one exists to catch. + _refuse_two_dsns_that_select_different_schemas(served, migration) + # AND, LAST, WHETHER THEY ARE SIMPLY EACH OTHER. Two DSNs that agree on + # where they land are ordinarily two credentials for the one database + # (`test_a_dsn_that_names_no_database_still_reaches_one`'s "single-role + # install" is exactly that, two DIFFERENT secrets for one role) — but + # agreement bought by pasting the SAME value into both settings is not a + # second decision at all, and this is the check the one before it cannot + # make. + _refuse_the_same_dsn_in_both_settings(served, migration) return RuntimeSettings( - database_url=_require(env, _by_name(PREFIX + "DATABASE_URL")), - migration_database_url=_optional(env, _by_name(PREFIX + "MIGRATION_DATABASE_URL")), + database_url=served, + migration_database_url=migration, oidc_issuer=_broker_url(env, _by_name(PREFIX + "OIDC_ISSUER"), required=True, is_a_base_url=True) or "", oidc_audience=_require(env, _by_name(PREFIX + "OIDC_AUDIENCE")), diff --git a/tests_runtime/conftest.py b/tests_runtime/conftest.py index 2c6e0b7b..93f53a4d 100644 --- a/tests_runtime/conftest.py +++ b/tests_runtime/conftest.py @@ -219,6 +219,29 @@ def postgres_dsn() -> str: return dsn +@pytest.fixture(scope="session") +def migration_dsn(postgres_dsn: str) -> str: + """A DSN for `OPENDOX_MIGRATION_DATABASE_URL`, distinct from `postgres_dsn`. + + Plan 034, 13.3: `load_settings` now REQUIRES this setting and refuses a + value that is simply `postgres_dsn` repeated (13.3's own collapse + refusal), so every fixture built from the pair below needs a second, + genuinely different string — not a second database. None of this + module's fixtures apply migrations through this identity: `database` + applies them directly with `MigrationRunner`, and the served app under + test is never asked to `migrate`. So `load_settings` is the only reader + that cares about this value at all, and what it asks of a DSN is a + PostgreSQL scheme (13.2), a STRING distinct from `postgres_dsn` (13.3), + and a database and schema that agree with it + (`_refuse_two_dsns_that_select_different_schemas`). A URI FRAGMENT is + invisible to `database_named_by`, `user_named_by` and `schema_selected_by` + alike — none of the three inspects `urlsplit(...).fragment` — so + appending one changes the STRING without moving the identity those + functions compare. + """ + return postgres_dsn + "#opendox-test-migration-identity" + + @pytest.fixture def database(postgres_dsn: str) -> Iterator[object]: """A `Database` on a throwaway schema, with the migrations already applied.""" @@ -357,7 +380,7 @@ def verifier(jwks_path: str): @pytest.fixture() -def client(database, postgres_dsn: str, verifier): +def client(database, postgres_dsn: str, migration_dsn: str, verifier): """A `TestClient` over the REAL application, on this test's own schema. The application is given its OWN `Database` on the same schema rather than @@ -376,6 +399,7 @@ def client(database, postgres_dsn: str, verifier): settings = load_settings({ PREFIX + "DATABASE_URL": postgres_dsn, + PREFIX + "MIGRATION_DATABASE_URL": migration_dsn, PREFIX + "OIDC_ISSUER": TEST_ISSUER, PREFIX + "OIDC_AUDIENCE": TEST_AUDIENCE, }) @@ -399,8 +423,8 @@ def project_repository_root(tmp_path): @pytest.fixture() -def client_with_repositories(database, postgres_dsn: str, verifier, - project_repository_root): +def client_with_repositories(database, postgres_dsn: str, migration_dsn: str, + verifier, project_repository_root): """`client`, with the repository root pointed at this test's own directory.""" fastapi_testclient = _import_fastapi_testclient() from opendox.runtime.app import create_app @@ -409,6 +433,7 @@ def client_with_repositories(database, postgres_dsn: str, verifier, settings = load_settings({ PREFIX + "DATABASE_URL": postgres_dsn, + PREFIX + "MIGRATION_DATABASE_URL": migration_dsn, PREFIX + "OIDC_ISSUER": TEST_ISSUER, PREFIX + "OIDC_AUDIENCE": TEST_AUDIENCE, PREFIX + "PROJECT_REPOSITORY_ROOT": str(project_repository_root), diff --git a/tests_runtime/test_api_endpoints.py b/tests_runtime/test_api_endpoints.py index 21897203..84caccfe 100644 --- a/tests_runtime/test_api_endpoints.py +++ b/tests_runtime/test_api_endpoints.py @@ -425,7 +425,7 @@ def test_the_unauthenticated_surface_is_exactly_the_two_probes( def test_the_schema_viewers_appear_only_when_the_install_says_so( - database, postgres_dsn: str, verifier) -> None: + database, postgres_dsn: str, migration_dsn: str, verifier) -> None: from fastapi.testclient import TestClient from opendox.runtime.app import create_app @@ -435,6 +435,7 @@ def test_the_schema_viewers_appear_only_when_the_install_says_so( settings = load_settings({ PREFIX + "DATABASE_URL": postgres_dsn, + PREFIX + "MIGRATION_DATABASE_URL": migration_dsn, PREFIX + "OIDC_ISSUER": TEST_ISSUER, PREFIX + "OIDC_AUDIENCE": TEST_AUDIENCE, PREFIX + "PUBLISH_OPENAPI": "true", @@ -448,7 +449,7 @@ def test_the_schema_viewers_appear_only_when_the_install_says_so( def test_readiness_refuses_an_unmigrated_database_by_name( - postgres_dsn: str, verifier) -> None: + postgres_dsn: str, migration_dsn: str, verifier) -> None: """`select 1` succeeds against a schema with no tables in it at all. Readiness without a migration check therefore turns a fresh install READY @@ -471,6 +472,7 @@ def test_readiness_refuses_an_unmigrated_database_by_name( try: settings = load_settings({ PREFIX + "DATABASE_URL": postgres_dsn, + PREFIX + "MIGRATION_DATABASE_URL": migration_dsn, PREFIX + "OIDC_ISSUER": TEST_ISSUER, PREFIX + "OIDC_AUDIENCE": TEST_AUDIENCE, }) @@ -588,7 +590,8 @@ def test_draft_pagination_returns_the_principals_drafts_not_an_empty_page( def test_readiness_refuses_a_database_whose_migration_file_has_changed( - database, postgres_dsn: str, verifier, tmp_path) -> None: + database, postgres_dsn: str, migration_dsn: str, verifier, + tmp_path) -> None: """Nothing pending is not the same as matching this tree.""" import shutil @@ -609,6 +612,7 @@ def test_readiness_refuses_a_database_whose_migration_file_has_changed( settings = load_settings({ PREFIX + "DATABASE_URL": postgres_dsn, + PREFIX + "MIGRATION_DATABASE_URL": migration_dsn, PREFIX + "OIDC_ISSUER": TEST_ISSUER, PREFIX + "OIDC_AUDIENCE": TEST_AUDIENCE, PREFIX + "MIGRATIONS_DIR": str(tmp_path), @@ -852,7 +856,8 @@ def test_a_hidden_user_and_a_missing_one_answer_byte_for_byte_the_same( def test_readiness_refuses_a_migrations_directory_without_the_pinned_0001( - database, postgres_dsn: str, verifier, tmp_path) -> None: + database, postgres_dsn: str, migration_dsn: str, verifier, + tmp_path) -> None: """An EMPTY directory made `plan()` and `drift()` both empty. `discover()` returns `[]` for a directory that exists and holds no @@ -874,6 +879,7 @@ def test_readiness_refuses_a_migrations_directory_without_the_pinned_0001( empty.mkdir() settings = load_settings({ PREFIX + "DATABASE_URL": postgres_dsn, + PREFIX + "MIGRATION_DATABASE_URL": migration_dsn, PREFIX + "OIDC_ISSUER": TEST_ISSUER, PREFIX + "OIDC_AUDIENCE": TEST_AUDIENCE, PREFIX + "MIGRATIONS_DIR": str(empty), @@ -1552,7 +1558,8 @@ def transaction(): def test_an_anonymous_token_naming_a_key_of_the_wrong_type_is_401_not_500( - database, postgres_dsn: str, rsa_key_pair, tmp_path: Path) -> None: + database, postgres_dsn: str, migration_dsn: str, rsa_key_pair, + tmp_path: Path) -> None: """The end of A25-2, measured where it was reported: at the HTTP boundary. A realm publishing an RSA and an EC signing key — Keycloak, the moment a @@ -1593,6 +1600,7 @@ def test_an_anonymous_token_naming_a_key_of_the_wrong_type_is_401_not_500( settings = load_settings({ PREFIX + "DATABASE_URL": postgres_dsn, + PREFIX + "MIGRATION_DATABASE_URL": migration_dsn, PREFIX + "OIDC_ISSUER": TEST_ISSUER, PREFIX + "OIDC_AUDIENCE": TEST_AUDIENCE, }) diff --git a/tests_runtime/test_migrations_apply.py b/tests_runtime/test_migrations_apply.py index 2afc0fe1..1b657628 100644 --- a/tests_runtime/test_migrations_apply.py +++ b/tests_runtime/test_migrations_apply.py @@ -345,6 +345,11 @@ def test_status_reports_a_reachable_database_and_its_applied_migrations( # libpq startup parameter `Database(schema=…)` sets for the harness. scoped = (f"{postgres_dsn}?options=-c%20search_path%3D{database.schema}") monkeypatch.setenv(PREFIX + "DATABASE_URL", scoped) + # A DISTINCT STRING (13.3's collapse refusal) THAT SELECTS THE SAME SCHEMA + # (`_refuse_two_dsns_that_select_different_schemas`): a URI FRAGMENT moves + # neither, since neither reader inspects one. + monkeypatch.setenv(PREFIX + "MIGRATION_DATABASE_URL", + scoped + "#opendox-test-migration-identity") monkeypatch.setenv(PREFIX + "OIDC_ISSUER", "https://broker.test/realms/x") monkeypatch.setenv(PREFIX + "OIDC_AUDIENCE", "opendox-runtime") monkeypatch.setenv(PREFIX + "MIGRATIONS_DIR", str(ROOT / "migrations")) @@ -720,6 +725,8 @@ def test_status_calls_an_unmigrated_database_unhealthy_and_blames_the_tree( conn.execute(f"create schema {schema}") try: monkeypatch.setenv(PREFIX + "DATABASE_URL", scoped) + monkeypatch.setenv(PREFIX + "MIGRATION_DATABASE_URL", + scoped + "#opendox-test-migration-identity") monkeypatch.setenv(PREFIX + "OIDC_ISSUER", "https://broker/realms/x") monkeypatch.setenv(PREFIX + "OIDC_AUDIENCE", "opendox-runtime") monkeypatch.setenv(PREFIX + "MIGRATIONS_DIR", str(ROOT / "migrations")) diff --git a/tests_runtime/test_runtime_cli.py b/tests_runtime/test_runtime_cli.py index 4a6d9d13..363caced 100644 --- a/tests_runtime/test_runtime_cli.py +++ b/tests_runtime/test_runtime_cli.py @@ -211,6 +211,8 @@ def test_init_creates_the_project_repository_root_and_touches_no_database( root = tmp_path / "projects" monkeypatch.setenv(PREFIX + "DATABASE_URL", "postgresql://nobody@127.0.0.1:1/none") + monkeypatch.setenv(PREFIX + "MIGRATION_DATABASE_URL", + "postgresql://migrate@127.0.0.1:1/none") monkeypatch.setenv(PREFIX + "OIDC_ISSUER", "https://broker/realms/x") monkeypatch.setenv(PREFIX + "OIDC_AUDIENCE", "opendox-runtime") monkeypatch.setenv(PREFIX + "PROJECT_REPOSITORY_ROOT", str(root)) @@ -362,6 +364,8 @@ def test_status_reports_every_declared_setting_and_none_as_null( from opendox.runtime.config import SETTING_NAMES monkeypatch.setenv(PREFIX + "DATABASE_URL", "postgresql://u:pw@127.0.0.1:1/x") + monkeypatch.setenv(PREFIX + "MIGRATION_DATABASE_URL", + "postgresql://m:pw@127.0.0.1:1/x") monkeypatch.setenv(PREFIX + "OIDC_ISSUER", "https://broker/realms/x") monkeypatch.setenv(PREFIX + "OIDC_AUDIENCE", "opendox-runtime") monkeypatch.setenv(PREFIX + "RUNTIME_PG_ROLE", "opendox_runtime") @@ -432,6 +436,8 @@ def test_init_refuses_a_repository_root_that_is_not_a_directory( root.write_text("not a directory", encoding="utf-8") monkeypatch.setenv(PREFIX + "DATABASE_URL", "postgresql://nobody@127.0.0.1:1/none") + monkeypatch.setenv(PREFIX + "MIGRATION_DATABASE_URL", + "postgresql://migrate@127.0.0.1:1/none") monkeypatch.setenv(PREFIX + "OIDC_ISSUER", "https://broker/realms/x") monkeypatch.setenv(PREFIX + "OIDC_AUDIENCE", "opendox-runtime") monkeypatch.setenv(PREFIX + "PROJECT_REPOSITORY_ROOT", str(root)) @@ -496,6 +502,8 @@ def run(self) -> None: monkeypatch.setitem(sys.modules, "opendox.runtime.app", app_stub) monkeypatch.setenv(PREFIX + "DATABASE_URL", "postgresql://nobody@127.0.0.1:1/none") + monkeypatch.setenv(PREFIX + "MIGRATION_DATABASE_URL", + "postgresql://migrate@127.0.0.1:1/none") monkeypatch.setenv(PREFIX + "OIDC_ISSUER", "https://broker/realms/x") monkeypatch.setenv(PREFIX + "OIDC_AUDIENCE", "opendox-runtime") code, evidence = _run(cli.build_parser().parse_args(["runtime", "serve"])) @@ -541,6 +549,8 @@ def run(self) -> None: monkeypatch.setitem(sys.modules, "opendox.runtime.app", app_stub) monkeypatch.setenv(PREFIX + "DATABASE_URL", "postgresql://nobody@127.0.0.1:1/none") + monkeypatch.setenv(PREFIX + "MIGRATION_DATABASE_URL", + "postgresql://migrate@127.0.0.1:1/none") monkeypatch.setenv(PREFIX + "OIDC_ISSUER", "https://broker/realms/x") monkeypatch.setenv(PREFIX + "OIDC_AUDIENCE", "opendox-runtime") code, evidence = _run(cli.build_parser().parse_args(["runtime", "serve"])) @@ -624,6 +634,8 @@ def __exit__(self, *exc: object) -> None: db_stub.Database = _Exploding # type: ignore[attr-defined] monkeypatch.setitem(sys.modules, "opendox.runtime.db", db_stub) monkeypatch.setenv(PREFIX + "DATABASE_URL", dsn) + monkeypatch.setenv(PREFIX + "MIGRATION_DATABASE_URL", + "postgresql://migrate@db.internal:5432/opendox") monkeypatch.setenv(PREFIX + "OIDC_ISSUER", "https://broker/realms/x") monkeypatch.setenv(PREFIX + "OIDC_AUDIENCE", "opendox-runtime") @@ -677,6 +689,8 @@ def __init__(self, config: object) -> None: monkeypatch.setitem(sys.modules, "opendox.runtime.app", app_stub) monkeypatch.setenv(PREFIX + "DATABASE_URL", "postgresql://nobody:hunter2@127.0.0.1:1/none") + monkeypatch.setenv(PREFIX + "MIGRATION_DATABASE_URL", + "postgresql://migrate@127.0.0.1:1/none") monkeypatch.setenv(PREFIX + "OIDC_ISSUER", "https://broker/realms/x") monkeypatch.setenv(PREFIX + "OIDC_AUDIENCE", "opendox-runtime") @@ -794,6 +808,8 @@ def connection(self): monkeypatch.setitem(sys.modules, "opendox.runtime.db", db_stub) monkeypatch.setenv(PREFIX + "DATABASE_URL", "postgresql://runtime:hunter2@127.0.0.1:5432/opendox") + monkeypatch.setenv(PREFIX + "MIGRATION_DATABASE_URL", + "postgresql://migrate@127.0.0.1:5432/opendox") monkeypatch.setenv(PREFIX + "OIDC_ISSUER", "https://broker/realms/x") monkeypatch.setenv(PREFIX + "OIDC_AUDIENCE", "opendox-runtime") monkeypatch.setenv(PREFIX + "MIGRATIONS_DIR", @@ -862,6 +878,8 @@ def transaction(self): monkeypatch.setitem(sys.modules, "opendox.runtime.db", db_stub) monkeypatch.setenv(PREFIX + "DATABASE_URL", "postgresql://runtime@127.0.0.1:5432/opendox") + monkeypatch.setenv(PREFIX + "MIGRATION_DATABASE_URL", + "postgresql://migrate@127.0.0.1:5432/opendox") monkeypatch.setenv(PREFIX + "OIDC_ISSUER", "https://broker/realms/x") monkeypatch.setenv(PREFIX + "OIDC_AUDIENCE", "opendox-runtime") monkeypatch.setenv(PREFIX + "PROJECT_REPOSITORY_ROOT", str(tmp_path)) @@ -1148,6 +1166,7 @@ def test_no_broker_url_this_runtime_prints_can_carry_a_credential() -> None: redacted_url) base = {PREFIX + "DATABASE_URL": "postgresql://u:p@h/db", + PREFIX + "MIGRATION_DATABASE_URL": "postgresql://m:p@h/db", PREFIX + "OIDC_AUDIENCE": "opendox-runtime"} for name in ("OIDC_ISSUER", "OIDC_JWKS_URL"): env = dict(base, **{PREFIX + "OIDC_ISSUER": "https://broker/realms/x"}) @@ -1195,6 +1214,7 @@ def test_a_credential_in_a_broker_urls_query_is_refused_like_one_in_its_userinfo redacted_url) base = {PREFIX + "DATABASE_URL": "postgresql://u:p@h/db", + PREFIX + "MIGRATION_DATABASE_URL": "postgresql://m:p@h/db", PREFIX + "OIDC_AUDIENCE": "opendox-runtime", PREFIX + "OIDC_ISSUER": "https://broker/realms/x"} for name, value, why in ( @@ -1243,6 +1263,7 @@ def test_the_broker_url_must_be_https_because_it_is_the_trust_anchor() -> None: from opendox.runtime.config import ConfigurationError, load_settings base = {PREFIX + "DATABASE_URL": "postgresql://u:p@h/db", + PREFIX + "MIGRATION_DATABASE_URL": "postgresql://m:p@h/db", PREFIX + "OIDC_AUDIENCE": "opendox-runtime"} for issuer in ("https://broker/realms/x", "http://localhost:8080/realms/x", "http://127.0.0.1:8080/realms/x", "http://[::1]:8080/x"): @@ -1424,6 +1445,7 @@ def test_a_broker_url_that_names_no_host_is_refused_at_the_door() -> None: from opendox.runtime.config import ConfigurationError, load_settings base = {PREFIX + "DATABASE_URL": "postgresql://u:p@h/db", + PREFIX + "MIGRATION_DATABASE_URL": "postgresql://m:p@h/db", PREFIX + "OIDC_AUDIENCE": "opendox-runtime"} for issuer in ("https:///realms/x", "https://", "https:///", "broker/realms/x", "https://user:hunter2@/realms/x"): @@ -1509,6 +1531,7 @@ def test_a_broker_url_whose_port_is_not_a_number_is_refused_at_the_door( from opendox.runtime.config import ConfigurationError, load_settings base = {PREFIX + "DATABASE_URL": "postgresql://u:p@h/db", + PREFIX + "MIGRATION_DATABASE_URL": "postgresql://m:p@h/db", PREFIX + "OIDC_AUDIENCE": "opendox-runtime"} for issuer in ("https://broker:not-a-port/realm", "https://broker:99999/realm", @@ -1554,6 +1577,7 @@ def test_the_issuer_carries_no_query_or_fragment_because_paths_are_appended( from opendox.runtime.config import ConfigurationError, load_settings base = {PREFIX + "DATABASE_URL": "postgresql://u:p@h/db", + PREFIX + "MIGRATION_DATABASE_URL": "postgresql://m:p@h/db", PREFIX + "OIDC_AUDIENCE": "opendox-runtime"} for issuer, component in (("https://broker/realms/x?tenant=a", "query"), ("https://broker/realms/x#frag", "fragment"), @@ -1599,6 +1623,7 @@ def test_the_loopback_exception_is_for_http_and_not_for_every_other_scheme( from opendox.runtime.config import ConfigurationError, load_settings base = {PREFIX + "DATABASE_URL": "postgresql://u:p@h/db", + PREFIX + "MIGRATION_DATABASE_URL": "postgresql://m:p@h/db", PREFIX + "OIDC_AUDIENCE": "opendox-runtime"} for issuer in ("ftp://localhost/realms/x", "file://127.0.0.1/realms/x", "ws://localhost:8080/realms/x", "ftp://[::1]/realms/x"): @@ -1910,6 +1935,7 @@ def test_a_malformed_broker_url_never_prints_its_own_password() -> None: issuer = "https://svc:hunter2@broker\u2100evil.example/realms/x" env = {PREFIX + "DATABASE_URL": "postgresql://u:p@h/db", + PREFIX + "MIGRATION_DATABASE_URL": "postgresql://m:p@h/db", PREFIX + "OIDC_AUDIENCE": "opendox", PREFIX + "OIDC_ISSUER": issuer} @@ -2126,9 +2152,16 @@ def test_two_dsns_that_select_different_schemas_are_refused() -> None: PREFIX + "DATABASE_URL": "postgresql://u:p@h/db", PREFIX + "MIGRATION_DATABASE_URL": "postgresql://m:p@h/db"}) - # AND A MIGRATION DSN THAT IS SIMPLY ABSENT is the documented single-role - # deployment, not a mismatch. - assert load_settings({**base, PREFIX + "DATABASE_URL": served}) + # AND A MIGRATION DSN THAT IS SIMPLY ABSENT is refused NOW (plan 034, + # 13.3), where it used to be accepted as "the documented single-role + # deployment, not a mismatch": that reading is exactly what `load_settings` + # reading the setting as OPTIONAL bought, and the setting stopped being + # optional. A single-role install still names both DSNs, distinctly — + # `test_a_dsn_that_names_no_database_still_reaches_one`'s "two different + # secrets for one role" is what single-role now looks like. + with pytest.raises(ConfigurationError) as absent: + load_settings({**base, PREFIX + "DATABASE_URL": served}) + assert PREFIX + "MIGRATION_DATABASE_URL" in str(absent.value) # THE OTHER DIRECTION IS REFUSED TOO: a served DSN that names no schema # beside a migration DSN that names one is the same split, and the @@ -2140,6 +2173,86 @@ def test_two_dsns_that_select_different_schemas_are_refused() -> None: assert "the connection default" in str(either_way.value) +def test_a_non_postgresql_dsn_is_refused_naming_the_dialect_kept() -> None: + """13.2: a second dialect is refused, not supported. + + RULING Q1 keeps this database DOCUMENT-FREE, so a second dialect would + double every migration and every schema test forever for a database that + holds nothing. `_refuse_non_postgresql_dsn` asks it of both DSNs + `load_settings` holds, before either reaches the checks above that + compare them. + """ + from opendox.runtime.config import ConfigurationError, load_settings + + base = {PREFIX + "OIDC_ISSUER": "https://broker/realms/x", + PREFIX + "OIDC_AUDIENCE": "opendox"} + with pytest.raises(ConfigurationError) as served: + load_settings({**base, + PREFIX + "DATABASE_URL": "sqlite:///x.db", + PREFIX + "MIGRATION_DATABASE_URL": + "postgresql://m:p@h/db"}) + assert "postgres" in str(served.value).lower() + assert PREFIX + "DATABASE_URL" in str(served.value) + + with pytest.raises(ConfigurationError) as migration: + load_settings({**base, + PREFIX + "DATABASE_URL": "postgresql://u:p@h/db", + PREFIX + "MIGRATION_DATABASE_URL": "mysql://m:p@h/db"}) + assert "postgres" in str(migration.value).lower() + assert PREFIX + "MIGRATION_DATABASE_URL" in str(migration.value) + + # `postgres://` IS THE OTHER SPELLING LIBPQ ACCEPTS, not a second dialect. + assert load_settings({**base, + PREFIX + "DATABASE_URL": "postgres://u:p@h/db", + PREFIX + "MIGRATION_DATABASE_URL": + "postgres://m:p@h/db"}) + + # AND THE KEYWORD/VALUE FORM NAMES NO DIALECT AT ALL, so it is not refused + # here: libpq's own conninfo grammar reaches no other driver, and this + # module already reads an empty scheme as "says nothing" the way + # `schema_selected_by` does for a DSN that names no schema. + assert load_settings({**base, + PREFIX + "DATABASE_URL": "host=h dbname=db", + PREFIX + "MIGRATION_DATABASE_URL": + "host=h dbname=db user=m"}) + + +def test_the_same_dsn_in_both_settings_is_refused_naming_the_migration_one( +) -> None: + """13.3: one credential pasted into both settings is refused. + + `_refuse_the_same_dsn_in_both_settings` is asked only once the two DSNs + are known to AGREE on where they land + (`_refuse_two_dsns_that_select_different_schemas`, above it): agreement + bought by two DIFFERENT secrets for the one role is the accepted + single-role shape + (`test_a_dsn_that_names_no_database_still_reaches_one`'s last case); + agreement bought by writing the SAME value into both settings is this + refusal instead. + """ + from opendox.runtime.config import ConfigurationError, load_settings + + base = {PREFIX + "OIDC_ISSUER": "https://broker/realms/x", + PREFIX + "OIDC_AUDIENCE": "opendox"} + one = "postgresql://one:hunter2@h/opendox" + with pytest.raises(ConfigurationError) as collapsed: + load_settings({**base, + PREFIX + "DATABASE_URL": one, + PREFIX + "MIGRATION_DATABASE_URL": one}) + message = str(collapsed.value) + assert PREFIX + "MIGRATION_DATABASE_URL" in message + assert PREFIX + "DATABASE_URL" in message + # THE VALUE IS NOT REPEATED: a DSN carries a password. + assert "hunter2" not in message and "one:" not in message + + # DIFFERENT STRINGS THAT STILL AGREE are NOT this refusal, whether the + # difference is the secret alone (single-role) or the whole identity. + assert load_settings({**base, + PREFIX + "DATABASE_URL": "postgresql://a:p@h/db", + PREFIX + "MIGRATION_DATABASE_URL": + "postgresql://b:p@h/db"}) + + def test_two_dsns_naming_different_databases_are_refused_too() -> None: """The schema comparison means nothing across two databases. @@ -2348,6 +2461,7 @@ def test_a_credential_shaped_parameter_name_is_a_WORD_and_not_a_substring() -> N assert redacted_url("https://broker/certs?token=x" ) == "https://broker/certs?token=" base = {PREFIX + "DATABASE_URL": "postgresql://u:p@h/db", + PREFIX + "MIGRATION_DATABASE_URL": "postgresql://m:p@h/db", PREFIX + "OIDC_ISSUER": "https://broker/realms/x", PREFIX + "OIDC_AUDIENCE": "opendox"} settings = load_settings({**base, @@ -2548,6 +2662,8 @@ def test_init_creates_the_repository_root_private_whatever_the_umask_is( root = tmp_path / "projects" monkeypatch.setenv(PREFIX + "DATABASE_URL", "postgresql://nobody@127.0.0.1:1/none") + monkeypatch.setenv(PREFIX + "MIGRATION_DATABASE_URL", + "postgresql://migrate@127.0.0.1:1/none") monkeypatch.setenv(PREFIX + "OIDC_ISSUER", "https://broker/realms/x") monkeypatch.setenv(PREFIX + "OIDC_AUDIENCE", "opendox-runtime") monkeypatch.setenv(PREFIX + "PROJECT_REPOSITORY_ROOT", str(root)) @@ -2584,6 +2700,8 @@ def test_init_reports_an_existing_root_s_mode_and_does_not_change_it( os.chmod(root, 0o755) monkeypatch.setenv(PREFIX + "DATABASE_URL", "postgresql://nobody@127.0.0.1:1/none") + monkeypatch.setenv(PREFIX + "MIGRATION_DATABASE_URL", + "postgresql://migrate@127.0.0.1:1/none") monkeypatch.setenv(PREFIX + "OIDC_ISSUER", "https://broker/realms/x") monkeypatch.setenv(PREFIX + "OIDC_AUDIENCE", "opendox-runtime") monkeypatch.setenv(PREFIX + "PROJECT_REPOSITORY_ROOT", str(root)) diff --git a/tests_runtime/test_runtime_surface.py b/tests_runtime/test_runtime_surface.py index ff6a73a3..b36c6203 100644 --- a/tests_runtime/test_runtime_surface.py +++ b/tests_runtime/test_runtime_surface.py @@ -258,6 +258,7 @@ def test_only_asymmetric_algorithms_can_be_configured_and_they_keep_pyjwts_spell ) base = {PREFIX + "DATABASE_URL": "postgresql://x/y", + PREFIX + "MIGRATION_DATABASE_URL": "postgresql://m@x/y", PREFIX + "OIDC_ISSUER": "https://broker/realms/x", PREFIX + "OIDC_AUDIENCE": "opendox-runtime"} for name in ASYMMETRIC_ALGORITHMS: @@ -318,6 +319,7 @@ def test_every_integer_setting_is_bounded_above_as_well_as_below() -> None: ) base = {PREFIX + "DATABASE_URL": "postgresql://u:p@h/db", + PREFIX + "MIGRATION_DATABASE_URL": "postgresql://m:p@h/db", PREFIX + "OIDC_AUDIENCE": "opendox", PREFIX + "OIDC_ISSUER": "https://broker/realms/x"} From 91f797329ce81fa325f3c9b24512f209478316de Mon Sep 17 00:00:00 2001 From: Brett Heap <1513478+brettheap@users.noreply.github.com> Date: Mon, 28 Sep 2026 19:04:30 +0000 Subject: [PATCH 02/88] Fix round: _refuse_non_postgresql_dsn never raises a bare ValueError (Copilot review of this PR) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `urlsplit` itself raises for a DSN it cannot parse — MEASURED, ValueError("Invalid IPv6 URL") for an unbracketed IPv6 host, which tests_runtime/conftest.py's own postgres_dsn docstring names as "the ordinary way to mis-set this variable". `_refuse_non_postgresql_dsn` called `urlsplit(dsn).scheme` unguarded, so that ValueError escaped load_settings as a bare exception instead of the promised ConfigurationError — the CLI's boundary catches only ConfigurationError, so a malformed OPENDOX_DATABASE_URL or OPENDOX_MIGRATION_DATABASE_URL would have printed a traceback instead of a redacted refusal. Wrapped the same way _split_url already wraps it for the broker settings (Copilot review of openDox-code#25, round 24), with DSN-appropriate wording rather than reused verbatim ("set it to the broker endpoint" does not fit a database DSN). New test test_an_unparseable_dsn_is_refused_and_never_raises_a_bare_valueerror proves both DSNs are covered and that the value is never repeated in the message. Arc: neutral-product-standalone-operability Lane: openxfactory-4 (openXfactory-4-openDox_extraction) Co-Authored-By: Claude Opus 5.5 (1M context) --- src/opendox/runtime/config.py | 17 ++++++++++++- tests_runtime/test_runtime_cli.py | 40 +++++++++++++++++++++++++++++++ 2 files changed, 56 insertions(+), 1 deletion(-) diff --git a/src/opendox/runtime/config.py b/src/opendox/runtime/config.py index 046a46ab..ecc53d65 100644 --- a/src/opendox/runtime/config.py +++ b/src/opendox/runtime/config.py @@ -1313,8 +1313,23 @@ def _refuse_non_postgresql_dsn(name: str, dsn: str) -> None: scheme for the keyword/value form (there is no `://` to split on), and an empty scheme is read as "says nothing" here, exactly as `schema_selected_by` reads a DSN that names no schema as `None` rather than as a refusal. + + `urlsplit` ITSELF RAISES for a DSN it cannot parse — MEASURED, + `ValueError("Invalid IPv6 URL")` for an unbracketed IPv6 host, which + `tests_runtime/conftest.py`'s own `postgres_dsn` docstring names as "the + ordinary way to mis-set this variable". `_split_url` exists for exactly + this shape in the broker settings (Copilot review of openDox-code#25, + round 24); this is its DSN-flavoured twin; a bad `OPENDOX_DATABASE_URL` + is not "set it to the broker endpoint", so it is not reused verbatim. """ - scheme = urllib.parse.urlsplit(dsn).scheme + try: + scheme = urllib.parse.urlsplit(dsn).scheme + except ValueError as exc: + raise ConfigurationError( + f"{name} is not a DSN this runtime can parse " + f"({type(exc).__name__}); the value is not repeated here, " + "because a DSN this runtime cannot parse can still carry a " + "password") from None if scheme and scheme not in POSTGRESQL_SCHEMES: raise ConfigurationError( f"{name} names the {scheme!r} dialect. PostgreSQL " diff --git a/tests_runtime/test_runtime_cli.py b/tests_runtime/test_runtime_cli.py index 363caced..2c3c97eb 100644 --- a/tests_runtime/test_runtime_cli.py +++ b/tests_runtime/test_runtime_cli.py @@ -2217,6 +2217,46 @@ def test_a_non_postgresql_dsn_is_refused_naming_the_dialect_kept() -> None: "host=h dbname=db user=m"}) +def test_an_unparseable_dsn_is_refused_and_never_raises_a_bare_valueerror( +) -> None: + """`urlsplit` itself raises for a DSN it cannot parse, and this module's + whole contract is that a bad variable produces a named + `ConfigurationError`, never a bare exception the CLI's boundary does not + catch (Copilot review of this PR). + + MEASURED: `urllib.parse.urlsplit("postgresql://u:p@[::1/db")` raises + `ValueError("Invalid IPv6 URL")` — an unbracketed IPv6 host, which + `tests_runtime/conftest.py`'s own `postgres_dsn` docstring names as "the + ordinary way to mis-set this variable". `_split_url` exists for exactly + this shape in the broker settings (Copilot review of openDox-code#25, + round 24); `_refuse_non_postgresql_dsn` is its own boundary for the two + DSNs, asked of both. + """ + from opendox.runtime.config import ConfigurationError, load_settings + + base = {PREFIX + "OIDC_ISSUER": "https://broker/realms/x", + PREFIX + "OIDC_AUDIENCE": "opendox"} + broken = "postgresql://opendox:hunter2@[::1/opendox" + + with pytest.raises(ConfigurationError) as served: + load_settings({**base, + PREFIX + "DATABASE_URL": broken, + PREFIX + "MIGRATION_DATABASE_URL": + "postgresql://m:p@h/db"}) + message = str(served.value) + assert PREFIX + "DATABASE_URL" in message + assert "ValueError" in message + # THE VALUE IS NOT REPEATED: a DSN this runtime cannot parse can still + # carry a password. + assert "hunter2" not in message and "opendox:" not in message + + with pytest.raises(ConfigurationError) as migration: + load_settings({**base, + PREFIX + "DATABASE_URL": "postgresql://u:p@h/db", + PREFIX + "MIGRATION_DATABASE_URL": broken}) + assert PREFIX + "MIGRATION_DATABASE_URL" in str(migration.value) + + def test_the_same_dsn_in_both_settings_is_refused_naming_the_migration_one( ) -> None: """13.3: one credential pasted into both settings is refused. From b5296f91c5df003964bf5c8f0126f6005da1fbbd Mon Sep 17 00:00:00 2001 From: Brett Heap <1513478+brettheap@users.noreply.github.com> Date: Mon, 28 Sep 2026 22:45:45 +0000 Subject: [PATCH 03/88] Rework #60 to Brett's ruling: OPENDOX_MIGRATION_DATABASE_URL required only for migrate MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Brett ruled on the held conflict (openxFactory#656, on the claim thread for plan 034's T071, 2026-09-28), choosing "Required only for migrate (Recommended)" over making the setting required everywhere: - OPENDOX_MIGRATION_DATABASE_URL goes back to OPTIONAL in `load_settings` (the `Setting` row's `required` flag, `RuntimeSettings.migration_ database_url`'s type back to `str | None`, `_optional` in place of `_require`). `load_migration_settings` is unaffected either way — it already independently required one, for `migrate`/`reset` alone. - Both refusals from the previous commits stay, and are now no-ops on an ABSENT migration DSN rather than being unreachable: `_refuse_non_ postgresql_dsn` and `_refuse_the_same_dsn_in_both_settings` each return early when the migration value is falsy, exactly the way `_refuse_two_ dsns_that_select_different_schemas` already treated "nothing to compare" as nothing to fault. When BOTH are given, every check still runs, in the same order as before (dialect, then schema-mismatch, then collapse). It is never defaulted from OPENDOX_DATABASE_URL. - This matches #1144 13.3's own text and `deploy/compose/docker-compose. yaml`'s separation (the `opendox` service never gets a migration DSN; `docs/runtime.md` § 3 never lists it as required) — neither file needed a change; both already said the now-ruled behavior. The plan's "stops being optional" line is a holder-side correction, not part of this PR, and #1144's own wording is unchanged. Reverted the 27-call-site ripple the `required` flip had forced, now that it is not needed: `tests_runtime/conftest.py`'s `migration_dsn` fixture is gone; `test_api_endpoints.py`, `test_migrations_apply.py`, `test_runtime_ cli.py` and `test_runtime_surface.py` are back to threading only the served DSN through every call site that does not itself test the migration path. All four files after conftest.py are byte-for-byte `main` again. `test_two_dsns_that_select_different_schemas_are_refused`'s "absent migration" case is back to ACCEPTED (with a note on why it was briefly the opposite), which is what the setting being optional again means for that test. Added three tests showing the ruled behavior, at the CLI dispatch level rather than only `load_settings` directly, next to the existing `migrate` counterpart: - `test_serve_and_status_load_with_no_migration_dsn_configured`: `status` reports no configuration refusal and `settings[…MIGRATION_DATABASE_URL] ` as `null` with only the served DSN set; `serve` starts (`ok: true`) the same way. - `test_the_collapse_is_refused_through_the_served_workload_too`: 13.3's collapse refusal still fires through `status`, not only through `load_settings` called directly, the moment both DSNs are given and are the same value. - `test_migrate_refuses_rather_than_borrowing_the_served_identity` (pre-existing, untouched) already covers "migrate refuses without it". Measured locally against this change (own Postgres container, bridge IP): `python -m pytest -q` reports 2472 passed, 11 skipped, 1 failed — the one failure is the same `tests/test_model_provider_broker.py:: test_the_broker_child_inherits_no_credential_shaped_environment` LC_CTYPE sandbox artifact already characterized as pre-existing and unrelated in the first commit on this branch. Against main's 2479 selected / 11 skipped in this same environment, this change is +5/+5/+0 (five tests: the three already on this branch plus the two new ones above) — `validate.yml`'s `Pin the triple` floors (`MIN_SELECTED=2476`, `MIN_PASSED=2465`, `EXPECT_SKIPPED=11`) permit the rise unchanged, and the exact skip count is unchanged. Arc: neutral-product-standalone-operability Lane: openxfactory-4 (openXfactory-4-openDox_extraction) Co-Authored-By: Claude Sonnet 5 --- src/opendox/runtime/config.py | 61 +++++++++++---- tests_runtime/conftest.py | 31 +------- tests_runtime/test_api_endpoints.py | 18 ++--- tests_runtime/test_migrations_apply.py | 7 -- tests_runtime/test_runtime_cli.py | 101 +++++++++++++++---------- tests_runtime/test_runtime_surface.py | 2 - 6 files changed, 113 insertions(+), 107 deletions(-) diff --git a/src/opendox/runtime/config.py b/src/opendox/runtime/config.py index ecc53d65..deaf53e9 100644 --- a/src/opendox/runtime/config.py +++ b/src/opendox/runtime/config.py @@ -81,7 +81,7 @@ class Setting: "it must not be the identity migrations run as", ), Setting( - PREFIX + "MIGRATION_DATABASE_URL", None, True, True, + PREFIX + "MIGRATION_DATABASE_URL", None, False, True, "the PRIVILEGED DSN ordered-SQL migrations are applied with, used by " "`opendox runtime migrate` alone and never by the served application", ), @@ -181,13 +181,17 @@ class RuntimeSettings: Construct with :func:`load_settings`; the fields are in `SETTINGS` order. - `migration_database_url` is never `None` from either loader (plan 034, - 13.3): `load_settings` requires it exactly as it requires `database_url`, - and `load_migration_settings` already refused to load without one. + `migration_database_url` is `None` from `load_settings` whenever + `OPENDOX_MIGRATION_DATABASE_URL` is unset — RULED "required only for + migrate" (openxFactory#656, on the claim thread for plan 034's T071, + 2026-09-28): the served workload never needs it, `deploy/compose/ + docker-compose.yaml`'s `opendox` service and `docs/runtime.md` § 3 never + supply it, and `load_migration_settings` is the loader that actually + requires one (unaffected by this: it already refused to load without one). """ database_url: str - migration_database_url: str + migration_database_url: str | None oidc_issuer: str oidc_audience: str oidc_jwks_url: str | None @@ -1304,9 +1308,16 @@ def effective_schema(dsn: str) -> str | None: POSTGRESQL_SCHEMES = frozenset({"postgresql", "postgres"}) -def _refuse_non_postgresql_dsn(name: str, dsn: str) -> None: +def _refuse_non_postgresql_dsn(name: str, dsn: str | None) -> None: """`name`'s DSN is refused unless it selects a PostgreSQL scheme. + `None` OR EMPTY IS A NO-OP, not a refusal: `OPENDOX_MIGRATION_DATABASE_URL` + is optional for `load_settings` (RULED "required only for migrate", + openxFactory#656, on the claim thread for plan 034's T071, 2026-09-28), + so an absent migration DSN has no dialect to check — the same shape + `_refuse_two_dsns_that_select_different_schemas` below already reads as + "nothing to compare" rather than as a fault. + A DSN in the keyword/value form (`host=h dbname=d …`) names NO DIALECT AT ALL — that syntax is libpq's own conninfo grammar, and no other driver reads it — so only the URI form is checked: `urlsplit` reports an EMPTY @@ -1322,6 +1333,8 @@ def _refuse_non_postgresql_dsn(name: str, dsn: str) -> None: round 24); this is its DSN-flavoured twin; a bad `OPENDOX_DATABASE_URL` is not "set it to the broker endpoint", so it is not reused verbatim. """ + if not dsn: + return try: scheme = urllib.parse.urlsplit(dsn).scheme except ValueError as exc: @@ -1339,9 +1352,18 @@ def _refuse_non_postgresql_dsn(name: str, dsn: str) -> None: "document (RULING Q1)") -def _refuse_the_same_dsn_in_both_settings(served: str, migration: str) -> None: +def _refuse_the_same_dsn_in_both_settings( + served: str, migration: str | None) -> None: """One credential pasted into both settings is refused (plan 034, 13.3). + A NO-OP WHEN MIGRATION IS ABSENT, exactly like + `_refuse_two_dsns_that_select_different_schemas` below: with nothing to + compare, there is nothing to have collapsed. `OPENDOX_MIGRATION_DATABASE_ + URL` is optional (RULED "required only for migrate", openxFactory#656, on + the claim thread for plan 034's T071, 2026-09-28) — but WHEN BOTH ARE + GIVEN, this refusal still applies, on every path `load_settings` serves, + not only the falsifier's. + `OPENDOX_DATABASE_URL` is the least-privileged identity the API serves with; `OPENDOX_MIGRATION_DATABASE_URL` is the privileged one ordered-SQL migrations run as — the whole point of keeping two settings. A @@ -1354,6 +1376,8 @@ def _refuse_the_same_dsn_in_both_settings(served: str, migration: str) -> None: credentials, not one pasted twice, and this checks the value actually given, not the identity it happens to resolve to. """ + if not migration: + return if served == migration: raise ConfigurationError( f"{PREFIX}MIGRATION_DATABASE_URL is the same value as " @@ -1477,21 +1501,26 @@ def load_settings(env: Mapping[str, str] | None = None) -> RuntimeSettings: fetch from the broker's JWKS, and discovering that on the first request means it is already serving. - BOTH DSNs ARE REQUIRED (plan 034, 13.3): `OPENDOX_MIGRATION_DATABASE_URL` - used to default to `None`, so the collapsed, single-role shape this - refuses (below) was reachable only by accident, through a caller who - happened to set both to the same value — an install that left the - migration credential unset was never asked the question at all. Naming it - explicitly, even at the same database a served DSN already names, is what - keeps the two identities two DECISIONS rather than one remembered twice. + `OPENDOX_MIGRATION_DATABASE_URL` STAYS OPTIONAL HERE (RULED "required only + for migrate", openxFactory#656, on the claim thread for plan 034's T071, + 2026-09-28): the served workload never needs it — + `deploy/compose/docker-compose.yaml`'s `opendox` service and + `docs/runtime.md` § 3 never supply it, keeping the two identities in + different containers — and `load_migration_settings` below is the loader + that actually requires one. It is never DEFAULTED from + `OPENDOX_DATABASE_URL` either way. WHEN BOTH ARE GIVEN, though, the two + checks below still apply: a non-PostgreSQL migration DSN is refused + (13.2), and the two being the exact same value is refused (13.3) — + optional does not mean unchecked. """ env = os.environ if env is None else env algorithms = _algorithms(env) served = _require(env, _by_name(PREFIX + "DATABASE_URL")) - migration = _require(env, _by_name(PREFIX + "MIGRATION_DATABASE_URL")) + migration = _optional(env, _by_name(PREFIX + "MIGRATION_DATABASE_URL")) # THE DIALECT FIRST: a scheme this module cannot parse as PostgreSQL is not - # yet a DSN worth comparing at all. + # yet a DSN worth comparing at all. A no-op on an ABSENT migration DSN — + # see `_refuse_non_postgresql_dsn`. _refuse_non_postgresql_dsn(PREFIX + "DATABASE_URL", served) _refuse_non_postgresql_dsn(PREFIX + "MIGRATION_DATABASE_URL", migration) # THEN WHETHER THEY DISAGREE. See `_refuse_two_dsns_that_select_different_ diff --git a/tests_runtime/conftest.py b/tests_runtime/conftest.py index 93f53a4d..2c6e0b7b 100644 --- a/tests_runtime/conftest.py +++ b/tests_runtime/conftest.py @@ -219,29 +219,6 @@ def postgres_dsn() -> str: return dsn -@pytest.fixture(scope="session") -def migration_dsn(postgres_dsn: str) -> str: - """A DSN for `OPENDOX_MIGRATION_DATABASE_URL`, distinct from `postgres_dsn`. - - Plan 034, 13.3: `load_settings` now REQUIRES this setting and refuses a - value that is simply `postgres_dsn` repeated (13.3's own collapse - refusal), so every fixture built from the pair below needs a second, - genuinely different string — not a second database. None of this - module's fixtures apply migrations through this identity: `database` - applies them directly with `MigrationRunner`, and the served app under - test is never asked to `migrate`. So `load_settings` is the only reader - that cares about this value at all, and what it asks of a DSN is a - PostgreSQL scheme (13.2), a STRING distinct from `postgres_dsn` (13.3), - and a database and schema that agree with it - (`_refuse_two_dsns_that_select_different_schemas`). A URI FRAGMENT is - invisible to `database_named_by`, `user_named_by` and `schema_selected_by` - alike — none of the three inspects `urlsplit(...).fragment` — so - appending one changes the STRING without moving the identity those - functions compare. - """ - return postgres_dsn + "#opendox-test-migration-identity" - - @pytest.fixture def database(postgres_dsn: str) -> Iterator[object]: """A `Database` on a throwaway schema, with the migrations already applied.""" @@ -380,7 +357,7 @@ def verifier(jwks_path: str): @pytest.fixture() -def client(database, postgres_dsn: str, migration_dsn: str, verifier): +def client(database, postgres_dsn: str, verifier): """A `TestClient` over the REAL application, on this test's own schema. The application is given its OWN `Database` on the same schema rather than @@ -399,7 +376,6 @@ def client(database, postgres_dsn: str, migration_dsn: str, verifier): settings = load_settings({ PREFIX + "DATABASE_URL": postgres_dsn, - PREFIX + "MIGRATION_DATABASE_URL": migration_dsn, PREFIX + "OIDC_ISSUER": TEST_ISSUER, PREFIX + "OIDC_AUDIENCE": TEST_AUDIENCE, }) @@ -423,8 +399,8 @@ def project_repository_root(tmp_path): @pytest.fixture() -def client_with_repositories(database, postgres_dsn: str, migration_dsn: str, - verifier, project_repository_root): +def client_with_repositories(database, postgres_dsn: str, verifier, + project_repository_root): """`client`, with the repository root pointed at this test's own directory.""" fastapi_testclient = _import_fastapi_testclient() from opendox.runtime.app import create_app @@ -433,7 +409,6 @@ def client_with_repositories(database, postgres_dsn: str, migration_dsn: str, settings = load_settings({ PREFIX + "DATABASE_URL": postgres_dsn, - PREFIX + "MIGRATION_DATABASE_URL": migration_dsn, PREFIX + "OIDC_ISSUER": TEST_ISSUER, PREFIX + "OIDC_AUDIENCE": TEST_AUDIENCE, PREFIX + "PROJECT_REPOSITORY_ROOT": str(project_repository_root), diff --git a/tests_runtime/test_api_endpoints.py b/tests_runtime/test_api_endpoints.py index 84caccfe..21897203 100644 --- a/tests_runtime/test_api_endpoints.py +++ b/tests_runtime/test_api_endpoints.py @@ -425,7 +425,7 @@ def test_the_unauthenticated_surface_is_exactly_the_two_probes( def test_the_schema_viewers_appear_only_when_the_install_says_so( - database, postgres_dsn: str, migration_dsn: str, verifier) -> None: + database, postgres_dsn: str, verifier) -> None: from fastapi.testclient import TestClient from opendox.runtime.app import create_app @@ -435,7 +435,6 @@ def test_the_schema_viewers_appear_only_when_the_install_says_so( settings = load_settings({ PREFIX + "DATABASE_URL": postgres_dsn, - PREFIX + "MIGRATION_DATABASE_URL": migration_dsn, PREFIX + "OIDC_ISSUER": TEST_ISSUER, PREFIX + "OIDC_AUDIENCE": TEST_AUDIENCE, PREFIX + "PUBLISH_OPENAPI": "true", @@ -449,7 +448,7 @@ def test_the_schema_viewers_appear_only_when_the_install_says_so( def test_readiness_refuses_an_unmigrated_database_by_name( - postgres_dsn: str, migration_dsn: str, verifier) -> None: + postgres_dsn: str, verifier) -> None: """`select 1` succeeds against a schema with no tables in it at all. Readiness without a migration check therefore turns a fresh install READY @@ -472,7 +471,6 @@ def test_readiness_refuses_an_unmigrated_database_by_name( try: settings = load_settings({ PREFIX + "DATABASE_URL": postgres_dsn, - PREFIX + "MIGRATION_DATABASE_URL": migration_dsn, PREFIX + "OIDC_ISSUER": TEST_ISSUER, PREFIX + "OIDC_AUDIENCE": TEST_AUDIENCE, }) @@ -590,8 +588,7 @@ def test_draft_pagination_returns_the_principals_drafts_not_an_empty_page( def test_readiness_refuses_a_database_whose_migration_file_has_changed( - database, postgres_dsn: str, migration_dsn: str, verifier, - tmp_path) -> None: + database, postgres_dsn: str, verifier, tmp_path) -> None: """Nothing pending is not the same as matching this tree.""" import shutil @@ -612,7 +609,6 @@ def test_readiness_refuses_a_database_whose_migration_file_has_changed( settings = load_settings({ PREFIX + "DATABASE_URL": postgres_dsn, - PREFIX + "MIGRATION_DATABASE_URL": migration_dsn, PREFIX + "OIDC_ISSUER": TEST_ISSUER, PREFIX + "OIDC_AUDIENCE": TEST_AUDIENCE, PREFIX + "MIGRATIONS_DIR": str(tmp_path), @@ -856,8 +852,7 @@ def test_a_hidden_user_and_a_missing_one_answer_byte_for_byte_the_same( def test_readiness_refuses_a_migrations_directory_without_the_pinned_0001( - database, postgres_dsn: str, migration_dsn: str, verifier, - tmp_path) -> None: + database, postgres_dsn: str, verifier, tmp_path) -> None: """An EMPTY directory made `plan()` and `drift()` both empty. `discover()` returns `[]` for a directory that exists and holds no @@ -879,7 +874,6 @@ def test_readiness_refuses_a_migrations_directory_without_the_pinned_0001( empty.mkdir() settings = load_settings({ PREFIX + "DATABASE_URL": postgres_dsn, - PREFIX + "MIGRATION_DATABASE_URL": migration_dsn, PREFIX + "OIDC_ISSUER": TEST_ISSUER, PREFIX + "OIDC_AUDIENCE": TEST_AUDIENCE, PREFIX + "MIGRATIONS_DIR": str(empty), @@ -1558,8 +1552,7 @@ def transaction(): def test_an_anonymous_token_naming_a_key_of_the_wrong_type_is_401_not_500( - database, postgres_dsn: str, migration_dsn: str, rsa_key_pair, - tmp_path: Path) -> None: + database, postgres_dsn: str, rsa_key_pair, tmp_path: Path) -> None: """The end of A25-2, measured where it was reported: at the HTTP boundary. A realm publishing an RSA and an EC signing key — Keycloak, the moment a @@ -1600,7 +1593,6 @@ def test_an_anonymous_token_naming_a_key_of_the_wrong_type_is_401_not_500( settings = load_settings({ PREFIX + "DATABASE_URL": postgres_dsn, - PREFIX + "MIGRATION_DATABASE_URL": migration_dsn, PREFIX + "OIDC_ISSUER": TEST_ISSUER, PREFIX + "OIDC_AUDIENCE": TEST_AUDIENCE, }) diff --git a/tests_runtime/test_migrations_apply.py b/tests_runtime/test_migrations_apply.py index 1b657628..2afc0fe1 100644 --- a/tests_runtime/test_migrations_apply.py +++ b/tests_runtime/test_migrations_apply.py @@ -345,11 +345,6 @@ def test_status_reports_a_reachable_database_and_its_applied_migrations( # libpq startup parameter `Database(schema=…)` sets for the harness. scoped = (f"{postgres_dsn}?options=-c%20search_path%3D{database.schema}") monkeypatch.setenv(PREFIX + "DATABASE_URL", scoped) - # A DISTINCT STRING (13.3's collapse refusal) THAT SELECTS THE SAME SCHEMA - # (`_refuse_two_dsns_that_select_different_schemas`): a URI FRAGMENT moves - # neither, since neither reader inspects one. - monkeypatch.setenv(PREFIX + "MIGRATION_DATABASE_URL", - scoped + "#opendox-test-migration-identity") monkeypatch.setenv(PREFIX + "OIDC_ISSUER", "https://broker.test/realms/x") monkeypatch.setenv(PREFIX + "OIDC_AUDIENCE", "opendox-runtime") monkeypatch.setenv(PREFIX + "MIGRATIONS_DIR", str(ROOT / "migrations")) @@ -725,8 +720,6 @@ def test_status_calls_an_unmigrated_database_unhealthy_and_blames_the_tree( conn.execute(f"create schema {schema}") try: monkeypatch.setenv(PREFIX + "DATABASE_URL", scoped) - monkeypatch.setenv(PREFIX + "MIGRATION_DATABASE_URL", - scoped + "#opendox-test-migration-identity") monkeypatch.setenv(PREFIX + "OIDC_ISSUER", "https://broker/realms/x") monkeypatch.setenv(PREFIX + "OIDC_AUDIENCE", "opendox-runtime") monkeypatch.setenv(PREFIX + "MIGRATIONS_DIR", str(ROOT / "migrations")) diff --git a/tests_runtime/test_runtime_cli.py b/tests_runtime/test_runtime_cli.py index 2c3c97eb..3b220eee 100644 --- a/tests_runtime/test_runtime_cli.py +++ b/tests_runtime/test_runtime_cli.py @@ -206,13 +206,66 @@ def test_migrate_refuses_rather_than_borrowing_the_served_identity( assert PREFIX + "MIGRATION_DATABASE_URL" in evidence["message"] +def test_serve_and_status_load_with_no_migration_dsn_configured( + monkeypatch: pytest.MonkeyPatch) -> None: + """Brett's ruling on #1144 13.3 ("Required only for migrate + (Recommended)"): `OPENDOX_MIGRATION_DATABASE_URL` stays OPTIONAL for the + served workload — `serve` and `status` both go through `load_settings`, + and this is the CLI-level proof that neither refuses at configuration + when only the served DSN is set. This is the shape + `deploy/compose/docker-compose.yaml`'s `opendox` service and + `docs/runtime.md` § 3 already document: the migration credential lives + only in the separate `migrate` service/profile. + """ + monkeypatch.delenv(PREFIX + "MIGRATION_DATABASE_URL", raising=False) + monkeypatch.setenv(PREFIX + "DATABASE_URL", + "postgresql://nobody@127.0.0.1:1/none") + monkeypatch.setenv(PREFIX + "OIDC_ISSUER", "https://broker/realms/x") + monkeypatch.setenv(PREFIX + "OIDC_AUDIENCE", "opendox-runtime") + + code, evidence = _run(cli.build_parser().parse_args( + ["runtime", "status", "--probe-timeout", "0.2"])) + assert evidence.get("refusal") != "configuration", evidence + assert "settings" in evidence, evidence + assert evidence["settings"][PREFIX + "MIGRATION_DATABASE_URL"] is None + + def _ok(self) -> None: + self.started = True + + _stub_uvicorn(monkeypatch, _ok) + monkeypatch.delenv(PREFIX + "MIGRATION_DATABASE_URL", raising=False) + code, evidence = _run_serve() + assert code == 0, evidence + assert evidence["ok"] is True, evidence + + +def test_the_collapse_is_refused_through_the_served_workload_too( + monkeypatch: pytest.MonkeyPatch) -> None: + """13.3's collapse refusal is `load_settings`'s own, not the falsifier's + special case: it fires for `status` (and every other served verb) too, + the moment an operator gives BOTH DSNs and they happen to be the exact + same value — even though migration is optional here (Brett's ruling, + above). + """ + same = "postgresql://opendox:hunter2@127.0.0.1:1/none" + monkeypatch.setenv(PREFIX + "DATABASE_URL", same) + monkeypatch.setenv(PREFIX + "MIGRATION_DATABASE_URL", same) + monkeypatch.setenv(PREFIX + "OIDC_ISSUER", "https://broker/realms/x") + monkeypatch.setenv(PREFIX + "OIDC_AUDIENCE", "opendox-runtime") + + code, evidence = _run(cli.build_parser().parse_args( + ["runtime", "status", "--probe-timeout", "0.2"])) + assert code == 1, evidence + assert evidence["refusal"] == "configuration", evidence + assert PREFIX + "MIGRATION_DATABASE_URL" in evidence["message"] + assert "hunter2" not in evidence["message"], evidence["message"] + + def test_init_creates_the_project_repository_root_and_touches_no_database( monkeypatch: pytest.MonkeyPatch, tmp_path) -> None: root = tmp_path / "projects" monkeypatch.setenv(PREFIX + "DATABASE_URL", "postgresql://nobody@127.0.0.1:1/none") - monkeypatch.setenv(PREFIX + "MIGRATION_DATABASE_URL", - "postgresql://migrate@127.0.0.1:1/none") monkeypatch.setenv(PREFIX + "OIDC_ISSUER", "https://broker/realms/x") monkeypatch.setenv(PREFIX + "OIDC_AUDIENCE", "opendox-runtime") monkeypatch.setenv(PREFIX + "PROJECT_REPOSITORY_ROOT", str(root)) @@ -364,8 +417,6 @@ def test_status_reports_every_declared_setting_and_none_as_null( from opendox.runtime.config import SETTING_NAMES monkeypatch.setenv(PREFIX + "DATABASE_URL", "postgresql://u:pw@127.0.0.1:1/x") - monkeypatch.setenv(PREFIX + "MIGRATION_DATABASE_URL", - "postgresql://m:pw@127.0.0.1:1/x") monkeypatch.setenv(PREFIX + "OIDC_ISSUER", "https://broker/realms/x") monkeypatch.setenv(PREFIX + "OIDC_AUDIENCE", "opendox-runtime") monkeypatch.setenv(PREFIX + "RUNTIME_PG_ROLE", "opendox_runtime") @@ -436,8 +487,6 @@ def test_init_refuses_a_repository_root_that_is_not_a_directory( root.write_text("not a directory", encoding="utf-8") monkeypatch.setenv(PREFIX + "DATABASE_URL", "postgresql://nobody@127.0.0.1:1/none") - monkeypatch.setenv(PREFIX + "MIGRATION_DATABASE_URL", - "postgresql://migrate@127.0.0.1:1/none") monkeypatch.setenv(PREFIX + "OIDC_ISSUER", "https://broker/realms/x") monkeypatch.setenv(PREFIX + "OIDC_AUDIENCE", "opendox-runtime") monkeypatch.setenv(PREFIX + "PROJECT_REPOSITORY_ROOT", str(root)) @@ -502,8 +551,6 @@ def run(self) -> None: monkeypatch.setitem(sys.modules, "opendox.runtime.app", app_stub) monkeypatch.setenv(PREFIX + "DATABASE_URL", "postgresql://nobody@127.0.0.1:1/none") - monkeypatch.setenv(PREFIX + "MIGRATION_DATABASE_URL", - "postgresql://migrate@127.0.0.1:1/none") monkeypatch.setenv(PREFIX + "OIDC_ISSUER", "https://broker/realms/x") monkeypatch.setenv(PREFIX + "OIDC_AUDIENCE", "opendox-runtime") code, evidence = _run(cli.build_parser().parse_args(["runtime", "serve"])) @@ -549,8 +596,6 @@ def run(self) -> None: monkeypatch.setitem(sys.modules, "opendox.runtime.app", app_stub) monkeypatch.setenv(PREFIX + "DATABASE_URL", "postgresql://nobody@127.0.0.1:1/none") - monkeypatch.setenv(PREFIX + "MIGRATION_DATABASE_URL", - "postgresql://migrate@127.0.0.1:1/none") monkeypatch.setenv(PREFIX + "OIDC_ISSUER", "https://broker/realms/x") monkeypatch.setenv(PREFIX + "OIDC_AUDIENCE", "opendox-runtime") code, evidence = _run(cli.build_parser().parse_args(["runtime", "serve"])) @@ -634,8 +679,6 @@ def __exit__(self, *exc: object) -> None: db_stub.Database = _Exploding # type: ignore[attr-defined] monkeypatch.setitem(sys.modules, "opendox.runtime.db", db_stub) monkeypatch.setenv(PREFIX + "DATABASE_URL", dsn) - monkeypatch.setenv(PREFIX + "MIGRATION_DATABASE_URL", - "postgresql://migrate@db.internal:5432/opendox") monkeypatch.setenv(PREFIX + "OIDC_ISSUER", "https://broker/realms/x") monkeypatch.setenv(PREFIX + "OIDC_AUDIENCE", "opendox-runtime") @@ -689,8 +732,6 @@ def __init__(self, config: object) -> None: monkeypatch.setitem(sys.modules, "opendox.runtime.app", app_stub) monkeypatch.setenv(PREFIX + "DATABASE_URL", "postgresql://nobody:hunter2@127.0.0.1:1/none") - monkeypatch.setenv(PREFIX + "MIGRATION_DATABASE_URL", - "postgresql://migrate@127.0.0.1:1/none") monkeypatch.setenv(PREFIX + "OIDC_ISSUER", "https://broker/realms/x") monkeypatch.setenv(PREFIX + "OIDC_AUDIENCE", "opendox-runtime") @@ -808,8 +849,6 @@ def connection(self): monkeypatch.setitem(sys.modules, "opendox.runtime.db", db_stub) monkeypatch.setenv(PREFIX + "DATABASE_URL", "postgresql://runtime:hunter2@127.0.0.1:5432/opendox") - monkeypatch.setenv(PREFIX + "MIGRATION_DATABASE_URL", - "postgresql://migrate@127.0.0.1:5432/opendox") monkeypatch.setenv(PREFIX + "OIDC_ISSUER", "https://broker/realms/x") monkeypatch.setenv(PREFIX + "OIDC_AUDIENCE", "opendox-runtime") monkeypatch.setenv(PREFIX + "MIGRATIONS_DIR", @@ -878,8 +917,6 @@ def transaction(self): monkeypatch.setitem(sys.modules, "opendox.runtime.db", db_stub) monkeypatch.setenv(PREFIX + "DATABASE_URL", "postgresql://runtime@127.0.0.1:5432/opendox") - monkeypatch.setenv(PREFIX + "MIGRATION_DATABASE_URL", - "postgresql://migrate@127.0.0.1:5432/opendox") monkeypatch.setenv(PREFIX + "OIDC_ISSUER", "https://broker/realms/x") monkeypatch.setenv(PREFIX + "OIDC_AUDIENCE", "opendox-runtime") monkeypatch.setenv(PREFIX + "PROJECT_REPOSITORY_ROOT", str(tmp_path)) @@ -1166,7 +1203,6 @@ def test_no_broker_url_this_runtime_prints_can_carry_a_credential() -> None: redacted_url) base = {PREFIX + "DATABASE_URL": "postgresql://u:p@h/db", - PREFIX + "MIGRATION_DATABASE_URL": "postgresql://m:p@h/db", PREFIX + "OIDC_AUDIENCE": "opendox-runtime"} for name in ("OIDC_ISSUER", "OIDC_JWKS_URL"): env = dict(base, **{PREFIX + "OIDC_ISSUER": "https://broker/realms/x"}) @@ -1214,7 +1250,6 @@ def test_a_credential_in_a_broker_urls_query_is_refused_like_one_in_its_userinfo redacted_url) base = {PREFIX + "DATABASE_URL": "postgresql://u:p@h/db", - PREFIX + "MIGRATION_DATABASE_URL": "postgresql://m:p@h/db", PREFIX + "OIDC_AUDIENCE": "opendox-runtime", PREFIX + "OIDC_ISSUER": "https://broker/realms/x"} for name, value, why in ( @@ -1263,7 +1298,6 @@ def test_the_broker_url_must_be_https_because_it_is_the_trust_anchor() -> None: from opendox.runtime.config import ConfigurationError, load_settings base = {PREFIX + "DATABASE_URL": "postgresql://u:p@h/db", - PREFIX + "MIGRATION_DATABASE_URL": "postgresql://m:p@h/db", PREFIX + "OIDC_AUDIENCE": "opendox-runtime"} for issuer in ("https://broker/realms/x", "http://localhost:8080/realms/x", "http://127.0.0.1:8080/realms/x", "http://[::1]:8080/x"): @@ -1445,7 +1479,6 @@ def test_a_broker_url_that_names_no_host_is_refused_at_the_door() -> None: from opendox.runtime.config import ConfigurationError, load_settings base = {PREFIX + "DATABASE_URL": "postgresql://u:p@h/db", - PREFIX + "MIGRATION_DATABASE_URL": "postgresql://m:p@h/db", PREFIX + "OIDC_AUDIENCE": "opendox-runtime"} for issuer in ("https:///realms/x", "https://", "https:///", "broker/realms/x", "https://user:hunter2@/realms/x"): @@ -1531,7 +1564,6 @@ def test_a_broker_url_whose_port_is_not_a_number_is_refused_at_the_door( from opendox.runtime.config import ConfigurationError, load_settings base = {PREFIX + "DATABASE_URL": "postgresql://u:p@h/db", - PREFIX + "MIGRATION_DATABASE_URL": "postgresql://m:p@h/db", PREFIX + "OIDC_AUDIENCE": "opendox-runtime"} for issuer in ("https://broker:not-a-port/realm", "https://broker:99999/realm", @@ -1577,7 +1609,6 @@ def test_the_issuer_carries_no_query_or_fragment_because_paths_are_appended( from opendox.runtime.config import ConfigurationError, load_settings base = {PREFIX + "DATABASE_URL": "postgresql://u:p@h/db", - PREFIX + "MIGRATION_DATABASE_URL": "postgresql://m:p@h/db", PREFIX + "OIDC_AUDIENCE": "opendox-runtime"} for issuer, component in (("https://broker/realms/x?tenant=a", "query"), ("https://broker/realms/x#frag", "fragment"), @@ -1623,7 +1654,6 @@ def test_the_loopback_exception_is_for_http_and_not_for_every_other_scheme( from opendox.runtime.config import ConfigurationError, load_settings base = {PREFIX + "DATABASE_URL": "postgresql://u:p@h/db", - PREFIX + "MIGRATION_DATABASE_URL": "postgresql://m:p@h/db", PREFIX + "OIDC_AUDIENCE": "opendox-runtime"} for issuer in ("ftp://localhost/realms/x", "file://127.0.0.1/realms/x", "ws://localhost:8080/realms/x", "ftp://[::1]/realms/x"): @@ -1935,7 +1965,6 @@ def test_a_malformed_broker_url_never_prints_its_own_password() -> None: issuer = "https://svc:hunter2@broker\u2100evil.example/realms/x" env = {PREFIX + "DATABASE_URL": "postgresql://u:p@h/db", - PREFIX + "MIGRATION_DATABASE_URL": "postgresql://m:p@h/db", PREFIX + "OIDC_AUDIENCE": "opendox", PREFIX + "OIDC_ISSUER": issuer} @@ -2152,16 +2181,11 @@ def test_two_dsns_that_select_different_schemas_are_refused() -> None: PREFIX + "DATABASE_URL": "postgresql://u:p@h/db", PREFIX + "MIGRATION_DATABASE_URL": "postgresql://m:p@h/db"}) - # AND A MIGRATION DSN THAT IS SIMPLY ABSENT is refused NOW (plan 034, - # 13.3), where it used to be accepted as "the documented single-role - # deployment, not a mismatch": that reading is exactly what `load_settings` - # reading the setting as OPTIONAL bought, and the setting stopped being - # optional. A single-role install still names both DSNs, distinctly — - # `test_a_dsn_that_names_no_database_still_reaches_one`'s "two different - # secrets for one role" is what single-role now looks like. - with pytest.raises(ConfigurationError) as absent: - load_settings({**base, PREFIX + "DATABASE_URL": served}) - assert PREFIX + "MIGRATION_DATABASE_URL" in str(absent.value) + # AND A MIGRATION DSN THAT IS SIMPLY ABSENT is the documented single-role + # deployment, not a mismatch (RULED "required only for migrate", + # openxFactory#656, on the claim thread for plan 034's T071, 2026-09-28 — + # this assertion was briefly the opposite of itself, reverted here). + assert load_settings({**base, PREFIX + "DATABASE_URL": served}) # THE OTHER DIRECTION IS REFUSED TOO: a served DSN that names no schema # beside a migration DSN that names one is the same split, and the @@ -2501,7 +2525,6 @@ def test_a_credential_shaped_parameter_name_is_a_WORD_and_not_a_substring() -> N assert redacted_url("https://broker/certs?token=x" ) == "https://broker/certs?token=" base = {PREFIX + "DATABASE_URL": "postgresql://u:p@h/db", - PREFIX + "MIGRATION_DATABASE_URL": "postgresql://m:p@h/db", PREFIX + "OIDC_ISSUER": "https://broker/realms/x", PREFIX + "OIDC_AUDIENCE": "opendox"} settings = load_settings({**base, @@ -2702,8 +2725,6 @@ def test_init_creates_the_repository_root_private_whatever_the_umask_is( root = tmp_path / "projects" monkeypatch.setenv(PREFIX + "DATABASE_URL", "postgresql://nobody@127.0.0.1:1/none") - monkeypatch.setenv(PREFIX + "MIGRATION_DATABASE_URL", - "postgresql://migrate@127.0.0.1:1/none") monkeypatch.setenv(PREFIX + "OIDC_ISSUER", "https://broker/realms/x") monkeypatch.setenv(PREFIX + "OIDC_AUDIENCE", "opendox-runtime") monkeypatch.setenv(PREFIX + "PROJECT_REPOSITORY_ROOT", str(root)) @@ -2740,8 +2761,6 @@ def test_init_reports_an_existing_root_s_mode_and_does_not_change_it( os.chmod(root, 0o755) monkeypatch.setenv(PREFIX + "DATABASE_URL", "postgresql://nobody@127.0.0.1:1/none") - monkeypatch.setenv(PREFIX + "MIGRATION_DATABASE_URL", - "postgresql://migrate@127.0.0.1:1/none") monkeypatch.setenv(PREFIX + "OIDC_ISSUER", "https://broker/realms/x") monkeypatch.setenv(PREFIX + "OIDC_AUDIENCE", "opendox-runtime") monkeypatch.setenv(PREFIX + "PROJECT_REPOSITORY_ROOT", str(root)) diff --git a/tests_runtime/test_runtime_surface.py b/tests_runtime/test_runtime_surface.py index b36c6203..ff6a73a3 100644 --- a/tests_runtime/test_runtime_surface.py +++ b/tests_runtime/test_runtime_surface.py @@ -258,7 +258,6 @@ def test_only_asymmetric_algorithms_can_be_configured_and_they_keep_pyjwts_spell ) base = {PREFIX + "DATABASE_URL": "postgresql://x/y", - PREFIX + "MIGRATION_DATABASE_URL": "postgresql://m@x/y", PREFIX + "OIDC_ISSUER": "https://broker/realms/x", PREFIX + "OIDC_AUDIENCE": "opendox-runtime"} for name in ASYMMETRIC_ALGORITHMS: @@ -319,7 +318,6 @@ def test_every_integer_setting_is_bounded_above_as_well_as_below() -> None: ) base = {PREFIX + "DATABASE_URL": "postgresql://u:p@h/db", - PREFIX + "MIGRATION_DATABASE_URL": "postgresql://m:p@h/db", PREFIX + "OIDC_AUDIENCE": "opendox", PREFIX + "OIDC_ISSUER": "https://broker/realms/x"} From f097fd889c1b07bad291794d41bad5a1969f6c5c Mon Sep 17 00:00:00 2001 From: Brett Heap <1513478+brettheap@users.noreply.github.com> Date: Mon, 28 Sep 2026 22:57:28 +0000 Subject: [PATCH 04/88] Fix round: load_migration_settings gets the same dialect gate load_settings has MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Copilot review of this PR (thread on _refuse_non_postgresql_dsn's own definition): 13.2's dialect gate was wired into `load_settings` only. `load_migration_settings` — the loader `runtime migrate`/`reset` actually use — read OPENDOX_MIGRATION_DATABASE_URL, checked only that it was non-empty, and handed it straight to `Database`, so a non-PostgreSQL migration DSN (`sqlite:///x.db`, say) reached the driver instead of being refused by name at configuration. That is the same un-named failure 13.2 exists to prevent for the served loader, just reachable through the one path F13.1's falsifier does not call. One call to the existing `_refuse_non_postgresql_dsn`, right after the existing empty-DSN refusal and before `database_url`/`migration_database_ url` are both set to the same value. New test `test_migrate_refuses_a_non_postgresql_migration_dsn_at_configuration` is the dialect-refused twin of the existing `test_migrate_and_reset_need_ no_served_identity_and_no_broker`, which already shows an unreachable but valid-dialect migration DSN getting PAST configuration — this one shows a wrong-dialect one refused AT configuration, naming the setting and never repeating the DSN. Measured locally (own Postgres container, bridge IP): 2473 passed (+1), 11 skipped, 1 failed (the same pre-existing, unrelated LC_CTYPE sandbox artifact) — the new test is the only change to the count. Arc: neutral-product-standalone-operability Lane: openxfactory-4 (openXfactory-4-openDox_extraction) Co-Authored-By: Claude Sonnet 5 --- src/opendox/runtime/config.py | 8 ++++++++ tests_runtime/test_runtime_cli.py | 23 +++++++++++++++++++++++ 2 files changed, 31 insertions(+) diff --git a/src/opendox/runtime/config.py b/src/opendox/runtime/config.py index deaf53e9..5c755006 100644 --- a/src/opendox/runtime/config.py +++ b/src/opendox/runtime/config.py @@ -1590,6 +1590,14 @@ def load_migration_settings(env: Mapping[str, str] | None = None) -> RuntimeSett f"{PREFIX}MIGRATION_DATABASE_URL is required to apply migrations; " f"{PREFIX}DATABASE_URL is the served runtime's least-privileged " "identity and is deliberately not used for schema changes") + # THE SAME DIALECT GATE `load_settings` ASKS, asked here too (Copilot + # review of this PR): this loader is the one path 13.2's own falsifier + # does not reach, and without this call a non-PostgreSQL migration DSN + # sailed past configuration entirely and reached `Database` instead, + # which is exactly the un-named, un-refused failure 13.2 exists to + # prevent for `load_settings`. `database_url` is set to this same `dsn` + # immediately below, so one call here covers both fields. + _refuse_non_postgresql_dsn(PREFIX + "MIGRATION_DATABASE_URL", dsn) return RuntimeSettings( database_url=dsn, migration_database_url=dsn, diff --git a/tests_runtime/test_runtime_cli.py b/tests_runtime/test_runtime_cli.py index 3b220eee..feadfa1a 100644 --- a/tests_runtime/test_runtime_cli.py +++ b/tests_runtime/test_runtime_cli.py @@ -386,6 +386,29 @@ def test_migrate_and_reset_need_no_served_identity_and_no_broker( assert evidence["ok"] is False +def test_migrate_refuses_a_non_postgresql_migration_dsn_at_configuration( + monkeypatch: pytest.MonkeyPatch) -> None: + """13.2 covers `load_migration_settings` too, not only `load_settings`. + + The test above shows an unreachable but VALID-dialect migration DSN + getting past configuration; this is its dialect-refused twin (Copilot + review of this PR): `load_migration_settings` read the DSN and handed it + straight to `Database` with no dialect check of its own, so a + non-PostgreSQL migration DSN reached the driver instead of being refused + by name here — the same un-named failure 13.2 exists to prevent for the + served loader. + """ + for name in ("DATABASE_URL", "OIDC_ISSUER", "OIDC_AUDIENCE"): + monkeypatch.delenv(PREFIX + name, raising=False) + monkeypatch.setenv(PREFIX + "MIGRATION_DATABASE_URL", "sqlite:///x.db") + code, evidence = _run(cli.build_parser().parse_args( + ["runtime", "migrate", "--connect-timeout", "0.2"])) + assert code == 1, evidence + assert evidence["refusal"] == "configuration", evidence + assert "postgres" in evidence["message"].lower(), evidence + assert PREFIX + "MIGRATION_DATABASE_URL" in evidence["message"] + + def test_the_entrypoint_turns_an_escaped_exception_into_evidence( monkeypatch: pytest.MonkeyPatch) -> None: """Every outcome is one redacted JSON object; none is a traceback.""" From b50e3b1dcb8c01a2619c987f3242ab9ba11e577d Mon Sep 17 00:00:00 2001 From: Brett Heap <1513478+brettheap@users.noreply.github.com> Date: Wed, 30 Sep 2026 00:21:49 +0000 Subject: [PATCH 05/88] =?UTF-8?q?T070:=2013.4,=2013.5=20and=2013.6=20?= =?UTF-8?q?=E2=80=94=20OPENDOX=5FINSTALL=5FMODE=20and=20generate-and-open?= =?UTF-8?q?=20--local=20(plan=20034)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit OPENDOX_INSTALL_MODE (`local` | `hosted`, default `hosted`) is read in runtime/config.py beside OPENDOX_OIDC_ISSUER and decides the install shape (#1144 13.4). `generate-and-open --local` makes the same selection (R1Q15 (b), as T007 batch H's 13.4 addendum reads); with neither the install is hosted (13.5). - A flag and a setting that disagree (`--local` beside OPENDOX_INSTALL_MODE=hosted) are refused, naming both. This is plan 034's fail-closed reading (Principle VII); no answer rules it and batch H does not write it into #1144. - LOCAL needs no broker: issuer, audience and key-set URL are empty. - LOCAL binds loopback only, with no opt-in. A non-loopback `--host` or OPENDOX_BIND_HOST is refused, naming the rule. The set is serve.py's own LOOPBACK_HOSTS, and a test holds the two equal. - HOSTED, set or by default, with no issuer refuses, naming OPENDOX_OIDC_ISSUER. generate-and-open asks the issuer first, so a run with nothing configured names it and `--local`. The hosted mode is otherwise unchanged (13.6). Holder readings on openxFactory#656 (Brett may overrule): - `runtime serve` refuses under local, because the API's identity is the broker's. - `runtime status` under local reports broker_keys "not configured (local mode)" and does not count it as a fault. - A broker setting beside local is refused by name. - An unrecognised mode value is refused, case-sensitively. The document server's generate-and-open resolves the shape before it scans, mints or binds anything. The hosted path loads the whole runtime configuration (R1Q16 (i); 13.4a). Also: - deploy/compose/.env.example gains OPENDOX_INSTALL_MODE=hosted, which test_every_runtime_setting_is_documented_in_env_example requires of every SETTINGS entry. - tests/test_doxbench_entrypoint.py's fixture now selects `--local` and scrubs the runtime settings, since the unset default is hosted and refuses with no issuer. Arc: neutral-product-standalone-operability Lane: openxfactory-4 (openXfactory-4-openDox_extraction) Co-Authored-By: Claude Opus 5.5 (1M context) --- deploy/compose/.env.example | 9 + src/opendox/cli.py | 64 ++++- src/opendox/runtime/cli.py | 39 ++- src/opendox/runtime/config.py | 240 ++++++++++++++++++- tests/test_doxbench_entrypoint.py | 11 + tests/test_install_mode_entrypoint.py | 208 ++++++++++++++++ tests_runtime/test_install_mode.py | 329 ++++++++++++++++++++++++++ 7 files changed, 889 insertions(+), 11 deletions(-) create mode 100644 tests/test_install_mode_entrypoint.py create mode 100644 tests_runtime/test_install_mode.py diff --git a/deploy/compose/.env.example b/deploy/compose/.env.example index 7d579a30..f8783755 100644 --- a/deploy/compose/.env.example +++ b/deploy/compose/.env.example @@ -48,6 +48,15 @@ OPENDOX_DATABASE_URL=postgresql://opendox_runtime:change-me-local-only-too@postg # `opendox-runtime runtime migrate` alone; the served application never receives it. OPENDOX_MIGRATION_DATABASE_URL=postgresql://opendox:change-me-local-only@postgres:5432/opendox +# The install shape: `hosted` or `local`, and this package is HOSTED — a +# broker, its pinned issuer and this file's own database. Unset means hosted +# too, which is the point: a hosted install that forgets its issuer REFUSES +# rather than falling into the local single-user mode (plan 034 T070; #1144 +# 13.4, 13.5). `local` is the one-user install `opendox generate-and-open +# --local` starts on a laptop, with no broker and loopback only; it refuses the +# broker settings below, so it is never selected here. +OPENDOX_INSTALL_MODE=hosted + # REQUIRED. The Keycloak broker's issuer, pinned (RULING Q2): a token from any # other issuer is refused rather than trusted. OPENDOX_OIDC_ISSUER=https://keycloak.example/realms/opendox diff --git a/src/opendox/cli.py b/src/opendox/cli.py index 994111dc..c7923abd 100644 --- a/src/opendox/cli.py +++ b/src/opendox/cli.py @@ -83,6 +83,11 @@ # `opendox.corpus_adapter` besides the stdlib. from opendox import corpus_adapter # noqa: E402 from opendox.runtime import local_git_adapter # noqa: E402 +# THE INSTALL SHAPE (plan 034 T070; #1144 13.4-13.6): `generate-and-open` +# resolves `--local` against `OPENDOX_INSTALL_MODE` here, before it generates +# or serves anything. Stdlib-only, like `local_git_adapter` above, which +# already imports it, so this adds no reach and no import weight. +from opendox.runtime import config as runtime_config # noqa: E402 from opendox.boundary import ( # noqa: E402 BoundaryViolation, HumanGate, OutputBoundary, ) @@ -409,11 +414,57 @@ def _validate(written: Path, args: argparse.Namespace, *, return 0 +def _resolve_install_shape(args: argparse.Namespace, + env=None) -> str: + """The install shape this run serves as, or `ConfigurationError` naming why. + + `--local` and `OPENDOX_INSTALL_MODE` are resolved by + `runtime_config.install_mode`, the one reading of the selector, which + refuses the two disagreeing (plan 034 T070's fail-closed reading) and + defaults to HOSTED (#1144 13.4, 13.5). Then each shape asks what it needs: + + * LOCAL binds loopback only, with no opt-in: a non-loopback `--host` is + refused naming the rule (13.4), and so is anything a local install + cannot be (`refuse_what_a_local_install_cannot_be`: a broker setting + beside it, or a non-loopback `OPENDOX_BIND_HOST`). It needs no broker. + Its datastore is 13.1's, and arrives with T072. + * HOSTED, set or by default, refuses with no issuer, NAMING THE ISSUER + (13.5), and then loads the runtime's whole configuration, because the + serving process is the one whose settings are the install's (13.4a; + R1Q16 (i)). Otherwise unchanged (13.6). + + Asked before anything is scanned, minted or bound, so a refused run leaves + nothing behind and exits at once rather than starting a server that a + bound would have to kill (F13.1's `test "$rc" -ne 124`). + """ + env = os.environ if env is None else env + mode = runtime_config.install_mode( + env, local_flag=bool(getattr(args, "local", False))) + if mode == runtime_config.INSTALL_MODE_LOCAL: + runtime_config.refuse_a_non_loopback_local_bind("--host", args.host) + runtime_config.refuse_what_a_local_install_cannot_be(env) + else: + runtime_config.require_the_hosted_issuer(env) + runtime_config.load_settings(env) + return mode + + def cmd_generate_and_open(args: argparse.Namespace, *, opener=webbrowser.open) -> int: """Regenerate the snapshot from the working tree into a run dir, start the local server, print the URL (ALWAYS), and open the browser. `--no-open` suppresses the browser; `--no-serve` returns after printing the URL without - blocking (used by tests). `opener` is injectable for testing.""" + blocking (used by tests). `opener` is injectable for testing. + + THE INSTALL SHAPE IS RESOLVED FIRST (plan 034 T070): `--local`, or + `OPENDOX_INSTALL_MODE=local`, selects the local single-user install, and + with neither the install is hosted — see `_resolve_install_shape`. A + refusal there is printed on stderr and the command exits 1, before any + other work.""" + try: + args.install_mode = _resolve_install_shape(args) + except runtime_config.ConfigurationError as exc: + print(f"generate-and-open refused: {exc}", file=sys.stderr) + return 1 # Ahead of minting the run dir, so a refused root leaves not even an empty # temp directory behind. `_generate_and_write` is still the guard that MATTERS # (it is the one no caller can skip); these are the same checks, earlier. @@ -968,6 +1019,17 @@ def build_parser(*, subcommand_extensions: tuple = ()) -> argparse.ArgumentParse gao.add_argument("--actor", default=None, help="human identity for loopback gate actions " "(default: the checkout's git user.name)") + # THE INSTALL SHAPE'S FLAG (plan 034 T070; R1Q15 (b), as T007 batch H's + # 13.4 addendum reads): the documented command is + # `opendox generate-and-open --local …`. The same selection as + # `OPENDOX_INSTALL_MODE=local`; with neither the install is hosted, and the + # flag beside `OPENDOX_INSTALL_MODE=hosted` is refused. + gao.add_argument(runtime_config.LOCAL_FLAG, action="store_true", + dest="local", + help="the LOCAL single-user install: no identity broker, " + "loopback only (the same selection as " + "OPENDOX_INSTALL_MODE=local; with neither, the " + "install is hosted and needs its broker's issuer)") gao.add_argument("--host", default=serve_mod.DEFAULT_HOST, help="bind host (default: 127.0.0.1, loopback only)") gao.add_argument("--port", type=int, default=0, help="bind port (default: ephemeral)") diff --git a/src/opendox/runtime/cli.py b/src/opendox/runtime/cli.py index 42921498..af0ac390 100644 --- a/src/opendox/runtime/cli.py +++ b/src/opendox/runtime/cli.py @@ -90,6 +90,8 @@ from opendox.runtime import identity, migrations from opendox.runtime.config import ( + INSTALL_MODE_LOCAL, + LOCAL_FLAG, SECRET_NAMES, SETTINGS, ConfigurationError, @@ -341,6 +343,11 @@ def _redacted_settings(settings: RuntimeSettings) -> dict[str, Any]: values = { "OPENDOX_DATABASE_URL": settings.database_url, "OPENDOX_MIGRATION_DATABASE_URL": settings.migration_database_url, + # THE INSTALL SHAPE this process loaded (plan 034 T070): a name and + # never a credential, and the first thing an operator reading `status` + # needs to know, because it decides whether the broker lines below + # mean anything at all. + "OPENDOX_INSTALL_MODE": settings.install_mode, # THE BROKER URLS ARE REDACTED HERE TOO. `load_settings` refuses # userinfo in the issuer and in an explicit JWKS URL — but this report # prints a DERIVED value, and a settings object can also be built by @@ -565,10 +572,31 @@ def cmd_migrate(args: argparse.Namespace) -> int: def cmd_serve(args: argparse.Namespace) -> int: - """Run the API. The pool is opened by the application's lifespan.""" + """Run the API. The pool is opened by the application's lifespan. + + NOT IN A LOCAL INSTALL (plan 034 T070; a holder reading on + openxFactory#656 that Brett may overrule). Every `/api/v1` route verifies + a token the BROKER signed (`oidc.build_verifier`), and the local mode has + no broker (#1144 13.4), so there is no identity this API could serve + with: started anyway, it would either refuse every request or, worse, + stand a local principal up that no task text defines. A local install is + served by `opendox generate-and-open --local`, and in release 1 its + document surface reads nothing from the store (R1Q16 (ii)). Refused + BEFORE anything is imported or bound, as evidence like every refusal. + """ settings = _settings_or_refusal(args) if isinstance(settings, int): return settings + if settings.install_mode == INSTALL_MODE_LOCAL: + return _emit({"verb": "serve", "refusal": "local-mode-has-no-broker", + "message": "the runtime API authenticates every request " + "with a token its identity broker signed, " + "and a LOCAL install has no broker, so this " + "API has no identity to serve with. A local " + "install is served by `opendox " + f"generate-and-open {LOCAL_FLAG}`; the " + "runtime API is a HOSTED install's surface " + "(13.4)"}, ok=False) try: import uvicorn @@ -744,6 +772,15 @@ def cmd_status(args: argparse.Namespace) -> int: f"unreachable: {type(exc).__name__}: {_safe_message(exc)}") ok = False + # A LOCAL INSTALL HAS NO BROKER TO PROBE (plan 034 T070; #1144 13.4), + # and that is its configuration rather than a fault: reported by name, and + # NOT counted against `ok`, so a healthy local install's `status` exits 0 + # — F13.1 runs it under `set -e`, and a verdict of "unhealthy" for a + # broker the install was never meant to have would be false. + if settings.install_mode == INSTALL_MODE_LOCAL: + report["broker_keys"] = "not configured (local mode)" + report["broker_discovery"] = None + return _emit(report, ok=ok) try: from opendox.runtime.oidc import build_verifier diff --git a/src/opendox/runtime/config.py b/src/opendox/runtime/config.py index 5c755006..6edd5c88 100644 --- a/src/opendox/runtime/config.py +++ b/src/opendox/runtime/config.py @@ -85,10 +85,23 @@ class Setting: "the PRIVILEGED DSN ordered-SQL migrations are applied with, used by " "`opendox runtime migrate` alone and never by the served application", ), + # THE INSTALL SHAPE, READ BESIDE THE ISSUER IT DECIDES ABOUT (plan 034 + # T070; #1144 13.4, 13.5). Not `required`: its default is the SAFE value, + # and "unset" is the case 13.4 names as the one that must be safe. + Setting( + PREFIX + "INSTALL_MODE", "hosted", False, False, + "the install shape: `hosted` (the default — the broker, the pinned " + "issuer and an operator's database, exactly as before) or `local` " + "(one user, no broker, loopback only). `generate-and-open --local` " + "makes the same selection; the two may not disagree, and with " + "neither the install is hosted, so a hosted install with no issuer " + "refuses rather than falling into local mode (13.4, 13.5)", + ), Setting( PREFIX + "OIDC_ISSUER", None, True, False, "the Keycloak broker's issuer, pinned: a token from any other issuer " - "is refused rather than trusted (RULING Q2)", + "is refused rather than trusted (RULING Q2). Required by a HOSTED " + "install; a LOCAL install has no broker and refuses one given here", ), Setting( PREFIX + "OIDC_AUDIENCE", None, True, False, @@ -188,10 +201,19 @@ class RuntimeSettings: docker-compose.yaml`'s `opendox` service and `docs/runtime.md` § 3 never supply it, and `load_migration_settings` is the loader that actually requires one (unaffected by this: it already refused to load without one). + + `install_mode` is `INSTALL_MODE_HOSTED` or `INSTALL_MODE_LOCAL` (plan 034 + T070; #1144 13.4). A LOCAL install has no broker, so its `oidc_issuer` and + `oidc_audience` are EMPTY and its `oidc_jwks_url` is `None` — never a + placeholder that looks like an endpoint — and `jwks_url()` and + `discovery_url()` answer the empty string for it rather than a path glued + onto nothing. A hosted install always carries a real issuer, because + `load_settings` refuses one without it. """ database_url: str migration_database_url: str | None + install_mode: str oidc_issuer: str oidc_audience: str oidc_jwks_url: str | None @@ -213,6 +235,7 @@ def __repr__(self) -> str: "RuntimeSettings(database_url=, " "migration_database_url=" f"{'' if self.migration_database_url else 'None'}, " + f"install_mode={self.install_mode!r}, " # REDACTED TOO, and not because `load_settings` allows userinfo # here — it refuses it. A `RuntimeSettings` built by hand, in a # test or by a future caller, does not go through that door, and @@ -241,13 +264,25 @@ def jwks_url(self) -> str: it saves one variable in the common case; setting it explicitly is what a broker behind a rewriting proxy needs, which is why the variable exists at all rather than the URL always being computed. + + EMPTY FOR A LOCAL INSTALL, which has no issuer to derive one from + (plan 034 T070): the derivation would otherwise answer the bare path + `/protocol/openid-connect/certs`, which `status` would print as if it + were a configured endpoint. """ if self.oidc_jwks_url: return self.oidc_jwks_url + if not self.oidc_issuer: + return "" return self.oidc_issuer.rstrip("/") + "/protocol/openid-connect/certs" def discovery_url(self) -> str: - """The issuer's discovery document, for `opendox runtime status`.""" + """The issuer's discovery document, for `opendox runtime status`. + + Empty for a local install, for the reason `jwks_url` gives. + """ + if not self.oidc_issuer: + return "" return self.oidc_issuer.rstrip("/") + "/.well-known/openid-configuration" @@ -1488,7 +1523,171 @@ def _refuse_two_dsns_that_select_different_schemas( "carries a password)") -def load_settings(env: Mapping[str, str] | None = None) -> RuntimeSettings: +#: THE TWO INSTALL SHAPES (plan 034 T070; #1144 13.4). One selector decides +#: the whole shape at once — the identity mode here, and the datastore source +#: 13.1 adds — because requirements 12 and 13 both describe "the standalone +#: install" and one deliberate choice should decide both. +INSTALL_MODE_HOSTED = "hosted" +INSTALL_MODE_LOCAL = "local" +INSTALL_MODES: tuple[str, ...] = (INSTALL_MODE_LOCAL, INSTALL_MODE_HOSTED) + +#: The flag that makes the same selection as `OPENDOX_INSTALL_MODE=local`, +#: spelled ONCE: `opendox.cli` declares `generate-and-open`'s option with this +#: constant, and every refusal below names it with the same one (R1Q15 (b), as +#: T007 batch H's 13.4 addendum reads). +LOCAL_FLAG = "--local" + +#: LOOPBACK, AS THE DOCUMENT SERVER ALREADY JUDGES IT. `serve.py` makes its +#: `session` capability conditional on a bind in exactly this set +#: (`serve.LOOPBACK_HOSTS`), and 13.4 asks the local mode to "make the same +#: judgement at the mode's own boundary" — so it is the same set, not +#: `_is_loopback` above: that one reads `127.0.0.0/8` as loopback, and a local +#: install bound to `127.0.0.2` would then pass here while the server it starts +#: treats that very bind as off-loopback. This module cannot import `serve` +#: (the import weight in `opendox/runtime/__init__.py`), so the set is spelled +#: here and `tests_runtime/test_install_mode.py` holds it equal to serve's. +LOCAL_BIND_HOSTS: frozenset[str] = frozenset({"127.0.0.1", "::1", "localhost"}) + +#: THE SETTINGS ONLY A HOSTED INSTALL READS. Given beside the local mode, each +#: is REFUSED, by name (a holder reading on openxFactory#656, plan 034 T070, +#: the same fail-closed reading as the disagreeing flag and setting): an issuer +#: next to `local` says a broker was meant, and honouring `local` over it would +#: silently drop the authentication the operator configured. T072 adds the two +#: DSNs, which the local install supplies itself (13.1). +HOSTED_ONLY_SETTINGS: tuple[str, ...] = ( + PREFIX + "OIDC_ISSUER", + PREFIX + "OIDC_AUDIENCE", + PREFIX + "OIDC_JWKS_URL", +) + + +def install_mode(env: Mapping[str, str] | None = None, *, + local_flag: bool = False) -> str: + """`INSTALL_MODE_LOCAL` or `INSTALL_MODE_HOSTED`, or a refusal naming why. + + THE DEFAULT IS HOSTED, and it is the default because it is the safe one + (#1144 13.4: "It is UNSET, not `local`, that must be safe"): an install + that sets nothing is hosted, and a hosted install with no issuer refuses + (13.5), so single-user operation is never reached by forgetting to + configure something. A BLANK value is unset, the reading `_optional` gives + every other setting. + + `local_flag` is `generate-and-open --local` (R1Q15 (b)). It selects local + exactly as `OPENDOX_INSTALL_MODE=local` does, and the two may not + DISAGREE: `--local` beside `OPENDOX_INSTALL_MODE=hosted` is refused naming + both, so no explicit selection is silently overridden by the other. No + answer on #656 rules that pair; the refusal is plan 034's fail-closed + reading (Principle VII, T070), recorded for Brett in + `evidence/analyze-round-2.md` and NOT written into #1144. + + AN UNRECOGNISED VALUE IS REFUSED, matched case-sensitively (a holder + reading on #656, T070): `Local` or `single-user` is not a spelling of + either shape, and guessing which one was meant is the one thing a + selector whose default is a safety property must not do. + """ + env = os.environ if env is None else env + setting = PREFIX + "INSTALL_MODE" + raw = env.get(setting, "").strip() + if raw and raw not in INSTALL_MODES: + raise ConfigurationError( + f"{setting} is {raw!r}, which is neither `local` nor `hosted` (the " + "two values are matched exactly, case included). Unset means " + "hosted; a single-user install selects `local` explicitly, with " + f"{setting}=local or `generate-and-open {LOCAL_FLAG}`") + if local_flag and raw == INSTALL_MODE_HOSTED: + raise ConfigurationError( + f"{LOCAL_FLAG} selects the LOCAL install and {setting}=hosted " + "selects the HOSTED one. Both are explicit selections and they " + "disagree, so neither is allowed to override the other: drop the " + f"flag for a hosted install, or unset {setting} (or set it to " + "`local`) for a local one") + if local_flag: + return INSTALL_MODE_LOCAL + return raw or INSTALL_MODE_HOSTED + + +def refuse_a_non_loopback_local_bind(name: str, host: str) -> None: + """A LOCAL install binds loopback only, with NO opt-in (#1144 13.4). + + `name` is what set the address — `--host` on `generate-and-open`, or + `OPENDOX_BIND_HOST` for the runtime's own listener — so the refusal names + the thing the operator actually typed. The local mode has no broker, so a + local install other machines can reach is an unauthenticated multi-user + service wearing the word "local"; an install that must be reachable from + another machine is a HOSTED install, with a broker. + """ + if host in LOCAL_BIND_HOSTS: + return + raise ConfigurationError( + f"{name} {host!r} is not a loopback address, and a LOCAL install binds " + f"LOOPBACK ONLY ({', '.join(sorted(LOCAL_BIND_HOSTS))}). The local " + "mode has no identity broker, so a local install another machine can " + "reach would be an unauthenticated multi-user service. There is no " + "opt-in: an install that must be reachable from another machine is a " + "HOSTED install, with a broker (13.4)") + + +def refuse_what_a_local_install_cannot_be(env: Mapping[str, str]) -> None: + """The two refusals a LOCAL install makes of its own environment. + + Every hosted-only setting given beside it (`HOSTED_ONLY_SETTINGS`), and a + non-loopback `OPENDOX_BIND_HOST`, the runtime's own listener. One function, + because `load_settings` and `generate-and-open --local` both ask it and the + two must not come to disagree about what a local install is. + """ + _refuse_hosted_only_settings(env) + refuse_a_non_loopback_local_bind( + PREFIX + "BIND_HOST", + _optional(env, _by_name(PREFIX + "BIND_HOST")) or "127.0.0.1") + + +def _refuse_hosted_only_settings(env: Mapping[str, str]) -> None: + """Every setting in `HOSTED_ONLY_SETTINGS` given beside `local`, named.""" + given = [name for name in HOSTED_ONLY_SETTINGS + if env.get(name, "").strip()] + if given: + raise ConfigurationError( + f"{' and '.join(given)} {'are' if len(given) > 1 else 'is'} set, " + "and this is a LOCAL install, which has no broker and reads " + f"{'none of them' if len(given) > 1 else 'none'}. A broker " + "setting beside the local mode says a HOSTED install was meant, " + "and honouring `local` over it would silently drop that " + f"authentication: unset {'them' if len(given) > 1 else 'it'} for " + f"a local install, or drop {LOCAL_FLAG} / " + f"{PREFIX}INSTALL_MODE=local for a hosted one (the values are not " + "repeated here)") + + +def require_the_hosted_issuer(env: Mapping[str, str] | None = None) -> None: + """A HOSTED install with no issuer refuses, NAMING THE ISSUER (#1144 13.5). + + `load_settings` asks for the served DSN before it asks for the issuer, so + a hosted `generate-and-open` run with NOTHING configured would otherwise + be refused naming `OPENDOX_DATABASE_URL` — true, and not the refusal 13.5 + and plan 034's requirement-13 scenario ask for, which is the one that + tells an operator this install is HOSTED and how to select the other one. + So the document server's entry point asks this first, and `load_settings` + keeps its own order for every verb that already relies on it (13.6: the + hosted mode is otherwise unchanged). + """ + env = os.environ if env is None else env + issuer = PREFIX + "OIDC_ISSUER" + if env.get(issuer, "").strip(): + return + selected = (f"{PREFIX}INSTALL_MODE=hosted" + if env.get(PREFIX + "INSTALL_MODE", "").strip() + else f"{PREFIX}INSTALL_MODE is unset, and unset means hosted") + raise ConfigurationError( + f"{issuer} is required and is not set, and this install is HOSTED " + f"({selected}). A hosted install authenticates through the broker " + "whose issuer this names, and it does NOT fall back to single-user " + "operation without one (13.5). A single-user install selects the " + f"local mode explicitly: `generate-and-open {LOCAL_FLAG}` or " + f"{PREFIX}INSTALL_MODE=local") + + +def load_settings(env: Mapping[str, str] | None = None, *, + local_flag: bool = False) -> RuntimeSettings: """Resolve :class:`RuntimeSettings` from `env` (default `os.environ`). Refuses with :class:`ConfigurationError` naming the variable — never with a @@ -1512,9 +1711,22 @@ def load_settings(env: Mapping[str, str] | None = None) -> RuntimeSettings: checks below still apply: a non-PostgreSQL migration DSN is refused (13.2), and the two being the exact same value is refused (13.3) — optional does not mean unchecked. + + THE INSTALL MODE FIRST (plan 034 T070; #1144 13.4-13.6). `local_flag` is + `generate-and-open --local`, resolved against `OPENDOX_INSTALL_MODE` by + `install_mode`, which refuses the two disagreeing. A HOSTED install — the + default — is exactly what this function has always loaded, in the same + order, with the issuer and audience required (13.6). A LOCAL install needs + no broker: its issuer, audience and key-set URL are empty, and any of the + three GIVEN beside it is refused (`HOSTED_ONLY_SETTINGS`); and its own + listener, `OPENDOX_BIND_HOST`, must be loopback, with no opt-in. """ env = os.environ if env is None else env + mode = install_mode(env, local_flag=local_flag) + local = mode == INSTALL_MODE_LOCAL + if local: + refuse_what_a_local_install_cannot_be(env) algorithms = _algorithms(env) served = _require(env, _by_name(PREFIX + "DATABASE_URL")) migration = _optional(env, _by_name(PREFIX + "MIGRATION_DATABASE_URL")) @@ -1538,18 +1750,23 @@ def load_settings(env: Mapping[str, str] | None = None) -> RuntimeSettings: # make. _refuse_the_same_dsn_in_both_settings(served, migration) + bind_host = _optional(env, _by_name(PREFIX + "BIND_HOST")) or "127.0.0.1" + return RuntimeSettings( database_url=served, migration_database_url=migration, - oidc_issuer=_broker_url(env, _by_name(PREFIX + "OIDC_ISSUER"), - required=True, is_a_base_url=True) or "", - oidc_audience=_require(env, _by_name(PREFIX + "OIDC_AUDIENCE")), - oidc_jwks_url=_broker_url(env, _by_name(PREFIX + "OIDC_JWKS_URL"), - required=False), + install_mode=mode, + oidc_issuer="" if local else _broker_url( + env, _by_name(PREFIX + "OIDC_ISSUER"), + required=True, is_a_base_url=True) or "", + oidc_audience="" if local else _require( + env, _by_name(PREFIX + "OIDC_AUDIENCE")), + oidc_jwks_url=None if local else _broker_url( + env, _by_name(PREFIX + "OIDC_JWKS_URL"), required=False), oidc_algorithms=algorithms, oidc_jwks_ttl_seconds=_positive_int(env, _by_name(PREFIX + "OIDC_JWKS_TTL_SECONDS")), oidc_leeway_seconds=_positive_int(env, _by_name(PREFIX + "OIDC_LEEWAY_SECONDS")), - bind_host=_optional(env, _by_name(PREFIX + "BIND_HOST")) or "127.0.0.1", + bind_host=bind_host, bind_port=_positive_int(env, _by_name(PREFIX + "BIND_PORT")), runtime_pg_role=_role_name(env), served_schema=_served_schema(env), @@ -1601,6 +1818,11 @@ def load_migration_settings(env: Mapping[str, str] | None = None) -> RuntimeSett return RuntimeSettings( database_url=dsn, migration_database_url=dsn, + # READ, SO AN UNRECOGNISED VALUE IS REFUSED HERE TOO (plan 034 T070): + # a migration run is part of the same install and one reading of the + # selector serves every verb. It changes nothing else a migration run + # does; the broker fields below are sentinels in either shape. + install_mode=install_mode(env), oidc_issuer=MIGRATION_SENTINEL_ISSUER, oidc_audience=MIGRATION_SENTINEL_AUDIENCE, oidc_jwks_url=None, diff --git a/tests/test_doxbench_entrypoint.py b/tests/test_doxbench_entrypoint.py index 0eac4803..2488fdf0 100644 --- a/tests/test_doxbench_entrypoint.py +++ b/tests/test_doxbench_entrypoint.py @@ -72,6 +72,7 @@ from opendox import doxbench_install as install_mod from opendox import doxbench_model from opendox import serve as serve_mod +from opendox.runtime import config as runtime_config def _handler_class(httpd): @@ -163,8 +164,18 @@ def _capture(*args, **kwargs): checkout = tmp_path / "checkout" checkout.mkdir() session_root = tmp_path / "model-sessions" + # THE LOCAL INSTALL, SELECTED EXPLICITLY (plan 034 T070; #1144 13.4, as + # T007 batch H's addendum reads). With neither `--local` nor + # `OPENDOX_INSTALL_MODE=local` the install is HOSTED, and a hosted install + # with no issuer refuses (13.5) before `build_server` is ever reached. This + # fixture drives the single-user entrypoint a student runs, so it says so, + # and it scrubs every runtime setting first: a broker setting inherited + # from the shell would be refused beside the local mode, by design. + for name in runtime_config.SETTING_NAMES: + monkeypatch.delenv(name, raising=False) args = cli_mod.build_parser().parse_args([ "generate-and-open", + runtime_config.LOCAL_FLAG, "--repo-root", str(checkout), "--repository", "fixture-repo", "--source-revision", PINNED_REVISION, diff --git a/tests/test_install_mode_entrypoint.py b/tests/test_install_mode_entrypoint.py new file mode 100644 index 00000000..2a121ef4 --- /dev/null +++ b/tests/test_install_mode_entrypoint.py @@ -0,0 +1,208 @@ +"""`generate-and-open`'s install shape: F13.1's refusals, run the way F13.1 +runs them (plan 034 T070; #1144 13.4, 13.5, 13.6). + +F13.1's refusal probes take the SAME `generate-and-open` path its local probe +takes, under `timeout 30`, and each must exit nonzero, must not be the bound's +124, and must name its rule on stderr. These cases run that path in a child +process for the same reason: a regression that silently STARTED a server +would block in-process forever, where a child is killed by the bound and the +case fails on `TimeoutExpired` instead of hanging the suite. The child is +`python -m opendox.cli`, which `cli.py`'s `__main__` block hands to the +package's `main()`, so it is the same `main()` the `opendox` console script +runs. + +Each probe gives a well-formed `--repo-root` (a fresh git repository) and every +other setting its case needs, so the install shape is the only fault; and the +install shape is resolved BEFORE the repo root is scanned, so the refusal is +the install's, whatever the tree holds. No child reaches a database, a broker +or a socket. + +The disagreeing flag and setting are plan 034's fail-closed reading (T070), +not a line of #1144; the rest is #1144's. +""" + +from __future__ import annotations + +import os +import subprocess +import sys +from pathlib import Path + +import pytest + +from opendox import cli as cli_mod +from opendox import serve as serve_mod +from opendox.runtime import config as runtime_config + +SRC = Path(__file__).resolve().parents[1] / "src" +PREFIX = runtime_config.PREFIX +MODE = PREFIX + "INSTALL_MODE" + +#: F13.1's hosted probes' environment: every setting a hosted install needs +#: EXCEPT the issuer, so the issuer is the only fault. +HOSTED_WITHOUT_ISSUER = { + PREFIX + "DATABASE_URL": "postgresql://serve@127.0.0.1:1/opendox", + PREFIX + "MIGRATION_DATABASE_URL": "postgresql://migrate@127.0.0.1:1/opendox", + PREFIX + "OIDC_AUDIENCE": "fixture", +} + + +@pytest.fixture() +def corpus(tmp_path: Path) -> Path: + """A fresh repository, as F13.1's preamble makes one.""" + root = tmp_path / "plain-documents" + root.mkdir() + (root / "note.md").write_text("# A note\n\nPlain text.\n", encoding="utf-8") + env = {**os.environ, + "GIT_AUTHOR_NAME": "fixture", "GIT_AUTHOR_EMAIL": "fixture@example.invalid", + "GIT_COMMITTER_NAME": "fixture", + "GIT_COMMITTER_EMAIL": "fixture@example.invalid"} + for argv in (["git", "init", "-q"], ["git", "add", "-A"], + ["git", "commit", "-qm", "fixture"]): + subprocess.run(argv, cwd=root, env=env, check=True) + return root + + +def _probe(corpus: Path, tmp_path: Path, *extra: str, + env: dict[str, str] | None = None) -> tuple[int, str]: + """One bounded `generate-and-open` child: `(returncode, stderr)`.""" + child_env = {name: value for name, value in os.environ.items() + if name not in runtime_config.SETTING_NAMES} + child_env.update(env or {}) + child_env["PYTHONPATH"] = os.pathsep.join( + [str(SRC), child_env.get("PYTHONPATH", "")]).rstrip(os.pathsep) + try: + done = subprocess.run( + [sys.executable, "-m", "opendox.cli", "generate-and-open", + "--repo-root", str(corpus), "--repository", "fixture", + "--run-dir", str(tmp_path / "run"), "--no-open", *extra], + env=child_env, capture_output=True, text=True, timeout=30) + except subprocess.TimeoutExpired as exc: # F13.1's rc 124 + raise AssertionError( + "generate-and-open did not refuse: it was still running when the " + "30-second bound killed it, which is a server that STARTED") from exc + return done.returncode, done.stderr + + +def test_local_mode_refuses_a_non_loopback_bind_naming_the_rule( + corpus: Path, tmp_path: Path) -> None: + """F13.1: `OPENDOX_INSTALL_MODE=local … --host 0.0.0.0` is refused, BOUNDED, + and stderr names loopback (13.4).""" + rc, err = _probe(corpus, tmp_path, "--host", "0.0.0.0", "--port", "0", + env={MODE: "local"}) + assert rc != 0, err + assert "loopback" in err.lower(), err + assert "--host" in err, err + + +def test_the_flag_refuses_a_non_loopback_bind_exactly_as_the_setting_does( + corpus: Path, tmp_path: Path) -> None: + rc, err = _probe(corpus, tmp_path, runtime_config.LOCAL_FLAG, + "--host", "0.0.0.0") + assert rc != 0, err + assert "loopback" in err.lower(), err + + +@pytest.mark.parametrize("mode", ["hosted", None]) +def test_a_hosted_or_unset_install_with_no_issuer_refuses_naming_it( + corpus: Path, tmp_path: Path, mode) -> None: + """F13.1's last two probes: hosted, and then the selector UNSET, each with + every hosted setting except the issuer; each refuses, BOUNDED, naming + `OPENDOX_OIDC_ISSUER` (13.5). The second is what proves the unset DEFAULT + refuses exactly as `hosted` does.""" + env = dict(HOSTED_WITHOUT_ISSUER) + if mode is not None: + env[MODE] = mode + rc, err = _probe(corpus, tmp_path, "--port", "0", env=env) + assert rc != 0, err + assert PREFIX + "OIDC_ISSUER" in err, err + + +def test_with_nothing_configured_the_refusal_names_the_issuer_and_the_flag( + corpus: Path, tmp_path: Path) -> None: + """Plan 034's requirement-13 scenario 2: with no setting at all the + install is hosted, and the refusal is about the ISSUER and names how to + select local — not `OPENDOX_DATABASE_URL`, which `load_settings` would + have asked for first.""" + rc, err = _probe(corpus, tmp_path, "--port", "0") + assert rc != 0, err + assert PREFIX + "OIDC_ISSUER" in err, err + assert runtime_config.LOCAL_FLAG in err, err + assert PREFIX + "DATABASE_URL" not in err, err + + +def test_a_flag_and_a_setting_that_disagree_are_refused_naming_both( + corpus: Path, tmp_path: Path) -> None: + """T070's own case (plan 034's fail-closed reading): `--local` beside + `OPENDOX_INSTALL_MODE=hosted`, with a COMPLETE hosted configuration, so + either selection alone would have been accepted.""" + env = {**HOSTED_WITHOUT_ISSUER, + PREFIX + "OIDC_ISSUER": "https://issuer.example.invalid/realms/x", + MODE: "hosted"} + rc, err = _probe(corpus, tmp_path, runtime_config.LOCAL_FLAG, env=env) + assert rc != 0, err + assert runtime_config.LOCAL_FLAG in err, err + assert f"{MODE}=hosted" in err, err + + +def test_a_broker_setting_beside_the_local_flag_is_refused_by_name( + corpus: Path, tmp_path: Path) -> None: + rc, err = _probe(corpus, tmp_path, runtime_config.LOCAL_FLAG, + env={PREFIX + "OIDC_ISSUER": + "https://issuer.example.invalid/realms/x"}) + assert rc != 0, err + assert PREFIX + "OIDC_ISSUER" in err, err + + +# -- in process: what each shape resolves to ---------------------------------- + + +def _args(*extra: str): + return cli_mod.build_parser().parse_args( + ["generate-and-open", "--repo-root", "/nonexistent", + "--repository", "fixture", *extra]) + + +def test_the_local_shape_needs_no_broker_and_no_setting_at_all() -> None: + """13.4: `local` needs no broker. Resolved with an EMPTY environment.""" + assert cli_mod._resolve_install_shape( + _args(runtime_config.LOCAL_FLAG), env={}) == \ + runtime_config.INSTALL_MODE_LOCAL + assert cli_mod._resolve_install_shape( + _args(), env={MODE: "local"}) == runtime_config.INSTALL_MODE_LOCAL + + +@pytest.mark.parametrize("host", sorted(serve_mod.LOOPBACK_HOSTS)) +def test_the_local_shape_accepts_each_loopback_host(host: str) -> None: + assert cli_mod._resolve_install_shape( + _args(runtime_config.LOCAL_FLAG, "--host", host), env={}) == \ + runtime_config.INSTALL_MODE_LOCAL + + +def test_a_complete_hosted_configuration_resolves_hosted_unchanged() -> None: + """13.6: a hosted install with its broker configured serves as before.""" + env = {**HOSTED_WITHOUT_ISSUER, + PREFIX + "OIDC_ISSUER": "https://issuer.example.invalid/realms/x"} + assert cli_mod._resolve_install_shape(_args(), env=env) == \ + runtime_config.INSTALL_MODE_HOSTED + # a hosted document server may still bind beyond loopback, as it always + # could: the loopback rule is the LOCAL mode's + assert cli_mod._resolve_install_shape( + _args("--host", "0.0.0.0"), env=env) == \ + runtime_config.INSTALL_MODE_HOSTED + + +def test_the_local_bind_rule_is_the_document_servers_own_loopback_set() -> None: + """13.4: "the same judgement at the mode's own boundary". `config` cannot + import `serve`, so it spells the set; this holds the two equal.""" + assert runtime_config.LOCAL_BIND_HOSTS == serve_mod.LOOPBACK_HOSTS + + +def test_the_flag_is_declared_on_generate_and_open_and_follows_the_verb() -> None: + """10.1: every option follows its verb; `--local` is `generate-and-open`'s.""" + assert _args(runtime_config.LOCAL_FLAG).local is True + assert _args().local is False + with pytest.raises(SystemExit): + cli_mod.build_parser().parse_args( + [runtime_config.LOCAL_FLAG, "generate-and-open", "--repo-root", + "/x", "--repository", "fixture"]) diff --git a/tests_runtime/test_install_mode.py b/tests_runtime/test_install_mode.py new file mode 100644 index 00000000..de9ca524 --- /dev/null +++ b/tests_runtime/test_install_mode.py @@ -0,0 +1,329 @@ +"""`OPENDOX_INSTALL_MODE`: the local single-user install, and the hosted one +it cannot be reached from by omission (plan 034 T070; #1144 13.4, 13.5, 13.6). + +HERMETIC: standard library plus `opendox.runtime.config` and the runtime CLI, +both stdlib-only at import. No database is reached: every DSN below is a +well-formed PostgreSQL URI aimed at port 1 of the loopback, so a refusal here +is always a CONFIGURATION refusal, which is the whole of what 13.4-13.6 ask of +`load_settings`. + +WHAT IS RULED AND WHAT IS READ, so a reviewer can tell them apart: + + * RULED: the selector, its two values and its hosted default (#1144 13.4); + a hosted install with no issuer refuses naming it (13.5); the hosted mode + unchanged (13.6); local binds loopback only with no opt-in (13.4); + `generate-and-open --local` is the same selection (R1Q15 (b), T007 batch + H's 13.4 addendum). + * PLAN 034's FAIL-CLOSED READING, not in #1144: a `--local` flag and an + `OPENDOX_INSTALL_MODE` setting that disagree are refused, naming both + (T070; `evidence/analyze-round-2.md` U2-1, V2-6). + * HOLDER READINGS on openxFactory#656 that Brett may overrule (T070): an + unrecognised value is refused, case-sensitively; a broker setting given + beside `local` is refused by name; `runtime serve` refuses under `local`; + `runtime status` under `local` reports the broker as not configured and + does not count it as a fault. +""" + +from __future__ import annotations + +import argparse +import io +import json +from contextlib import redirect_stdout + +import pytest + +from opendox.runtime import cli +from opendox.runtime.config import ( + HOSTED_ONLY_SETTINGS, + INSTALL_MODE_HOSTED, + INSTALL_MODE_LOCAL, + INSTALL_MODES, + LOCAL_BIND_HOSTS, + LOCAL_FLAG, + PREFIX, + SETTING_NAMES, + ConfigurationError, + install_mode, + load_migration_settings, + load_settings, + require_the_hosted_issuer, +) + +MODE = PREFIX + "INSTALL_MODE" + +#: Two DSNs that pass T071's three refusals — one dialect, one database, two +#: different values — so the install shape is the only thing under test. +DSNS = { + PREFIX + "DATABASE_URL": "postgresql://serve@127.0.0.1:1/opendox", + PREFIX + "MIGRATION_DATABASE_URL": "postgresql://migrate@127.0.0.1:1/opendox", +} +#: Every setting a HOSTED install needs, well-formed. +HOSTED = {**DSNS, + PREFIX + "OIDC_ISSUER": "https://issuer.example.invalid/realms/fixture", + PREFIX + "OIDC_AUDIENCE": "fixture"} + + +def _refusal(env: dict, **kwargs) -> str: + try: + load_settings(env, **kwargs) + except ConfigurationError as exc: + return str(exc) + raise AssertionError(f"accepted: {env} {kwargs}") + + +@pytest.fixture() +def scrubbed(monkeypatch: pytest.MonkeyPatch) -> pytest.MonkeyPatch: + """No runtime setting inherited from the shell reaches a CLI verb.""" + for name in SETTING_NAMES: + monkeypatch.delenv(name, raising=False) + return monkeypatch + + +def _run(argv: list[str]) -> tuple[int, dict]: + args: argparse.Namespace = cli.build_parser().parse_args(argv) + buffer = io.StringIO() + with redirect_stdout(buffer): + code = args.func(args) + return code, json.loads(buffer.getvalue()) + + +# -- the selector ------------------------------------------------------------- + + +def test_the_selector_has_two_values_and_its_default_is_hosted() -> None: + """13.4: `local` and `hosted`, defaulting to `hosted`; UNSET is the safe one.""" + assert set(INSTALL_MODES) == {"local", "hosted"} + assert install_mode({}) == INSTALL_MODE_HOSTED + assert install_mode({MODE: ""}) == INSTALL_MODE_HOSTED + assert install_mode({MODE: " "}) == INSTALL_MODE_HOSTED + assert install_mode({MODE: "hosted"}) == INSTALL_MODE_HOSTED + assert install_mode({MODE: "local"}) == INSTALL_MODE_LOCAL + + +def test_the_flag_selects_local_exactly_as_the_setting_does() -> None: + """R1Q15 (b), as T007 batch H's 13.4 addendum reads.""" + assert LOCAL_FLAG == "--local" + assert install_mode({}, local_flag=True) == INSTALL_MODE_LOCAL + assert install_mode({MODE: "local"}, local_flag=True) == INSTALL_MODE_LOCAL + assert install_mode({MODE: "local"}) == install_mode({}, local_flag=True) + + +def test_a_flag_and_a_setting_that_disagree_are_refused_naming_both() -> None: + """T070's falsifier's second half: the disagreeing pair (plan 034's reading). + + Neither explicit selection may silently override the other, in either + direction the loader can see: the flag cannot win over a hosted setting, + and the setting cannot win over the flag. + """ + with pytest.raises(ConfigurationError) as caught: + install_mode({MODE: "hosted"}, local_flag=True) + message = str(caught.value) + assert LOCAL_FLAG in message, message + assert f"{MODE}=hosted" in message, message + # ...and the same refusal on the loader every served verb goes through, + # so the pair cannot be resolved one way here and another way there. + assert f"{MODE}=hosted" in _refusal({**HOSTED, MODE: "hosted"}, + local_flag=True) + + +@pytest.mark.parametrize("value", ["Local", "LOCAL", "Hosted", "single-user", + "locals", "loc al"]) +def test_an_unrecognised_value_is_refused_naming_the_two(value: str) -> None: + """A holder reading (#656, T070): matched case-sensitively, never guessed.""" + with pytest.raises(ConfigurationError) as caught: + install_mode({MODE: value}) + message = str(caught.value) + assert MODE in message and "`local`" in message and "`hosted`" in message + # every loader reads the one selector, so every loader refuses it + assert MODE in _refusal({**HOSTED, MODE: value}) + with pytest.raises(ConfigurationError): + load_migration_settings({**DSNS, MODE: value}) + + +# -- 13.5 and 13.6: the hosted install --------------------------------------- + + +@pytest.mark.parametrize("mode", [None, "hosted", ""]) +def test_a_hosted_install_with_no_issuer_refuses_naming_it(mode) -> None: + """13.5, set or by default, with every other hosted setting well-formed.""" + env = {k: v for k, v in HOSTED.items() if k != PREFIX + "OIDC_ISSUER"} + if mode is not None: + env[MODE] = mode + assert PREFIX + "OIDC_ISSUER" in _refusal(env) + with pytest.raises(ConfigurationError) as caught: + require_the_hosted_issuer(env) + assert PREFIX + "OIDC_ISSUER" in str(caught.value) + assert LOCAL_FLAG in str(caught.value), ( + "the refusal must say how a single-user install selects local") + + +def test_the_hosted_issuer_is_asked_first_by_the_entry_point_check() -> None: + """With NOTHING set, the entry point's refusal names the issuer (13.5), + not the served DSN `load_settings` happens to ask for first.""" + with pytest.raises(ConfigurationError) as caught: + require_the_hosted_issuer({}) + assert PREFIX + "OIDC_ISSUER" in str(caught.value) + assert "unset" in str(caught.value) + require_the_hosted_issuer(HOSTED) # and a present issuer passes + + +def test_the_hosted_mode_is_unchanged() -> None: + """13.6: same broker, same pinned issuer, loaded exactly as before.""" + for env in (HOSTED, {**HOSTED, MODE: "hosted"}): + settings = load_settings(env) + assert settings.install_mode == INSTALL_MODE_HOSTED + assert settings.oidc_issuer == HOSTED[PREFIX + "OIDC_ISSUER"] + assert settings.oidc_audience == "fixture" + assert settings.jwks_url() == (HOSTED[PREFIX + "OIDC_ISSUER"] + + "/protocol/openid-connect/certs") + # and a hosted install may still bind wherever its operator says + assert load_settings({**HOSTED, PREFIX + "BIND_HOST": "0.0.0.0"} + ).bind_host == "0.0.0.0" + # and it still refuses a missing audience, as it always did + env = {k: v for k, v in HOSTED.items() if k != PREFIX + "OIDC_AUDIENCE"} + assert PREFIX + "OIDC_AUDIENCE" in _refusal(env) + + +# -- 13.4: the local install ------------------------------------------------- + + +@pytest.mark.parametrize("selection", ["setting", "flag"]) +def test_a_local_install_needs_no_broker(selection: str) -> None: + env = dict(DSNS) + kwargs = {} + if selection == "setting": + env[MODE] = "local" + else: + kwargs["local_flag"] = True + settings = load_settings(env, **kwargs) + assert settings.install_mode == INSTALL_MODE_LOCAL + assert settings.oidc_issuer == "" + assert settings.oidc_audience == "" + assert settings.oidc_jwks_url is None + # no endpoint is derived from an issuer that does not exist + assert settings.jwks_url() == "" + assert settings.discovery_url() == "" + assert "install_mode='local'" in repr(settings) + + +@pytest.mark.parametrize("name", HOSTED_ONLY_SETTINGS) +def test_a_broker_setting_beside_the_local_mode_is_refused_by_name( + name: str) -> None: + """A holder reading (#656, T070): a broker setting says hosted was meant.""" + assert set(HOSTED_ONLY_SETTINGS) == {PREFIX + "OIDC_ISSUER", + PREFIX + "OIDC_AUDIENCE", + PREFIX + "OIDC_JWKS_URL"} + secret = "https://svc:hunter2@broker.example.invalid/realms/x" + message = _refusal({**DSNS, MODE: "local", name: secret}) + assert name in message, message + assert "hunter2" not in message, "the value must not be repeated" + # and the flag spelling of the same selection refuses it the same way + assert name in _refusal({**DSNS, name: secret}, local_flag=True) + + +def test_every_broker_setting_given_is_named_at_once() -> None: + env = {**HOSTED, MODE: "local"} + message = _refusal(env) + for name in (PREFIX + "OIDC_ISSUER", PREFIX + "OIDC_AUDIENCE"): + assert name in message, message + + +@pytest.mark.parametrize("host", sorted(LOCAL_BIND_HOSTS)) +def test_a_local_install_binds_each_loopback_spelling(host: str) -> None: + settings = load_settings({**DSNS, MODE: "local", + PREFIX + "BIND_HOST": host}) + assert settings.bind_host == host + + +@pytest.mark.parametrize("host", ["0.0.0.0", "::", "192.0.2.10", + "127.0.0.2", "example.invalid"]) +def test_a_local_install_refuses_a_non_loopback_bind_naming_the_rule( + host: str) -> None: + """13.4: loopback ONLY, and no opt-in. `127.0.0.2` is refused too: the + document server does not treat it as loopback (`serve.LOOPBACK_HOSTS`), + and the mode makes the SAME judgement at its own boundary.""" + message = _refusal({**DSNS, MODE: "local", PREFIX + "BIND_HOST": host}) + assert PREFIX + "BIND_HOST" in message, message + assert "loopback" in message.lower(), message + assert "no opt-in" in message.lower(), message + + +# -- the runtime CLI under the two modes -------------------------------------- + + +def test_runtime_serve_refuses_under_the_local_mode(scrubbed) -> None: + """A holder reading (#656, T070): the API's identity is the broker's. + + uvicorn and the application are STUBBED, so that a regression which + served anyway returns at once — and fails the assertions below — instead + of binding a real listener and blocking the suite forever (measured: the + un-stubbed form of this case hung under exactly that mutant). + """ + import sys + import types + + from opendox.runtime import app as app_module + + served = [] + + class _Config: + def __init__(self, app: object, **kwargs: object) -> None: + pass + + class _Server: + def __init__(self, config: object) -> None: + self.started = False + + def run(self) -> None: + served.append(True) + self.started = True + + stub = types.ModuleType("uvicorn") + stub.Config, stub.Server = _Config, _Server + scrubbed.setitem(sys.modules, "uvicorn", stub) + scrubbed.setattr(app_module, "create_app", lambda **kwargs: object()) + for name, value in DSNS.items(): + scrubbed.setenv(name, value) + scrubbed.setenv(MODE, "local") + code, evidence = _run(["runtime", "serve"]) + assert served == [], "the API was started for a LOCAL install" + assert code == 1 + assert evidence["ok"] is False + assert evidence["refusal"] == "local-mode-has-no-broker", evidence + assert f"generate-and-open {LOCAL_FLAG}" in evidence["message"] + + +def test_runtime_status_under_the_local_mode_probes_no_broker( + scrubbed, monkeypatch: pytest.MonkeyPatch) -> None: + """`broker_keys` is reported as not configured, and the verifier is never + built: a local install has no broker to reach, and a status verb that + called that a fault would exit nonzero for a healthy install.""" + from opendox.runtime import oidc + + def _no_broker(_settings): + raise AssertionError("status built a broker verifier for a LOCAL " + "install, which has no broker") + + monkeypatch.setattr(oidc, "build_verifier", _no_broker) + for name, value in DSNS.items(): + scrubbed.setenv(name, value) + scrubbed.setenv(MODE, "local") + code, evidence = _run(["runtime", "status", "--probe-timeout", "0.2"]) + assert evidence.get("refusal") is None, evidence + assert evidence["broker_keys"] == "not configured (local mode)" + assert evidence["broker_discovery"] is None + assert evidence["settings"][MODE] == INSTALL_MODE_LOCAL + assert evidence["settings"][PREFIX + "OIDC_ISSUER"] == "" + assert evidence["settings"][PREFIX + "OIDC_JWKS_URL"] == "" + # the database half (port 1, unreachable) is the ONLY reason `ok` is false + assert evidence["database"].startswith("unreachable"), evidence + assert code == 1 + + +def test_runtime_status_reports_the_hosted_mode_it_loaded(scrubbed) -> None: + for name, value in HOSTED.items(): + scrubbed.setenv(name, value) + _code, evidence = _run(["runtime", "status", "--probe-timeout", "0.2"]) + assert evidence["settings"][MODE] == INSTALL_MODE_HOSTED + assert evidence["broker_keys"].startswith("unreachable"), evidence From 2c96dfbbbfefe29cd75eb6000ec0ba8c087d5f19 Mon Sep 17 00:00:00 2001 From: Brett Heap <1513478+brettheap@users.noreply.github.com> Date: Wed, 30 Sep 2026 00:47:47 +0000 Subject: [PATCH 06/88] =?UTF-8?q?T072:=2013.1=20=E2=80=94=20the=20bundled?= =?UTF-8?q?=20PostgreSQL=20server,=20the=20local=20install's=20own=20child?= =?UTF-8?q?=20(plan=20034)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A LOCAL install (T070's `generate-and-open --local`, or OPENDOX_INSTALL_MODE=local) now brings its own database (#1144 13.1, as T007 batch H's addendum reads; RULED R1Q16 (i)-(iv), 5850003126). - (i) `generate-and-open --local` starts a PostgreSQL server as its own direct child (subprocess.Popen, never pg_ctl) and reports it. The document server a user reaches is the process that owns it. - (ii) The server is started AND migrated: initdb once, an idempotent bootstrap (the database, the served role, and the compose stack's grants narrowed to this install's owner), then migrations.MigrationRunner as the owner, with the served role and database declared. - (iii) It ships as the `opendox[local]` extra: `opendox[runtime]` plus `pgserver>=0.1.4`, whose bundled binaries link only libc and libz. The `test` extra joins it, so F9.1's `.[test]` install still runs every case. - (iv) It stops with the entry point. SIGTERM is read as the Ctrl-C the serve loop already stops on, followed by a fast shutdown. PR_SET_PDEATHSIG is the backstop when the entry point is SIGKILLed. - Its data and socket directories live under OPENDOX_STATE_DIR, a new setting that defaults per user and must be absolute. The server listens on a 0700 Unix socket with listen_addresses empty: no TCP listener at all. - Both DSNs are supplied: two users over the one socket, which pass T071's three checks. An operator DSN beside `local` is refused by name, joining T070's broker settings (a holder reading on openxFactory#656). - `runtime status` reports database_bundle (data_dir, socket_dir, pid). `runtime migrate` under local migrates the bundle. THE MIGRATIONS GAP (assigned to T072 by the holder). pyproject maps migrations/*.sql into the wheel's data directory (share/opendox/migrations), without moving the root migrations/ that the image copies. An unset OPENDOX_MIGRATIONS_DIR is `migrations` wherever the working directory has one (today's default, unchanged), and otherwise the copy the installed distribution records. A test builds the wheel, installs it outside the checkout, runs from a directory with no migrations/, and migrates the bundled server. Also: - deploy/compose/.env.example gains OPENDOX_STATE_DIR=, because every SETTINGS entry is named there. - tests/test_doxbench_entrypoint.py stands the bundle in, since those cases test the model port. - T070's own tests stop passing DSNs beside `local`. Arc: neutral-product-standalone-operability Lane: openxfactory-4 (openXfactory-4-openDox_extraction) Co-Authored-By: Claude Opus 5.5 (1M context) --- deploy/compose/.env.example | 7 + pyproject.toml | 48 +++ src/opendox/cli.py | 82 ++++- src/opendox/runtime/bundle.py | 444 +++++++++++++++++++++++ src/opendox/runtime/cli.py | 18 + src/opendox/runtime/config.py | 267 +++++++++++++- tests/test_doxbench_entrypoint.py | 26 ++ tests/test_install_mode_entrypoint.py | 22 +- tests_runtime/local_entrypoint_driver.py | 77 ++++ tests_runtime/test_bundled_postgres.py | 415 +++++++++++++++++++++ tests_runtime/test_install_mode.py | 37 +- tests_runtime/test_runtime_surface.py | 5 + 12 files changed, 1397 insertions(+), 51 deletions(-) create mode 100644 src/opendox/runtime/bundle.py create mode 100644 tests_runtime/local_entrypoint_driver.py create mode 100644 tests_runtime/test_bundled_postgres.py diff --git a/deploy/compose/.env.example b/deploy/compose/.env.example index f8783755..5916932d 100644 --- a/deploy/compose/.env.example +++ b/deploy/compose/.env.example @@ -57,6 +57,13 @@ OPENDOX_MIGRATION_DATABASE_URL=postgresql://opendox:change-me-local-only@postgre # broker settings below, so it is never selected here. OPENDOX_INSTALL_MODE=hosted +# Where a LOCAL install keeps its bundled PostgreSQL server's data directory and +# Unix socket (plan 034 T072; #1144 13.1). This package is hosted and never +# reads it; it is here because every setting the runtime reads is named here. +# Empty means the per-user default, `$XDG_STATE_HOME/opendox` or +# `~/.local/state/opendox`. +OPENDOX_STATE_DIR= + # REQUIRED. The Keycloak broker's issuer, pinned (RULING Q2): a token from any # other issuer is refused rather than trusted. OPENDOX_OIDC_ISSUER=https://keycloak.example/realms/opendox diff --git a/pyproject.toml b/pyproject.toml index 10a82163..7be64df0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -93,9 +93,44 @@ dependencies = [ # (`tests_runtime/conftest.py`), and a skipped case does not make a whole # suite. `opendox[runtime]` names the extra below instead of copying its five # lines, so the runtime's dependency list is still declared once. +# +# AND THE LOCAL EXTRA, the same way and for the same reason (plan 034 T072; +# R1Q16 (iii), `5850003126`): the local install's bundled PostgreSQL server is +# what `tests_runtime/test_bundled_postgres.py` starts, so F9.1's `.[test]` +# install still runs every case. `opendox[local]` carries `opendox[runtime]`; +# both are named so the T036 reason above stays readable on its own. +# +# AND `setuptools`, the backend `[build-system]` names: the same test BUILDS +# this package's wheel and installs it outside the checkout, to prove a wheel +# carries its migrations (plan 034 T072). Declared here so that build runs +# offline, with `--no-build-isolation`, in the environment the suite runs in. +# `>=70.1` and not `[build-system]`'s `>=68`: 70.1 is the first release that +# builds a wheel with no separate `wheel` package installed. test = [ "pytest>=8,<9", "opendox[runtime]", + "opendox[local]", + "setuptools>=70.1", +] + +# THE LOCAL EXTRA — the standalone install (plan 034 T072; #1144 13.1, as T007 +# batch H's addendum reads; RULED R1Q16 (iii), openxFactory#656 comment +# `5850003126`): `pip install "opendox[local]"`, then +# `opendox generate-and-open --local …`. It carries the runtime's packages and +# the bundled server's own, and nothing else. +# +# `pgserver` IS THE SERVER'S CARRIER, and only its binaries are used (see +# `opendox/runtime/bundle.py` for why its own manager is not): it ships +# PostgreSQL 16 as `initdb` and `postgres` inside the wheel, built to link only +# libc and libz, so a manylinux2014 host needs nothing else installed — no +# system PostgreSQL, no ICU. Apache-2.0; the server it carries is under the +# PostgreSQL License. `>=0.1.4` is the release this package has been exercised +# against (measured 2026-09-30, python 3.12.3: PostgreSQL 16.2), the same rule +# every floor in this file is set by. Its own three requirements (psutil, +# platformdirs, fasteners) arrive with it and are imported by nothing here. +local = [ + "opendox[runtime]", + "pgserver>=0.1.4", ] # THE RUNTIME EXTRA — `split-opendox-two-layer-product` § 3.5, RULED Q2 @@ -206,6 +241,19 @@ py-modules = ["route_extension", "subcommand_extension"] [tool.setuptools.packages.find] where = ["src"] +# THE MIGRATIONS, SHIPPED IN THE WHEEL (plan 034 T072). `migrations/` stays at +# the repository root, where the image copies it (`deploy/compose/Dockerfile`: +# `COPY migrations ./migrations`, run from `/app`) and every checkout run finds +# it; this maps the same files into the wheel's data directory, so an install +# run OUTSIDE a checkout still has the migrations its bundled server applies. +# `opendox.runtime.config.packaged_migrations_dir` finds them through the +# installed distribution's own record, and the canonical digest gate is what +# proves the copy found is the pinned one. A data directory and not package +# data because package data must live inside the package, and the one tree is +# not moved. +[tool.setuptools.data-files] +"share/opendox/migrations" = ["migrations/*.sql"] + # THE SERVED BUNDLE, PACKAGED — § 3.4 slice S5, and it is a gap this slice had # to close to make RULED Q5 true rather than true-in-a-source-checkout. # diff --git a/src/opendox/cli.py b/src/opendox/cli.py index c7923abd..11c3b203 100644 --- a/src/opendox/cli.py +++ b/src/opendox/cli.py @@ -18,6 +18,7 @@ import argparse import os +import signal import sys import tempfile import webbrowser @@ -88,6 +89,12 @@ # or serves anything. Stdlib-only, like `local_git_adapter` above, which # already imports it, so this adds no reach and no import weight. from opendox.runtime import config as runtime_config # noqa: E402 +# THE LOCAL INSTALL'S BUNDLED POSTGRESQL SERVER (plan 034 T072; #1144 13.1, +# R1Q16 (i)-(iv)): started as THIS process's child by `generate-and-open +# --local`, and stopped with it. Stdlib-only at import, like `runtime_config`; +# the driver is imported when the server is started, never here. +from opendox.runtime import bundle as bundle_mod # noqa: E402 +from opendox.runtime import migrations as migrations_mod # noqa: E402 from opendox.boundary import ( # noqa: E402 BoundaryViolation, HumanGate, OutputBoundary, ) @@ -415,8 +422,8 @@ def _validate(written: Path, args: argparse.Namespace, *, def _resolve_install_shape(args: argparse.Namespace, - env=None) -> str: - """The install shape this run serves as, or `ConfigurationError` naming why. + env=None) -> runtime_config.RuntimeSettings: + """The settings this run serves with, or `ConfigurationError` naming why. `--local` and `OPENDOX_INSTALL_MODE` are resolved by `runtime_config.install_mode`, the one reading of the selector, which @@ -426,27 +433,35 @@ def _resolve_install_shape(args: argparse.Namespace, * LOCAL binds loopback only, with no opt-in: a non-loopback `--host` is refused naming the rule (13.4), and so is anything a local install cannot be (`refuse_what_a_local_install_cannot_be`: a broker setting - beside it, or a non-loopback `OPENDOX_BIND_HOST`). It needs no broker. - Its datastore is 13.1's, and arrives with T072. + beside it, a DSN beside it, or a non-loopback `OPENDOX_BIND_HOST`). It + needs no broker, and it supplies BOTH DSNs itself, from the server it + bundles under `OPENDOX_STATE_DIR` (13.1; plan 034 T072). * HOSTED, set or by default, refuses with no issuer, NAMING THE ISSUER - (13.5), and then loads the runtime's whole configuration, because the - serving process is the one whose settings are the install's (13.4a; - R1Q16 (i)). Otherwise unchanged (13.6). + (13.5), and then loads the runtime's whole configuration. + Otherwise unchanged (13.6). + + Either way the result is the runtime's own `load_settings`, because the + serving process is the one whose settings are the install's (13.4a; + R1Q16 (i)). Asked before anything is scanned, minted or bound, so a refused run leaves nothing behind and exits at once rather than starting a server that a bound would have to kill (F13.1's `test "$rc" -ne 124`). """ env = os.environ if env is None else env - mode = runtime_config.install_mode( - env, local_flag=bool(getattr(args, "local", False))) + local_flag = bool(getattr(args, "local", False)) + mode = runtime_config.install_mode(env, local_flag=local_flag) if mode == runtime_config.INSTALL_MODE_LOCAL: runtime_config.refuse_a_non_loopback_local_bind("--host", args.host) runtime_config.refuse_what_a_local_install_cannot_be(env) else: runtime_config.require_the_hosted_issuer(env) - runtime_config.load_settings(env) - return mode + return runtime_config.load_settings(env, local_flag=local_flag) + + +def _terminate_as_interrupt(signum, frame): # pragma: no cover - a signal + """SIGTERM, read as the Ctrl-C the serve loop already stops cleanly on.""" + raise KeyboardInterrupt def cmd_generate_and_open(args: argparse.Namespace, *, opener=webbrowser.open) -> int: @@ -459,12 +474,53 @@ def cmd_generate_and_open(args: argparse.Namespace, *, opener=webbrowser.open) - `OPENDOX_INSTALL_MODE=local`, selects the local single-user install, and with neither the install is hosted — see `_resolve_install_shape`. A refusal there is printed on stderr and the command exits 1, before any - other work.""" + other work. + + A LOCAL RUN OWNS ITS DATABASE (plan 034 T072; #1144 13.1, R1Q16 (i)-(iv)). + Once the corpus root and the anchors are known good, the bundled + PostgreSQL server is started as THIS process's child, bootstrapped and + migrated — starting and migrating it is all release 1 asks of it — and it + is stopped when this command returns, however it returns: a served run + ended by Ctrl-C or by SIGTERM (read here as the same interrupt), a + `--no-serve` run, or a failure. A refused start exits 1 on stderr, like + every other refusal of this verb.""" try: - args.install_mode = _resolve_install_shape(args) + settings = _resolve_install_shape(args) except runtime_config.ConfigurationError as exc: print(f"generate-and-open refused: {exc}", file=sys.stderr) return 1 + args.install_mode = settings.install_mode + args.runtime_settings = settings + if settings.install_mode != runtime_config.INSTALL_MODE_LOCAL: + return _generate_and_open(args, opener=opener) + # THE CHEAP REFUSALS FIRST, so a mistyped root never costs a database start. + _refuse_non_corpus_repo_root(args) + _refuse_malformed_generated_at(args) + server = bundle_mod.BundledServer(settings) + args.database_bundle = server + try: + previous = signal.signal(signal.SIGTERM, _terminate_as_interrupt) + except ValueError: # not the main thread: no handler to own + previous = None + try: + try: + server.start() + except (bundle_mod.BundleRefused, runtime_config.ConfigurationError, + migrations_mod.MigrationError) as exc: + print(f"generate-and-open refused: {exc}", file=sys.stderr) + return 1 + report = server.report() + print(f" database {report['socket_dir']} (bundled, pid {report['pid']}, " + f"migrations applied now: {server.applied or 'none pending'})") + return _generate_and_open(args, opener=opener) + finally: + server.stop() + if previous is not None: + signal.signal(signal.SIGTERM, previous) + + +def _generate_and_open(args: argparse.Namespace, *, opener) -> int: + """`generate-and-open`'s generate-then-serve half, once the install is known.""" # Ahead of minting the run dir, so a refused root leaves not even an empty # temp directory behind. `_generate_and_write` is still the guard that MATTERS # (it is the one no caller can skip); these are the same checks, earlier. diff --git a/src/opendox/runtime/bundle.py b/src/opendox/runtime/bundle.py new file mode 100644 index 00000000..701aa1bf --- /dev/null +++ b/src/opendox/runtime/bundle.py @@ -0,0 +1,444 @@ +"""A LOCAL install's bundled PostgreSQL server (plan 034 T072; #1144 13.1). + +WHAT #1144 FIXES AND WHAT IT LEAVES. 13.1 fixes the server's IDENTITY: its data +directory and its Unix socket live under the install's own `OPENDOX_STATE_DIR`, +it listens on that socket and on NO TCP port, the product supplies BOTH DSNs +itself, and `runtime status` reports a `database_bundle` naming `data_dir`, +`socket_dir` and the server's `pid`. It leaves the PACKAGING to the realization, +and R1Q16's answer (openxFactory#656 `5850003126`, as T007 batch H's 13.1 +addendum records it) names it: + + (i) the document server starts the bundled server as ITS OWN CHILD and + reports it, so the process a user reaches is the one that owns it; + (ii) starting and migrating the store is all release 1 asks of it, since + the document surface reads nothing from it yet; + (iii) it ships as the `opendox[local]` extra, which carries the runtime's + packages and the server's own; + (iv) it stops with the entry point. + +THE SERVER'S OWN PACKAGE IS `pgserver` (pyproject.toml's `local` extra), and +only its BINARIES are used: `initdb` and `postgres` from the wheel's +`pginstall/bin`, found by `importlib.util.find_spec` without importing +`pgserver` at all. Its Python manager is deliberately not used — it daemonizes +the server through `pg_ctl`, which re-parents it away from this process +(against (i)), shares one server between processes by reference count and +stops it from `atexit` (which a SIGTERM never runs, against (iv)), and may put +the socket under the user's runtime directory instead of the state directory +(against 13.1). The binaries themselves link only libc and libz, so they run on +any manylinux2014 host, which a wheel that links the system's ICU does not. + +THE LIFECYCLE, IN FULL: + + * `initdb` once per data directory: the MIGRATION identity + (`config.BUNDLE_OWNER_ROLE`) is the bootstrap superuser, local connections + are `trust` and host connections are `reject`, UTF-8 in the `C` locale. + Trust is safe BECAUSE of the socket: its directory is 0700, owned by the + user running the install, and the server opens no TCP port at all, so + reaching the socket is the credential and nothing else can. + * `postgres` started as a DIRECT CHILD of this process (`subprocess.Popen`, + never `pg_ctl`), with `listen_addresses` empty and the socket directory + given. On Linux it also carries `PR_SET_PDEATHSIG`, so an entry point + killed without any chance to clean up (SIGKILL) still takes its server + with it; an ordinary stop is `stop()`, a fast shutdown. + * a bootstrap, idempotent: the database, the SERVED role with no password + and the grants `deploy/compose/init-runtime-role.sh` makes for the compose + stack's role of the same name, narrowed to what this install's owner + creates. + * the ordered-SQL migrations, through `migrations.MigrationRunner`, as the + owner, with the served role and the database declared, so the run narrows + the ledger and verifies the served role's access exactly as a hosted + `runtime migrate` does. + +IMPORT WEIGHT: standard library and `opendox.runtime.{config,migrations}` at +module level. `psycopg` and `opendox.runtime.db` are imported at CALL time, +inside the functions that connect, like every other module the +`tests_runtime/test_runtime_surface.py` contract names. +""" + +from __future__ import annotations + +import ctypes +import importlib.util +import os +import signal +import subprocess +import sys +import time +from pathlib import Path +from typing import Any + +from opendox.runtime import migrations +from opendox.runtime.config import ( + BUNDLE_DATABASE, + BUNDLE_OWNER_ROLE, + BUNDLE_PORT, + BUNDLE_SERVED_ROLE, + LOCAL_FLAG, + PREFIX, + DatabaseBundle, + RuntimeSettings, + database_bundle, +) + +#: The distribution the `local` extra installs for the server's binaries. +SERVER_DISTRIBUTION = "pgserver" + +#: How long a start may take before it is a failure: `initdb` on a slow disk, +#: plus the server's own recovery on a data directory an earlier run did not +#: stop cleanly. +START_TIMEOUT_SECONDS = 60.0 + +#: How long a fast shutdown may take before the server is told to stop NOW. +STOP_TIMEOUT_SECONDS = 30.0 + +#: `prctl(2)`'s option number for the parent-death signal (linux/prctl.h). +_PR_SET_PDEATHSIG = 1 + + +class BundleRefused(Exception): + """The bundled server could not be started, named. Carries no credential. + + One exception, because the caller does nothing different for any of them: + the local install does not start. + """ + + +def server_binaries() -> Path: + """The directory holding the bundled `initdb` and `postgres`, or a refusal. + + Found WITHOUT importing `pgserver`: its package initializer imports its + manager, which this module does not use and whose import creates a lock + object under the user's runtime directory as a side effect. + """ + spec = importlib.util.find_spec(SERVER_DISTRIBUTION) + locations = list(spec.submodule_search_locations or ()) if spec else [] + if not locations: + raise BundleRefused( + "the local install's PostgreSQL server is not installed: it " + "arrives with the `local` extra, `pip install \"opendox[local]\"` " + "(R1Q16 (iii)). A local install brings its own database and never " + "borrows one") + binaries = Path(locations[0]) / "pginstall" / "bin" + missing = [name for name in ("initdb", "postgres") + if not os.access(binaries / name, os.X_OK)] + if missing: + raise BundleRefused( + f"the `{SERVER_DISTRIBUTION}` package is installed but carries no " + f"executable {' or '.join(missing)} under {binaries}; reinstall " + "the `local` extra") + return binaries + + +def running_pid(bundle: DatabaseBundle) -> int | None: + """The pid of a live server on `bundle`'s data directory, or `None`. + + Read from the server's own `postmaster.pid` (its first line), and only + believed while that process exists: a file left by a server that did not + stop cleanly names a pid that is gone, or that the kernel has since given + to something else, and PostgreSQL itself treats such a file as stale. + """ + try: + first = (bundle.data_dir / "postmaster.pid").read_text( + encoding="utf-8").splitlines()[0] + pid = int(first.strip()) + except (OSError, IndexError, ValueError): + return None + try: + os.kill(pid, 0) + except ProcessLookupError: + return None + except PermissionError: + return pid # alive, and not ours to signal + # A REUSED PID IS NOT A SERVER: where the kernel says what runs there, it + # must be a postgres, or the file is stale whatever its first line says. + try: + command = Path(f"/proc/{pid}/cmdline").read_bytes() + except OSError: + return pid # no /proc to ask; believe the live pid + return pid if b"postgres" in command else None + + +def report(bundle: DatabaseBundle) -> dict[str, Any]: + """The `database_bundle` block `runtime status` prints (#1144 13.1).""" + return {"data_dir": str(bundle.data_dir), + "socket_dir": str(bundle.socket_dir), + "pid": running_pid(bundle)} + + +def _child_environment() -> dict[str, str]: + """This process's environment, less libpq's and PostgreSQL's own variables. + + `PGDATA`, `PGPORT`, `PGHOST`, `PGOPTIONS` and the rest would otherwise + steer `initdb` and the server somewhere other than the paths given on + their command lines; the command line is the configuration, stated once. + """ + return {name: value for name, value in os.environ.items() + if not name.startswith("PG")} + + +def _die_with_parent(): + """A `preexec_fn` that signals the server when its parent goes away (iv). + + `PR_SET_PDEATHSIG` with SIGINT, PostgreSQL's FAST shutdown, so an entry + point killed outright (SIGKILL, an OOM kill) still stops its server + instead of leaving it running with the socket held. Linux only, and `None` + elsewhere, where the ordinary `stop()` is the whole of (iv). + + `prctl` is RESOLVED HERE, in the parent, so the forked child only calls + it; and the child re-checks its parent afterwards, because a parent that + died between the fork and the `prctl` would never deliver the signal. + """ + if not sys.platform.startswith("linux"): + return None + try: + prctl = ctypes.CDLL(None, use_errno=True).prctl + except (OSError, AttributeError): # pragma: no cover - a libc without it + return None + parent = os.getpid() + + def _preexec() -> None: # pragma: no cover - runs in the child + prctl(_PR_SET_PDEATHSIG, int(signal.SIGINT)) + if os.getppid() != parent: + os._exit(1) + + return _preexec + + +class BundledServer: + """One local install's PostgreSQL server: started as this process's child. + + Built from the settings a LOCAL `load_settings` returned, so the DSNs it + serves are the ones every other verb of the same install derives. `start` + initializes, launches, bootstraps and migrates; `stop` shuts it down; both + are safe to call more than once, and it is a context manager. + """ + + def __init__(self, settings: RuntimeSettings) -> None: + self.settings = settings + self.bundle = database_bundle(settings.state_dir) + self.process: subprocess.Popen[bytes] | None = None + self.applied: list[str] = [] + + @property + def log_path(self) -> Path: + return self.bundle.data_dir.parent / "postgres.log" + + def report(self) -> dict[str, Any]: + live = self.process is not None and self.process.poll() is None + return {"data_dir": str(self.bundle.data_dir), + "socket_dir": str(self.bundle.socket_dir), + "pid": self.process.pid if live else None} + + # -- start ------------------------------------------------------------- + + def start(self) -> BundledServer: + if hasattr(os, "geteuid") and os.geteuid() == 0: + raise BundleRefused( + "the bundled PostgreSQL server refuses to run as root, and so " + "does this local install: run it as the ordinary user whose " + "documents it serves") + binaries = server_binaries() + already = running_pid(self.bundle) + if already is not None: + raise BundleRefused( + f"a server is already running on {self.bundle.data_dir} " + f"(pid {already}). One local install's database belongs to one " + "entry point at a time: stop the other `generate-and-open " + f"{LOCAL_FLAG}`, or give this one its own {PREFIX}STATE_DIR") + self._prepare_directories() + if not (self.bundle.data_dir / "PG_VERSION").is_file(): + self._initdb(binaries) + self._launch(binaries) + try: + self._wait_until_ready() + self._bootstrap() + self._migrate() + except (BundleRefused, migrations.MigrationError): + self.stop() + raise + except Exception as exc: # noqa: BLE001 - the driver's, named not quoted + # THE DRIVER'S TEXT IS NOT REPEATED, for the reason `runtime/cli.py` + # gives: it quotes the connection string it was handed. The class + # names what went wrong and the server's own log says the rest. + self.stop() + raise BundleRefused( + f"the bundled PostgreSQL server started but could not be " + f"prepared ({type(exc).__name__}); its log is " + f"{self.log_path}") from None + except BaseException: + self.stop() + raise + return self + + def _prepare_directories(self) -> None: + """The state, data and socket directories, each 0700 where this makes it. + + A directory that already exists is NOT re-moded, on the rule the + runtime's `init` keeps for an operator's own path — except the socket + directory, whose mode IS the access control of a `trust` server: it is + this install's own, under its own state directory, and it is narrowed + to 0700 whatever it was. + """ + for directory in (self.bundle.state_dir, self.bundle.data_dir.parent): + directory.mkdir(mode=0o700, parents=True, exist_ok=True) + self.bundle.socket_dir.mkdir(mode=0o700, parents=True, exist_ok=True) + os.chmod(self.bundle.socket_dir, 0o700) + + def _initdb(self, binaries: Path) -> None: + done = subprocess.run( + [str(binaries / "initdb"), "-D", str(self.bundle.data_dir), + "-U", BUNDLE_OWNER_ROLE, "--auth-local=trust", "--auth-host=reject", + "--encoding=UTF8", "--locale=C", "--no-instructions"], + env=_child_environment(), capture_output=True, text=True, + timeout=START_TIMEOUT_SECONDS) + if done.returncode != 0: + detail = (done.stderr or done.stdout).strip().splitlines()[-5:] + raise BundleRefused( + f"initdb could not initialize {self.bundle.data_dir} " + f"(exit {done.returncode}): {' | '.join(detail)}") + + def _launch(self, binaries: Path) -> None: + log = open(self.log_path, "ab") # noqa: SIM115 - the child keeps it + try: + self.process = subprocess.Popen( + [str(binaries / "postgres"), "-D", str(self.bundle.data_dir), + "-k", str(self.bundle.socket_dir), "-p", str(BUNDLE_PORT), + # NO TCP LISTENER (13.1): an empty `listen_addresses` opens + # no TCP socket at all, so no pre-existing service on any + # port can stand in for this server, and nothing off this + # machine can reach it. + "-c", "listen_addresses=", + "-c", "unix_socket_permissions=0700"], + stdin=subprocess.DEVNULL, stdout=log, stderr=subprocess.STDOUT, + env=_child_environment(), + # ITS OWN SESSION, so a terminal's Ctrl-C reaches this process + # and not the server behind its back: the stop is `stop()`'s, + # in order, after the document server has closed. + start_new_session=True, + preexec_fn=_die_with_parent()) + finally: + log.close() + + def _wait_until_ready(self) -> None: + import psycopg + + deadline = time.monotonic() + START_TIMEOUT_SECONDS + last = "no attempt" + while time.monotonic() < deadline: + if self.process is not None and self.process.poll() is not None: + raise BundleRefused( + f"the bundled PostgreSQL server exited during start " + f"(exit {self.process.returncode}); its log is " + f"{self.log_path}: {self._log_tail()}") + try: + with psycopg.connect(self._dsn(BUNDLE_OWNER_ROLE, "postgres"), + connect_timeout=2, autocommit=True) as conn: + conn.execute("select 1") + return + except psycopg.OperationalError as exc: + # THE CLASS NAME ONLY: a driver's message quotes the DSN it + # could not reach, and this package never repeats one + # (`runtime/cli.py`'s redaction contract). + last = type(exc).__name__ + time.sleep(0.1) + raise BundleRefused( + f"the bundled PostgreSQL server did not accept a connection within " + f"{START_TIMEOUT_SECONDS:.0f}s ({last}); its log is " + f"{self.log_path}") + + def _dsn(self, role: str, database: str) -> str: + """`DatabaseBundle.dsn`, aimed at a database other than the served one.""" + return self.bundle.dsn(role).replace( + f"@/{BUNDLE_DATABASE}?", f"@/{database}?", 1) + + def _bootstrap(self) -> None: + """The database, the served role and its grants — idempotent. + + The grants are the compose stack's (`init-runtime-role.sh`), narrowed + the same way: CONNECT on the database, USAGE on `public`, and DML on + every table the OWNER creates from here on, by default privileges — + which on this install's own fresh database is the six coordination + tables and the ledger, and nothing else. The migration then narrows + the served role's rights on the ledger to SELECT. + """ + import psycopg + from psycopg import sql + + with psycopg.connect(self._dsn(BUNDLE_OWNER_ROLE, "postgres"), + autocommit=True) as conn: + if conn.execute("select 1 from pg_database where datname = %s", + (BUNDLE_DATABASE,)).fetchone() is None: + conn.execute(sql.SQL("create database {} owner {}").format( + sql.Identifier(BUNDLE_DATABASE), + sql.Identifier(BUNDLE_OWNER_ROLE))) + if conn.execute("select 1 from pg_roles where rolname = %s", + (BUNDLE_SERVED_ROLE,)).fetchone() is None: + conn.execute(sql.SQL("create role {} login").format( + sql.Identifier(BUNDLE_SERVED_ROLE))) + with psycopg.connect(self.bundle.migration_dsn, autocommit=True) as conn: + served, owner = (sql.Identifier(BUNDLE_SERVED_ROLE), + sql.Identifier(BUNDLE_OWNER_ROLE)) + conn.execute(sql.SQL("grant connect on database {} to {}").format( + sql.Identifier(BUNDLE_DATABASE), served)) + conn.execute(sql.SQL("grant usage on schema public to {}").format( + served)) + conn.execute(sql.SQL( + "alter default privileges for role {} in schema public grant " + "select, insert, update, delete on tables to {}").format( + owner, served)) + + def _migrate(self) -> None: + """Starting AND MIGRATING is all release 1 asks of the store (R1Q16 (ii)).""" + from opendox.runtime.db import Database + + database = Database(self.bundle.migration_dsn, + application_name="opendox-local-migrate") + with database: + self.applied = migrations.MigrationRunner( + database, migrations_dir=self.settings.migrations_dir, + runtime_role=self.settings.runtime_pg_role, + served_schema=self.settings.served_schema, + served_database=self.settings.served_database).apply() + + def _log_tail(self) -> str: + try: + lines = self.log_path.read_text(encoding="utf-8", + errors="replace").splitlines() + except OSError: + return "(no log)" + return " | ".join(lines[-5:]) + + # -- stop -------------------------------------------------------------- + + def stop(self) -> None: + """A FAST shutdown (SIGINT), then an immediate one, then a kill. + + FAST, and not PostgreSQL's default "smart" shutdown, because the + document server that is its only client has already closed: waiting + for sessions to end would wait for nothing, or for a leaked + connection. Safe to call twice, and on a server that never started. + """ + process, self.process = self.process, None + if process is None or process.poll() is not None: + return + for sig, wait in ((signal.SIGINT, STOP_TIMEOUT_SECONDS), + (signal.SIGQUIT, 5.0), (signal.SIGKILL, 5.0)): + try: + process.send_signal(sig) + except ProcessLookupError: + return + try: + process.wait(timeout=wait) + return + except subprocess.TimeoutExpired: + continue + + def __enter__(self) -> BundledServer: + return self.start() + + def __exit__(self, *exc: object) -> None: + self.stop() + + +__all__ = ["BUNDLE_PORT", "BundleRefused", "BundledServer", "report", + "running_pid", "server_binaries"] diff --git a/src/opendox/runtime/cli.py b/src/opendox/runtime/cli.py index af0ac390..e1c7a041 100644 --- a/src/opendox/runtime/cli.py +++ b/src/opendox/runtime/cli.py @@ -348,6 +348,10 @@ def _redacted_settings(settings: RuntimeSettings) -> dict[str, Any]: # needs to know, because it decides whether the broker lines below # mean anything at all. "OPENDOX_INSTALL_MODE": settings.install_mode, + # WHERE A LOCAL INSTALL'S BUNDLED SERVER LIVES (plan 034 T072): a path + # and never a credential. Reported for a hosted install too, which + # never reads it, so the report covers the whole declared list. + "OPENDOX_STATE_DIR": str(settings.state_dir), # THE BROKER URLS ARE REDACTED HERE TOO. `load_settings` refuses # userinfo in the issuer and in an explicit JWKS URL — but this report # prints a DERIVED value, and a settings object can also be built by @@ -680,6 +684,20 @@ def cmd_status(args: argparse.Namespace) -> int: return _emit({"verb": "status", "refusal": "configuration", "message": _safe_message(exc)}, ok=False) report["settings"] = _redacted_settings(settings) + # THE BUNDLED SERVER THIS INSTALL OWNS (plan 034 T072; #1144 13.1): where + # its data directory and socket are, and the pid of the server running on + # them, read from the server's own `postmaster.pid`. `null` for a hosted + # install, which brings no server. Reported, never started: `status` + # changes nothing, and the process that owns the server is the document + # server that started it (R1Q16 (i)). + if settings.install_mode == INSTALL_MODE_LOCAL: + from opendox.runtime import bundle as bundle_mod + from opendox.runtime.config import database_bundle + + report["database_bundle"] = bundle_mod.report( + database_bundle(settings.state_dir)) + else: + report["database_bundle"] = None try: report["canonical_sha256"] = migrations.verify_canonical_digest( diff --git a/src/opendox/runtime/config.py b/src/opendox/runtime/config.py index 6edd5c88..fa8bf8db 100644 --- a/src/opendox/runtime/config.py +++ b/src/opendox/runtime/config.py @@ -26,14 +26,16 @@ from __future__ import annotations +import importlib.metadata import ipaddress import os import re import shlex +import sys import urllib.parse from collections.abc import Mapping from dataclasses import dataclass -from pathlib import Path +from pathlib import Path, PurePath #: The environment prefix. One string, so a rename is one edit. PREFIX = "OPENDOX_" @@ -97,6 +99,18 @@ class Setting: "neither the install is hosted, so a hosted install with no issuer " "refuses rather than falling into local mode (13.4, 13.5)", ), + # WHERE A LOCAL INSTALL KEEPS ITS OWN STATE (plan 034 T072; #1144 13.1): + # the bundled PostgreSQL server's data directory and its Unix socket. No + # default string, because the default is COMPUTED, per user — see + # `state_dir`. A hosted install never reads it. + Setting( + PREFIX + "STATE_DIR", None, False, False, + "the directory a LOCAL install owns: the bundled PostgreSQL server's " + "data directory and its Unix socket live under it, and the server " + "listens on that socket and on no TCP port (13.1). Unset, it is " + "`$XDG_STATE_HOME/opendox`, else `~/.local/state/opendox`. A hosted " + "install never reads it", + ), Setting( PREFIX + "OIDC_ISSUER", None, True, False, "the Keycloak broker's issuer, pinned: a token from any other issuer " @@ -172,7 +186,11 @@ class Setting: ), Setting( PREFIX + "MIGRATIONS_DIR", "migrations", False, False, - "the ordered-SQL directory, repository-root-relative", + "the ordered-SQL directory, repository-root-relative. Unset, it is " + "`migrations` where the working directory holds one (a checkout, or " + "the image's /app), and otherwise the copy the installed package " + "ships (plan 034 T072), so an install run from anywhere else still " + "has the migrations it applies", ), Setting( PREFIX + "PROJECT_REPOSITORY_ROOT", "var/projects", False, False, @@ -214,6 +232,7 @@ class RuntimeSettings: database_url: str migration_database_url: str | None install_mode: str + state_dir: Path oidc_issuer: str oidc_audience: str oidc_jwks_url: str | None @@ -236,6 +255,7 @@ def __repr__(self) -> str: "migration_database_url=" f"{'' if self.migration_database_url else 'None'}, " f"install_mode={self.install_mode!r}, " + f"state_dir={str(self.state_dir)!r}, " # REDACTED TOO, and not because `load_settings` allows userinfo # here — it refuses it. A `RuntimeSettings` built by hand, in a # test or by a future caller, does not go through that door, and @@ -1552,14 +1572,190 @@ def _refuse_two_dsns_that_select_different_schemas( #: is REFUSED, by name (a holder reading on openxFactory#656, plan 034 T070, #: the same fail-closed reading as the disagreeing flag and setting): an issuer #: next to `local` says a broker was meant, and honouring `local` over it would -#: silently drop the authentication the operator configured. T072 adds the two -#: DSNs, which the local install supplies itself (13.1). +#: silently drop the authentication the operator configured. +#: +#: AND THE TWO DSNs (plan 034 T072; the same holder reading): a local install +#: SUPPLIES BOTH ITSELF, from the server it bundles (#1144 13.1, "so no +#: pre-existing service can stand in for it"), so an operator's DSN beside +#: `local` is either about to be silently overridden or is another server +#: trying to stand in for the bundled one. Neither is accepted. HOSTED_ONLY_SETTINGS: tuple[str, ...] = ( PREFIX + "OIDC_ISSUER", PREFIX + "OIDC_AUDIENCE", PREFIX + "OIDC_JWKS_URL", + PREFIX + "DATABASE_URL", + PREFIX + "MIGRATION_DATABASE_URL", ) +#: THE BUNDLED SERVER'S IDENTITY (plan 034 T072; #1144 13.1, as T007 batch H's +#: addendum reads). The data directory and the socket directory live under the +#: install's own `OPENDOX_STATE_DIR`, at these paths — SHORT ONES, because a +#: Unix socket's whole path is bounded by the kernel (`sun_path`) and the +#: socket file is `/.s.PGSQL.`. +BUNDLE_DATA_DIR = PurePath("postgres", "data") +BUNDLE_SOCKET_DIR = PurePath("postgres", "run") +#: The port NUMBER, which names the socket file and opens NO TCP port: the +#: server is started with `listen_addresses` empty (13.1: "on NO TCP port"). +BUNDLE_PORT = 5432 +#: The two identities 13.3 keeps apart, and the one database. The MIGRATION +#: identity owns the database and every table it creates; the SERVED identity +#: is the least-privileged role the API reads and writes as, granted what +#: `deploy/compose/init-runtime-role.sh` grants the compose stack's role of the +#: same name. Two DSNs, two users, never one pasted twice (13.3). +BUNDLE_OWNER_ROLE = "opendox" +BUNDLE_SERVED_ROLE = "opendox_runtime" +BUNDLE_DATABASE = "opendox" + +#: The longest socket path the kernel takes, in bytes: `sizeof(sun_path)` less +#: its terminating NUL — 108 on Linux, 104 on macOS and the BSDs. PostgreSQL +#: refuses a longer one at startup; this refuses it at configuration, naming +#: the setting that made it long. +UNIX_SOCKET_PATH_MAX = 107 if sys.platform.startswith("linux") else 103 + + +@dataclass(frozen=True) +class DatabaseBundle: + """Where a LOCAL install's bundled PostgreSQL server lives, and its DSNs. + + Pure path and string arithmetic over `OPENDOX_STATE_DIR`, so `load_settings` + can name both DSNs without starting anything, and `runtime status` in a + second process derives the SAME ones and finds the same server. + `opendox.runtime.bundle` is what starts and stops it. + """ + + state_dir: Path + + @property + def data_dir(self) -> Path: + return self.state_dir / BUNDLE_DATA_DIR + + @property + def socket_dir(self) -> Path: + return self.state_dir / BUNDLE_SOCKET_DIR + + @property + def socket_path(self) -> Path: + return self.socket_dir / f".s.PGSQL.{BUNDLE_PORT}" + + def dsn(self, role: str) -> str: + """A DSN for `role` over the bundle's Unix socket, and never TCP. + + `host` is the socket DIRECTORY (libpq's rule for a value that starts + with `/`), percent-encoded so a state directory holding a space or a + `&` is still one value; `port` is spelled so a `PGPORT` in the + environment cannot send libpq to a different socket file. No password: + the socket directory is 0700 and the server's own `pg_hba.conf` + trusts local connections only, so reaching the socket IS the + credential, and a host connection is rejected outright. + """ + host = urllib.parse.quote(str(self.socket_dir), safe="/") + return (f"postgresql://{role}@/{BUNDLE_DATABASE}" + f"?host={host}&port={BUNDLE_PORT}") + + @property + def served_dsn(self) -> str: + return self.dsn(BUNDLE_SERVED_ROLE) + + @property + def migration_dsn(self) -> str: + return self.dsn(BUNDLE_OWNER_ROLE) + + +def state_dir(env: Mapping[str, str] | None = None) -> Path: + """The install's own state directory: `OPENDOX_STATE_DIR`, or the per-user one. + + ABSOLUTE, or refused: the document server that starts the bundled server + and a `runtime status` run from another directory must derive the same + socket, and a relative value would give each its own. Unset, it is + `$XDG_STATE_HOME/opendox` where that is absolute (the XDG rule ignores a + relative one), and otherwise `~/.local/state/opendox`. + """ + env = os.environ if env is None else env + setting = PREFIX + "STATE_DIR" + raw = env.get(setting, "").strip() + if raw: + path = Path(raw).expanduser() + if not path.is_absolute(): + raise ConfigurationError( + f"{setting} is {raw!r}, which is not an absolute path. The " + "document server that starts the bundled PostgreSQL server " + "and a `runtime status` run from another directory must find " + "the same socket, so the state directory is named absolutely") + return path + xdg = env.get("XDG_STATE_HOME", "").strip() + base = Path(xdg) if xdg and Path(xdg).is_absolute() else ( + Path.home() / ".local" / "state") + return base / "opendox" + + +def database_bundle(state: Path) -> DatabaseBundle: + """The bundle under `state`, refusing a socket path the kernel cannot bind.""" + bundle = DatabaseBundle(state_dir=state) + length = len(os.fsencode(str(bundle.socket_path))) + if length > UNIX_SOCKET_PATH_MAX: + raise ConfigurationError( + f"{PREFIX}STATE_DIR is too long for the bundled server's Unix " + f"socket: {bundle.socket_path} is {length} bytes and this kernel " + f"takes at most {UNIX_SOCKET_PATH_MAX}. The socket must live under " + "the install's own state directory (13.1), so choose a shorter " + f"{PREFIX}STATE_DIR") + return bundle + + +#: WHERE AN INSTALLED WHEEL KEEPS ITS MIGRATIONS (plan 034 T072). The +#: repository's `migrations/` stays where it is — the image copies it to +#: `/app/migrations` and runs from `/app` — and `pyproject.toml` maps the same +#: files into the wheel's data directory under this path, so an install run +#: outside any checkout still has the migrations it applies. The canonical +#: digest gate (`migrations.verify_canonical_digest`) is what proves any copy +#: found this way is the pinned one. +PACKAGED_MIGRATIONS = PurePath("share", "opendox", "migrations") + + +def packaged_migrations_dir() -> Path | None: + """The migrations this INSTALLATION carries, or `None` where it carries none. + + Two places, in order. The installed distribution's own data files — a + wheel install, whose `RECORD` lists them wherever the install scheme put + them. Then the source tree this module was imported from — an editable + install, which installs no data files, or `src/` on the path — whose + `migrations/` sits beside `src/`. + """ + try: + files = importlib.metadata.distribution("opendox").files or () + except importlib.metadata.PackageNotFoundError: + files = () + for entry in files: + parts = PurePath(entry).parts + if (entry.name.endswith(".sql") + and tuple(parts[-4:-1]) == PACKAGED_MIGRATIONS.parts): + return Path(entry.locate()).resolve().parent + source = Path(__file__).resolve().parents[3] + if (source / "pyproject.toml").is_file() and (source / "migrations").is_dir(): + return source / "migrations" + return None + + +def migrations_dir(env: Mapping[str, str] | None = None) -> Path: + """`OPENDOX_MIGRATIONS_DIR`, or where this install's migrations are. + + Set, it is used as given, as it always was. Unset, it is `migrations` + wherever the working directory holds one — a checkout, or the image's + `/app` — which is today's default, unchanged; and OTHERWISE the copy the + installed package carries (`packaged_migrations_dir`), so `pip install + "opendox[local]"` run from a user's home directory migrates its bundled + server instead of refusing for a missing directory. Where neither exists + it is still `migrations`, and the canonical gate refuses it by name. + """ + env = os.environ if env is None else env + raw = env.get(PREFIX + "MIGRATIONS_DIR", "").strip() + if raw: + return Path(raw) + here = Path("migrations") + if here.is_dir(): + return here + return packaged_migrations_dir() or here + def install_mode(env: Mapping[str, str] | None = None, *, local_flag: bool = False) -> str: @@ -1686,6 +1882,19 @@ def require_the_hosted_issuer(env: Mapping[str, str] | None = None) -> None: f"{PREFIX}INSTALL_MODE=local") +def _hosted_state_dir(env: Mapping[str, str]) -> Path: + """A HOSTED install's `state_dir`: reported, never read, never refused. + + A hosted install has no bundled server, so a value it will never use is + not a reason for it to refuse to start (13.6: otherwise unchanged). It is + still reported, so `status` describes the whole declared setting list. + """ + try: + return state_dir(env) + except ConfigurationError: + return Path(env.get(PREFIX + "STATE_DIR", "").strip()) + + def load_settings(env: Mapping[str, str] | None = None, *, local_flag: bool = False) -> RuntimeSettings: """Resolve :class:`RuntimeSettings` from `env` (default `os.environ`). @@ -1728,8 +1937,20 @@ def load_settings(env: Mapping[str, str] | None = None, *, if local: refuse_what_a_local_install_cannot_be(env) algorithms = _algorithms(env) - served = _require(env, _by_name(PREFIX + "DATABASE_URL")) - migration = _optional(env, _by_name(PREFIX + "MIGRATION_DATABASE_URL")) + # A LOCAL INSTALL SUPPLIES BOTH DSNs ITSELF (plan 034 T072; #1144 13.1), + # from the server it bundles under its own state directory, and an + # operator's DSN beside it was refused above. The two it supplies are two + # users over one socket, so T071's three checks below pass them for the + # reason they exist: one dialect, one database, and never one credential + # in both settings. + state = state_dir(env) if local else _hosted_state_dir(env) + if local: + bundle = database_bundle(state) + served: str = bundle.served_dsn + migration: str | None = bundle.migration_dsn + else: + served = _require(env, _by_name(PREFIX + "DATABASE_URL")) + migration = _optional(env, _by_name(PREFIX + "MIGRATION_DATABASE_URL")) # THE DIALECT FIRST: a scheme this module cannot parse as PostgreSQL is not # yet a DSN worth comparing at all. A no-op on an ABSENT migration DSN — # see `_refuse_non_postgresql_dsn`. @@ -1756,6 +1977,7 @@ def load_settings(env: Mapping[str, str] | None = None, *, database_url=served, migration_database_url=migration, install_mode=mode, + state_dir=state, oidc_issuer="" if local else _broker_url( env, _by_name(PREFIX + "OIDC_ISSUER"), required=True, is_a_base_url=True) or "", @@ -1768,11 +1990,15 @@ def load_settings(env: Mapping[str, str] | None = None, *, oidc_leeway_seconds=_positive_int(env, _by_name(PREFIX + "OIDC_LEEWAY_SECONDS")), bind_host=bind_host, bind_port=_positive_int(env, _by_name(PREFIX + "BIND_PORT")), - runtime_pg_role=_role_name(env), + # THE BUNDLE'S OWN NAMES where the operator declares none: the served + # role the migration narrows and verifies, and the database it may + # touch. An operator's declaration still wins, and a wrong one is + # refused by the migration run's own guards, as on a hosted install. + runtime_pg_role=_role_name(env) or (BUNDLE_SERVED_ROLE if local else None), served_schema=_served_schema(env), - served_database=_served_database(env), + served_database=_served_database(env) or (BUNDLE_DATABASE if local else None), publish_openapi=_boolean(env, _by_name(PREFIX + "PUBLISH_OPENAPI")), - migrations_dir=Path(_optional(env, _by_name(PREFIX + "MIGRATIONS_DIR")) or "migrations"), + migrations_dir=migrations_dir(env), project_repository_root=Path( _optional(env, _by_name(PREFIX + "PROJECT_REPOSITORY_ROOT")) or "var/projects" ), @@ -1801,7 +2027,18 @@ def load_migration_settings(env: Mapping[str, str] | None = None) -> RuntimeSett not, because those are the served runtime and must have the real thing. """ env = os.environ if env is None else env - dsn = env.get(PREFIX + "MIGRATION_DATABASE_URL", "").strip() + mode = install_mode(env) + local = mode == INSTALL_MODE_LOCAL + if local: + # THE BUNDLE'S OWNER, over its socket (plan 034 T072): a local install + # supplies its migration DSN as it supplies the served one, and an + # operator's beside it is refused, exactly as `load_settings` refuses. + refuse_what_a_local_install_cannot_be(env) + state = state_dir(env) + dsn = database_bundle(state).migration_dsn + else: + state = _hosted_state_dir(env) + dsn = env.get(PREFIX + "MIGRATION_DATABASE_URL", "").strip() if not dsn: raise ConfigurationError( f"{PREFIX}MIGRATION_DATABASE_URL is required to apply migrations; " @@ -1822,7 +2059,8 @@ def load_migration_settings(env: Mapping[str, str] | None = None) -> RuntimeSett # a migration run is part of the same install and one reading of the # selector serves every verb. It changes nothing else a migration run # does; the broker fields below are sentinels in either shape. - install_mode=install_mode(env), + install_mode=mode, + state_dir=state, oidc_issuer=MIGRATION_SENTINEL_ISSUER, oidc_audience=MIGRATION_SENTINEL_AUDIENCE, oidc_jwks_url=None, @@ -1831,12 +2069,11 @@ def load_migration_settings(env: Mapping[str, str] | None = None) -> RuntimeSett oidc_leeway_seconds=1, bind_host="127.0.0.1", bind_port=1, - runtime_pg_role=_role_name(env), + runtime_pg_role=_role_name(env) or (BUNDLE_SERVED_ROLE if local else None), served_schema=_served_schema(env), - served_database=_served_database(env), + served_database=_served_database(env) or (BUNDLE_DATABASE if local else None), publish_openapi=False, - migrations_dir=Path( - _optional(env, _by_name(PREFIX + "MIGRATIONS_DIR")) or "migrations"), + migrations_dir=migrations_dir(env), project_repository_root=Path( _optional(env, _by_name(PREFIX + "PROJECT_REPOSITORY_ROOT")) or "var/projects"), diff --git a/tests/test_doxbench_entrypoint.py b/tests/test_doxbench_entrypoint.py index 2488fdf0..47c3eae2 100644 --- a/tests/test_doxbench_entrypoint.py +++ b/tests/test_doxbench_entrypoint.py @@ -173,6 +173,31 @@ def _capture(*args, **kwargs): # from the shell would be refused beside the local mode, by design. for name in runtime_config.SETTING_NAMES: monkeypatch.delenv(name, raising=False) + # AND THE LOCAL INSTALL'S DATABASE IS STOOD IN, with a tripwire of its own + # (plan 034 T072). A local `generate-and-open` starts its bundled + # PostgreSQL server before it serves, and these cases are about the model + # port the entrypoint declares, which reads nothing from the store (R1Q16 + # (ii)). `tests_runtime/test_bundled_postgres.py` starts the real one, on + # this same entry point, and owns every assertion about it. + bundles = [] + + class _StandInBundle: + applied: list = [] + + def __init__(self, settings): + assert settings.install_mode == runtime_config.INSTALL_MODE_LOCAL + bundles.append(self) + + def start(self): + return self + + def stop(self): + pass + + def report(self): + return {"data_dir": None, "socket_dir": "(stood in)", "pid": None} + + monkeypatch.setattr(cli_mod.bundle_mod, "BundledServer", _StandInBundle) args = cli_mod.build_parser().parse_args([ "generate-and-open", runtime_config.LOCAL_FLAG, @@ -186,6 +211,7 @@ def _capture(*args, **kwargs): ]) rc = cli_mod.cmd_generate_and_open(args, opener=lambda url: None) assert rc == 0, "the entrypoint did not complete" + assert len(bundles) == 1, "a LOCAL entrypoint run must own one database" assert built, "the entrypoint never reached build_server" yield _handler_class(built[-1]), spawned, session_root diff --git a/tests/test_install_mode_entrypoint.py b/tests/test_install_mode_entrypoint.py index 2a121ef4..86e3e58d 100644 --- a/tests/test_install_mode_entrypoint.py +++ b/tests/test_install_mode_entrypoint.py @@ -164,31 +164,33 @@ def _args(*extra: str): def test_the_local_shape_needs_no_broker_and_no_setting_at_all() -> None: - """13.4: `local` needs no broker. Resolved with an EMPTY environment.""" - assert cli_mod._resolve_install_shape( - _args(runtime_config.LOCAL_FLAG), env={}) == \ - runtime_config.INSTALL_MODE_LOCAL - assert cli_mod._resolve_install_shape( - _args(), env={MODE: "local"}) == runtime_config.INSTALL_MODE_LOCAL + """13.4: `local` needs no broker. Resolved with an EMPTY environment, and + (plan 034 T072) with both DSNs supplied by the install itself.""" + for args, env in ((_args(runtime_config.LOCAL_FLAG), {}), + (_args(), {MODE: "local"})): + settings = cli_mod._resolve_install_shape(args, env=env) + assert settings.install_mode == runtime_config.INSTALL_MODE_LOCAL + assert settings.oidc_issuer == "" + assert settings.database_url != settings.migration_database_url @pytest.mark.parametrize("host", sorted(serve_mod.LOOPBACK_HOSTS)) def test_the_local_shape_accepts_each_loopback_host(host: str) -> None: assert cli_mod._resolve_install_shape( - _args(runtime_config.LOCAL_FLAG, "--host", host), env={}) == \ - runtime_config.INSTALL_MODE_LOCAL + _args(runtime_config.LOCAL_FLAG, "--host", host), env={} + ).install_mode == runtime_config.INSTALL_MODE_LOCAL def test_a_complete_hosted_configuration_resolves_hosted_unchanged() -> None: """13.6: a hosted install with its broker configured serves as before.""" env = {**HOSTED_WITHOUT_ISSUER, PREFIX + "OIDC_ISSUER": "https://issuer.example.invalid/realms/x"} - assert cli_mod._resolve_install_shape(_args(), env=env) == \ + assert cli_mod._resolve_install_shape(_args(), env=env).install_mode == \ runtime_config.INSTALL_MODE_HOSTED # a hosted document server may still bind beyond loopback, as it always # could: the loopback rule is the LOCAL mode's assert cli_mod._resolve_install_shape( - _args("--host", "0.0.0.0"), env=env) == \ + _args("--host", "0.0.0.0"), env=env).install_mode == \ runtime_config.INSTALL_MODE_HOSTED diff --git a/tests_runtime/local_entrypoint_driver.py b/tests_runtime/local_entrypoint_driver.py new file mode 100644 index 00000000..43e6d63c --- /dev/null +++ b/tests_runtime/local_entrypoint_driver.py @@ -0,0 +1,77 @@ +"""Run `opendox generate-and-open` for real, with ONLY its generation stood in. + +`tests_runtime/test_bundled_postgres.py` launches this file as a child process +(`python tests_runtime/local_entrypoint_driver.py generate-and-open --local …`) +so that F13.1's local probe can be run the way F13.1 runs it: in the +BACKGROUND, reached over HTTP, asked about by a SECOND process +(`runtime status`), and then stopped with a signal (plan 034 T072). + +WHAT IS STOOD IN, AND WHY IT IS ONLY THIS. At this stack's base, the generate +verbs still reach openXdox for three names (`corpus_root_refusal`, +`generate_snapshot`, `snapshot.write_snapshot`) and `serve` for two +(`_checkout_real`, `registry_mod`'s binding constants): phase 2's T055 and T056 +(openDox-code#59 and its successor) give openDox its own, and a lone checkout +cannot ask for them before then. They are replaced here exactly as +`tests/test_doxbench_entrypoint.py` replaces them, and NOTHING ELSE is: +`cli.main`, the install shape, the bundled server, `build_server` and the serve +loop all run as a user's `opendox generate-and-open --local` runs them. Once +T055 and T056 have landed on this branch's base, this driver's stand-ins go and +the probe runs the real entry point on `tests/fixtures/plain-documents`. +""" + +from __future__ import annotations + +import json +import sys +import types +from pathlib import Path + + +class _StandInSource: + """`build_server`'s snapshot source, as T011's `standalone` fixture has it.""" + + refresh_binding = None + baked_repository = None + + class registry: + active = None + + def bootstrap(self): + pass + + +def _generate_snapshot(repo_root, repository, *, source_revision=None, + **_ignored): + return {"repository": repository, + "generation": {"source_revision": source_revision}, + "documents": [{"id": "stand-in.md"}]} + + +def _write_snapshot(snapshot, output, boundary): + Path(output).write_text(json.dumps(snapshot), encoding="utf-8") + return output + + +def main(argv: list[str]) -> int: + from opendox import cli as cli_mod + from opendox import serve as serve_mod + + cli_mod.corpus_root_refusal = lambda root, shape=None: None + cli_mod.generate_snapshot = _generate_snapshot + cli_mod.snapshot_mod = types.SimpleNamespace(write_snapshot=_write_snapshot) + serve_mod._checkout_real = lambda root: False + serve_mod.registry_mod = types.SimpleNamespace( + BINDING_REGENERATE="regenerate", BINDING_REFETCH="refetch") + real_build_server = serve_mod.build_server + + def _build_server(*args, **kwargs): + kwargs.setdefault("snapshot_source", _StandInSource()) + return real_build_server(*args, **kwargs) + + serve_mod.build_server = _build_server + return cli_mod.main(argv) + + +if __name__ == "__main__": + sys.stdout.reconfigure(line_buffering=True) + sys.exit(main(sys.argv[1:])) diff --git a/tests_runtime/test_bundled_postgres.py b/tests_runtime/test_bundled_postgres.py new file mode 100644 index 00000000..895bcfb9 --- /dev/null +++ b/tests_runtime/test_bundled_postgres.py @@ -0,0 +1,415 @@ +"""The LOCAL install's bundled PostgreSQL server (plan 034 T072; #1144 13.1, as +T007 batch H's addendum reads; RULED R1Q16 (i)-(iv), `5850003126`). + +T072's falsifier is F13.1's TCP-listener block, which reads the kernel's +socket table at run time, and its `runtime status` block. Both are run here +against a server the REAL entry point started: `generate-and-open --local`, +launched in the background as F13.1 launches it, reached over HTTP, asked +about by a second process, and stopped with a signal. Where F13.1 reads the +server's pid from `caps.json`, these cases read the same pid from +`runtime status`'s `database_bundle`. `/capabilities`' `install` block is +T073's, and nothing here pretends it exists. + +R1Q16, each part asserted: + (i) the server is a CHILD of the entry point's process (its `PPid`); + (ii) it is started AND migrated, the ledger holding every migration; + (iii) the `local` extra carries it, and the `test` extra joins the extra; + (iv) it stops with the entry point: on SIGTERM, which the serve loop reads + as Ctrl-C, and — the backstop — on SIGKILL, through the parent-death + signal. + +And the migrations gap the holder assigned to T072: a WHEEL install, run from +a directory that is not a checkout, migrates its bundled server from the copy +the wheel carries. + +NOT SKIPPED IN CI. These cases need the `local` extra's server, which the +`test` extra installs; under `CI` its absence is a FAILURE, as a missing +`postgres:16` service is for the DB-backed suites (`conftest._skip_or_fail`'s +rule, restated below), because `validate.yml` pins the skip count exactly. +""" + +from __future__ import annotations + +import json +import os +import re +import shutil +import signal +import subprocess +import sys +import sysconfig +import tempfile +import time +import tomllib +import urllib.request +from pathlib import Path + +import pytest + +from opendox.runtime import bundle as bundle_mod +from opendox.runtime import config +from opendox.runtime.config import PREFIX + +ROOT = Path(__file__).resolve().parents[1] +SRC = ROOT / "src" +DRIVER = Path(__file__).resolve().parent / "local_entrypoint_driver.py" +MODE = PREFIX + "INSTALL_MODE" +STATE = PREFIX + "STATE_DIR" + + +def _in_ci() -> bool: + return os.environ.get("CI", "").strip().lower() in {"1", "true", "yes", "on"} + + +@pytest.fixture(scope="module", autouse=True) +def _the_server_is_installed() -> None: + """The `local` extra's binaries, or this module's refusal to pass silently.""" + try: + bundle_mod.server_binaries() + except bundle_mod.BundleRefused as exc: + if _in_ci(): + pytest.fail(f"CI is set, so the bundled-server suite must RUN: {exc}", + pytrace=False) + pytest.skip(str(exc)) + if hasattr(os, "geteuid") and os.geteuid() == 0: # pragma: no cover + pytest.fail("the bundled server refuses root; run the suite as a user") + + +@pytest.fixture() +def state_dir(): + """A state directory nothing else has touched, with a SHORT path. + + Short because the socket's whole path is bounded by the kernel, and + pytest's own `tmp_path` grows with the test's name. Removed afterwards, + once any server on it has been checked stopped. + """ + base = "/tmp" if os.path.isdir("/tmp") else None + path = Path(tempfile.mkdtemp(prefix="odx-", dir=base)) + yield path + pid = bundle_mod.running_pid(config.DatabaseBundle(path)) + if pid is not None: # pragma: no cover + os.kill(pid, signal.SIGKILL) + shutil.rmtree(path, ignore_errors=True) + + +def _clean_env(**extra: str) -> dict[str, str]: + env = {name: value for name, value in os.environ.items() + if name not in config.SETTING_NAMES and not name.startswith("PG")} + env["PYTHONPATH"] = os.pathsep.join( + [str(SRC), env.get("PYTHONPATH", "")]).rstrip(os.pathsep) + env.update(extra) + return env + + +def _parent_of(pid: int) -> int: + for line in Path(f"/proc/{pid}/status").read_text().splitlines(): + if line.startswith("PPid:"): + return int(line.split()[1]) + raise AssertionError(f"no PPid for {pid}") # pragma: no cover + + +def _tcp_listeners(pid: int) -> list[tuple[str, str]]: + """F13.1's TCP-listener block, verbatim in substance: every socket the + process holds, looked up in the KERNEL's TCP tables, never self-report.""" + inodes = set() + for fd in os.listdir(f"/proc/{pid}/fd"): + try: + m = re.match(r"socket:\[(\d+)\]", os.readlink(f"/proc/{pid}/fd/{fd}")) + except OSError: + continue + if m: + inodes.add(m.group(1)) + return [(tbl, row.split()[1]) for tbl in ("/proc/net/tcp", "/proc/net/tcp6") + for row in open(tbl).read().splitlines()[1:] + if row.split()[3] == "0A" and row.split()[9] in inodes] + + +def _wait_gone(pid: int, seconds: float = 30.0) -> bool: + deadline = time.monotonic() + seconds + while time.monotonic() < deadline: + try: + os.kill(pid, 0) + except ProcessLookupError: + return True + # a zombie is gone as a server: reaped by init once its parent is + try: + if "State:\tZ" in Path(f"/proc/{pid}/status").read_text(): + return True + except OSError: + return True + time.sleep(0.1) + return False + + +def _launch(corpus: Path, state: Path, run_dir: Path) -> tuple[subprocess.Popen, str]: + """`generate-and-open --local` in the BACKGROUND, and the URL it serves.""" + child = subprocess.Popen( + [sys.executable, str(DRIVER), "generate-and-open", config.LOCAL_FLAG, + "--repo-root", str(corpus), "--repository", "fixture", + "--run-dir", str(run_dir), "--no-open", "--no-validate", + "--port", "0"], + env=_clean_env(**{STATE: str(state)}), cwd=ROOT, + stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) + deadline = time.monotonic() + 90 + url = None + while time.monotonic() < deadline and url is None: + line = child.stdout.readline() + if not line: + break + if line.startswith("http://"): + url = line.strip() + if url is None: + child.kill() + out, err = child.communicate(timeout=30) + raise AssertionError(f"the entry point never served: {err[-2000:]}") + return child, url + + +def _status(state: Path) -> tuple[int, dict]: + """`OPENDOX_INSTALL_MODE=local opendox-runtime runtime status`, a SECOND process.""" + done = subprocess.run( + [sys.executable, "-m", "opendox.runtime.cli", "runtime", "status", + "--probe-timeout", "10"], + env=_clean_env(**{MODE: "local", STATE: str(state)}), cwd=ROOT, + capture_output=True, text=True, timeout=60) + return done.returncode, json.loads(done.stdout) + + +@pytest.fixture() +def corpus(tmp_path: Path) -> Path: + root = tmp_path / "plain-documents" + root.mkdir() + (root / "note.md").write_text("# A note\n\nPlain text.\n", encoding="utf-8") + return root + + +# -- (iii): the packaging ---------------------------------------------------- + + +def test_the_local_extra_carries_the_runtime_and_the_server_and_test_joins_it( +) -> None: + project = tomllib.loads((ROOT / "pyproject.toml").read_text()) + extras = project["project"]["optional-dependencies"] + assert "opendox[runtime]" in extras["local"] + assert any(req.startswith("pgserver") for req in extras["local"]) + assert "opendox[local]" in extras["test"], ( + "F9.1 installs `.[test]` alone; without the local extra there, this " + "suite could not start the server it tests") + lock = (ROOT / "constraints-cpython312-linux.txt").read_text() + assert re.search(r"(?m)^pgserver==", lock), "the lock does not pin pgserver" + files = project["tool"]["setuptools"]["data-files"] + assert files == {"share/opendox/migrations": ["migrations/*.sql"]} + + +# -- the layout, before anything starts ---------------------------------------- + + +def test_the_two_dsns_are_two_users_over_the_one_socket(state_dir: Path) -> None: + settings = config.load_settings({MODE: "local", STATE: str(state_dir)}) + bundle = config.database_bundle(state_dir) + assert settings.database_url == bundle.served_dsn + assert settings.migration_database_url == bundle.migration_dsn + assert settings.database_url != settings.migration_database_url # 13.3 + for dsn in (settings.database_url, settings.migration_database_url): + assert dsn.startswith("postgresql://") # 13.2 + assert f"host={bundle.socket_dir}" in dsn and "port=5432" in dsn + assert config.user_named_by(settings.database_url) == config.BUNDLE_SERVED_ROLE + assert config.user_named_by(settings.migration_database_url) == \ + config.BUNDLE_OWNER_ROLE + assert bundle.data_dir.parent == bundle.socket_dir.parent + assert bundle.data_dir.is_relative_to(state_dir) + assert bundle.socket_dir.is_relative_to(state_dir) + + +def test_a_state_dir_too_long_for_a_unix_socket_is_refused_naming_it() -> None: + long = "/tmp/" + "x" * 120 + with pytest.raises(config.ConfigurationError) as caught: + config.load_settings({MODE: "local", STATE: long}) + assert STATE in str(caught.value) and "socket" in str(caught.value) + + +def test_a_relative_state_dir_is_refused_naming_it() -> None: + with pytest.raises(config.ConfigurationError) as caught: + config.load_settings({MODE: "local", STATE: "var/state"}) + assert STATE in str(caught.value) + + +def test_the_default_state_dir_is_the_users_own(monkeypatch) -> None: + home = config.state_dir({"HOME": "/ignored"}) + assert home.name == "opendox" and home.is_absolute() + assert config.state_dir({"XDG_STATE_HOME": "/srv/state"}) == \ + Path("/srv/state/opendox") + # the XDG rule: a relative value is ignored, never joined onto the cwd + assert config.state_dir({"XDG_STATE_HOME": "relative"}) == \ + Path.home() / ".local" / "state" / "opendox" + + +# -- 13.1 and R1Q16 (i), (ii), (iv): F13.1's two blocks, on the real entry point + + +def test_the_entry_point_owns_a_migrated_server_with_no_tcp_listener( + corpus: Path, state_dir: Path, tmp_path: Path) -> None: + """F13.1's `runtime status` block and its TCP-listener block, against the + server `generate-and-open --local` started in the background; then + `kill "$SERVER"; wait`, and the server is gone with it.""" + server, url = _launch(corpus, state_dir, tmp_path / "run") + try: + with urllib.request.urlopen(url, timeout=10) as answer: # ready + assert answer.status == 200 + # -- F13.1's `runtime status` block -------------------------------- + code, status = _status(state_dir) + assert status.get("database") == "reachable", \ + f"no bundled database answered: {status}" + assert status.get("applied_migrations") and \ + not status.get("pending_migrations"), f"not migrated: {status}" + state = os.path.realpath(state_dir) + bundle = status.get("database_bundle") or {} + for key in ("data_dir", "socket_dir"): + got = os.path.realpath(bundle.get(key, "")) + assert got.startswith(state + os.sep), \ + f"{key} {got!r} is not under the install's state dir {state!r}" + # a local install has no broker, and that is not a fault (T070) + assert status["broker_keys"] == "not configured (local mode)" + assert code == 0 and status["ok"] is True, status + # -- F13.1's TCP-listener block, the pid from `database_bundle` ----- + pid = bundle.get("pid") + assert isinstance(pid, int), f"the bundle reports no server pid: {pid!r}" + assert not _tcp_listeners(pid), \ + f"the bundled server listens on TCP: {_tcp_listeners(pid)}" + # -- R1Q16 (i): the server is the document server's own child ------ + assert _parent_of(pid) == server.pid, ( + "the bundled server is not a child of the process serving the " + "document surface") + mode = os.stat(bundle["socket_dir"]).st_mode & 0o777 + assert mode == 0o700, f"the socket directory is {mode:o}" + finally: + server.send_signal(signal.SIGTERM) # `kill "$SERVER"` + server.communicate(timeout=60) # `wait "$SERVER"` + # -- R1Q16 (iv): it stopped with the entry point, and cleanly ---------- + assert server.returncode == 0, server.returncode + assert _wait_gone(pid), "the bundled server outlived its entry point" + assert config.DatabaseBundle(state_dir).data_dir.joinpath("PG_VERSION").is_file(), \ + "the data directory must survive a stop: it is the install's database" + + +@pytest.mark.skipif(not sys.platform.startswith("linux"), + reason="PR_SET_PDEATHSIG is Linux's") +def test_the_server_stops_even_when_the_entry_point_is_killed_outright( + corpus: Path, state_dir: Path, tmp_path: Path) -> None: + """(iv)'s backstop: SIGKILL gives the entry point no chance to stop + anything, and the parent-death signal stops the server all the same.""" + server, _url = _launch(corpus, state_dir, tmp_path / "run") + pid = bundle_mod.running_pid(config.DatabaseBundle(state_dir)) + assert isinstance(pid, int) + server.kill() + server.communicate(timeout=30) + assert _wait_gone(pid), "the bundled server outlived a SIGKILLed entry point" + + +def test_a_second_entry_point_on_the_same_state_dir_is_refused( + state_dir: Path) -> None: + settings = config.load_settings({MODE: "local", STATE: str(state_dir)}) + first = bundle_mod.BundledServer(settings).start() + try: + with pytest.raises(bundle_mod.BundleRefused) as caught: + bundle_mod.BundledServer(settings).start() + assert str(first.report()["pid"]) in str(caught.value) + # and a restart of the one that owns it re-migrates nothing + assert first.applied == ["0001", "0002"] + finally: + first.stop() + again = bundle_mod.BundledServer(settings).start() + try: + assert again.applied == [] + finally: + again.stop() + + +def test_migrate_under_the_local_mode_uses_the_bundle_and_refuses_a_dsn( + state_dir: Path) -> None: + """`runtime migrate` is part of the same install: it reads the bundle's + owner DSN, and an operator's migration DSN beside `local` is refused.""" + settings = config.load_settings({MODE: "local", STATE: str(state_dir)}) + with bundle_mod.BundledServer(settings): + done = subprocess.run( + [sys.executable, "-m", "opendox.runtime.cli", "runtime", "migrate"], + env=_clean_env(**{MODE: "local", STATE: str(state_dir)}), cwd=ROOT, + capture_output=True, text=True, timeout=60) + evidence = json.loads(done.stdout) + assert done.returncode == 0 and evidence["applied"] == [], evidence + done = subprocess.run( + [sys.executable, "-m", "opendox.runtime.cli", "runtime", "migrate"], + env=_clean_env(**{MODE: "local", STATE: str(state_dir), + PREFIX + "MIGRATION_DATABASE_URL": + "postgresql://m:hunter2@db.invalid/x"}), + cwd=ROOT, capture_output=True, text=True, timeout=60) + evidence = json.loads(done.stdout) + assert evidence["refusal"] == "configuration", evidence + assert PREFIX + "MIGRATION_DATABASE_URL" in evidence["message"] + assert "hunter2" not in done.stdout + + +# -- the migrations gap: a wheel, outside any checkout -------------------------- + + +def test_a_wheel_install_migrates_its_bundled_server_outside_a_checkout( + state_dir: Path, tmp_path: Path) -> None: + """Build this package's wheel, install it OUTSIDE the checkout, run it from + a directory with no `migrations/`, and migrate the bundled server from the + copy the wheel carries (the holder's assignment to T072).""" + source = tmp_path / "source" + source.mkdir() + for name in ("pyproject.toml", "src", "migrations"): + item = ROOT / name + (shutil.copytree if item.is_dir() else shutil.copy2)( + item, source / name, **({"ignore": shutil.ignore_patterns( + "__pycache__", "*.egg-info")} if item.is_dir() else {})) + wheels = tmp_path / "wheels" + built = subprocess.run( + [sys.executable, "-m", "pip", "wheel", "--no-deps", "--no-index", + "--no-build-isolation", "-q", "-w", str(wheels), str(source)], + capture_output=True, text=True, timeout=300) + assert built.returncode == 0, built.stderr[-3000:] + (wheel,) = wheels.glob("opendox-*.whl") + prefix = tmp_path / "prefix" + # `--ignore-installed` IS LOAD-BEARING: without it pip treats the suite's + # own (editable) `opendox` as the installed copy of the same project and + # UNINSTALLS it before writing the new one under `--prefix` — measured, it + # emptied the environment this very suite runs in. + installed = subprocess.run( + [sys.executable, "-m", "pip", "install", "--no-deps", "--no-index", + "--ignore-installed", "-q", "--prefix", str(prefix), str(wheel)], + capture_output=True, text=True, timeout=300) + assert installed.returncode == 0, installed.stderr[-3000:] + assert "uninstall" not in (installed.stdout + installed.stderr).lower() + # ...and the environment this suite runs in still has its own install + still = subprocess.run( + [sys.executable, "-c", "import importlib.metadata as m; " + "print(m.distribution('opendox').version)"], + env=_clean_env(PYTHONPATH=""), capture_output=True, text=True, timeout=60) + assert still.returncode == 0, "installing the wheel removed the suite's opendox" + site = Path(sysconfig.get_path("purelib", vars={"base": str(prefix), + "platbase": str(prefix)})) + elsewhere = tmp_path / "elsewhere" + elsewhere.mkdir() + program = f""" +import json, sys +from pathlib import Path +import opendox +from opendox.runtime import bundle, config +prefix = Path({str(prefix)!r}).resolve() +assert Path(opendox.__file__).resolve().is_relative_to(prefix), opendox.__file__ +assert not Path("migrations").exists() +settings = config.load_settings() +found = Path(settings.migrations_dir).resolve() +assert found.is_relative_to(prefix / "share" / "opendox"), found +with bundle.BundledServer(settings) as server: + print(json.dumps({{"applied": server.applied, "dir": str(found)}})) +""" + env = _clean_env(**{MODE: "local", STATE: str(state_dir)}) + env["PYTHONPATH"] = str(site) + done = subprocess.run([sys.executable, "-c", program], cwd=elsewhere, + env=env, capture_output=True, text=True, timeout=120) + assert done.returncode == 0, done.stderr[-3000:] + result = json.loads(done.stdout.strip().splitlines()[-1]) + assert result["applied"] == ["0001", "0002"], result diff --git a/tests_runtime/test_install_mode.py b/tests_runtime/test_install_mode.py index de9ca524..ad1225bf 100644 --- a/tests_runtime/test_install_mode.py +++ b/tests_runtime/test_install_mode.py @@ -62,6 +62,11 @@ HOSTED = {**DSNS, PREFIX + "OIDC_ISSUER": "https://issuer.example.invalid/realms/fixture", PREFIX + "OIDC_AUDIENCE": "fixture"} +#: A LOCAL install's whole environment: the selector and a state directory. +#: No DSN — the local install supplies both from the server it bundles +#: (plan 034 T072), and one given beside it is refused. Nothing is created +#: there: `load_settings` derives paths, and starts nothing. +LOCAL = {MODE: "local", PREFIX + "STATE_DIR": "/nonexistent/opendox-state"} def _refusal(env: dict, **kwargs) -> str: @@ -190,7 +195,7 @@ def test_the_hosted_mode_is_unchanged() -> None: @pytest.mark.parametrize("selection", ["setting", "flag"]) def test_a_local_install_needs_no_broker(selection: str) -> None: - env = dict(DSNS) + env = {PREFIX + "STATE_DIR": LOCAL[PREFIX + "STATE_DIR"]} kwargs = {} if selection == "setting": env[MODE] = "local" @@ -213,13 +218,19 @@ def test_a_broker_setting_beside_the_local_mode_is_refused_by_name( """A holder reading (#656, T070): a broker setting says hosted was meant.""" assert set(HOSTED_ONLY_SETTINGS) == {PREFIX + "OIDC_ISSUER", PREFIX + "OIDC_AUDIENCE", - PREFIX + "OIDC_JWKS_URL"} - secret = "https://svc:hunter2@broker.example.invalid/realms/x" - message = _refusal({**DSNS, MODE: "local", name: secret}) + PREFIX + "OIDC_JWKS_URL", + # and T072's two: the local install + # supplies both DSNs itself (13.1) + PREFIX + "DATABASE_URL", + PREFIX + "MIGRATION_DATABASE_URL"} + secret = ("https://svc:hunter2@broker.example.invalid/realms/x" + if "OIDC" in name else "postgresql://u:hunter2@db.invalid/x") + message = _refusal({**LOCAL, name: secret}) assert name in message, message assert "hunter2" not in message, "the value must not be repeated" # and the flag spelling of the same selection refuses it the same way - assert name in _refusal({**DSNS, name: secret}, local_flag=True) + assert name in _refusal({PREFIX + "STATE_DIR": LOCAL[PREFIX + "STATE_DIR"], + name: secret}, local_flag=True) def test_every_broker_setting_given_is_named_at_once() -> None: @@ -231,8 +242,7 @@ def test_every_broker_setting_given_is_named_at_once() -> None: @pytest.mark.parametrize("host", sorted(LOCAL_BIND_HOSTS)) def test_a_local_install_binds_each_loopback_spelling(host: str) -> None: - settings = load_settings({**DSNS, MODE: "local", - PREFIX + "BIND_HOST": host}) + settings = load_settings({**LOCAL, PREFIX + "BIND_HOST": host}) assert settings.bind_host == host @@ -243,7 +253,7 @@ def test_a_local_install_refuses_a_non_loopback_bind_naming_the_rule( """13.4: loopback ONLY, and no opt-in. `127.0.0.2` is refused too: the document server does not treat it as loopback (`serve.LOOPBACK_HOSTS`), and the mode makes the SAME judgement at its own boundary.""" - message = _refusal({**DSNS, MODE: "local", PREFIX + "BIND_HOST": host}) + message = _refusal({**LOCAL, PREFIX + "BIND_HOST": host}) assert PREFIX + "BIND_HOST" in message, message assert "loopback" in message.lower(), message assert "no opt-in" in message.lower(), message @@ -283,9 +293,8 @@ def run(self) -> None: stub.Config, stub.Server = _Config, _Server scrubbed.setitem(sys.modules, "uvicorn", stub) scrubbed.setattr(app_module, "create_app", lambda **kwargs: object()) - for name, value in DSNS.items(): + for name, value in LOCAL.items(): scrubbed.setenv(name, value) - scrubbed.setenv(MODE, "local") code, evidence = _run(["runtime", "serve"]) assert served == [], "the API was started for a LOCAL install" assert code == 1 @@ -306,9 +315,8 @@ def _no_broker(_settings): "install, which has no broker") monkeypatch.setattr(oidc, "build_verifier", _no_broker) - for name, value in DSNS.items(): + for name, value in LOCAL.items(): scrubbed.setenv(name, value) - scrubbed.setenv(MODE, "local") code, evidence = _run(["runtime", "status", "--probe-timeout", "0.2"]) assert evidence.get("refusal") is None, evidence assert evidence["broker_keys"] == "not configured (local mode)" @@ -316,8 +324,11 @@ def _no_broker(_settings): assert evidence["settings"][MODE] == INSTALL_MODE_LOCAL assert evidence["settings"][PREFIX + "OIDC_ISSUER"] == "" assert evidence["settings"][PREFIX + "OIDC_JWKS_URL"] == "" - # the database half (port 1, unreachable) is the ONLY reason `ok` is false + # the database half is the ONLY reason `ok` is false: no bundled server + # is running on this (nonexistent) state directory, and `status` reports + # that rather than starting one assert evidence["database"].startswith("unreachable"), evidence + assert evidence["database_bundle"]["pid"] is None, evidence assert code == 1 diff --git a/tests_runtime/test_runtime_surface.py b/tests_runtime/test_runtime_surface.py index ff6a73a3..5de1373c 100644 --- a/tests_runtime/test_runtime_surface.py +++ b/tests_runtime/test_runtime_surface.py @@ -50,6 +50,11 @@ # one would make openDox the only destination the corpus could not test. "opendox.runtime.local_git_adapter", "opendox.runtime.repository_act", + # THE LOCAL INSTALL'S BUNDLED SERVER (plan 034 T072). `opendox.cli` + # imports it at module level, and `opendox.cli` is what `opendox --help` + # runs on an install with no extra at all, so it connects through + # `psycopg` only inside the functions that start the server. + "opendox.runtime.bundle", ) #: The modules that legitimately need the `runtime` extra, and the only ones. From c3a70a2d1145a7fd7bcc56cbf851a4c7513bdd47 Mon Sep 17 00:00:00 2001 From: Brett Heap <1513478+brettheap@users.noreply.github.com> Date: Wed, 30 Sep 2026 00:47:47 +0000 Subject: [PATCH 07/88] T072: extend the dependency lock for the `local` extra and the test extra's setuptools The lock is extended under its own pins (`-c` this file), in a clean cpython 3.12.3 venv on linux x86_64, as its header asks. Five pins are new: - pgserver 0.1.4, with its own psutil, platformdirs and fasteners; - setuptools, for the wheel-install test's offline build. No earlier pin moved. Arc: neutral-product-standalone-operability Lane: openxfactory-4 (openXfactory-4-openDox_extraction) Co-Authored-By: Claude Opus 5.5 (1M context) --- constraints-cpython312-linux.txt | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/constraints-cpython312-linux.txt b/constraints-cpython312-linux.txt index 84ed0972..4772dbe1 100644 --- a/constraints-cpython312-linux.txt +++ b/constraints-cpython312-linux.txt @@ -30,6 +30,10 @@ # two installs, no second list of names to drift. # # Resolved 2026-09-18 on cpython 3.12.3 / linux x86_64. +# Extended 2026-09-30 on cpython 3.12.3 / linux x86_64, under the pins above +# (`-c` this file), for plan 034 T072's `local` extra and the `test` extra's +# `setuptools`: pgserver, its psutil, platformdirs and fasteners, and +# setuptools are new; no earlier pin moved. PyJWT==2.14.0 PyYAML==6.0.3 Pygments==2.21.0 @@ -41,6 +45,7 @@ cffi==2.1.1 click==8.5.0 cryptography==50.0.1 fastapi==0.141.1 +fasteners==0.20 h11==0.16.0 httpcore==1.0.9 httptools==0.8.0 @@ -48,7 +53,10 @@ httpx==0.28.1 idna==3.20 iniconfig==2.3.0 packaging==26.3 +pgserver==0.1.4 +platformdirs==4.12.2 pluggy==1.6.0 +psutil==7.2.2 psycopg-binary==3.3.6 psycopg-pool==3.3.2 psycopg==3.3.6 @@ -57,6 +65,7 @@ pydantic==2.13.5 pydantic_core==2.46.5 pytest==8.4.2 python-dotenv==1.2.3 +setuptools==84.0.0 starlette==1.6.0 typing-inspection==0.4.4 typing_extensions==4.16.0 From 32683e8f2ec5aa9443e572739ced835d4628161a Mon Sep 17 00:00:00 2001 From: Brett Heap <1513478+brettheap@users.noreply.github.com> Date: Wed, 30 Sep 2026 00:48:46 +0000 Subject: [PATCH 08/88] Fix round: the install-mode fixture's repository ignores the user's git config (Copilot review) `tests/test_install_mode_entrypoint.py`'s `corpus` fixture ran `git commit` under the caller's global and system git configuration. A global `commit.gpgsign=true` therefore failed the setup before any install-mode probe ran. Measured with a hostile global config (`commit.gpgsign = true`, `gpg.program = /bin/false`): 7 errors at b50e3b1, 14 passed here. The fixture now sets GIT_CONFIG_GLOBAL=/dev/null and GIT_CONFIG_NOSYSTEM=1, as tests/test_checkout_head.py does. Arc: neutral-product-standalone-operability Lane: openxfactory-4 (openXfactory-4-openDox_extraction) Co-Authored-By: Claude Opus 5.5 (1M context) --- tests/test_install_mode_entrypoint.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/tests/test_install_mode_entrypoint.py b/tests/test_install_mode_entrypoint.py index 2a121ef4..9c4c6203 100644 --- a/tests/test_install_mode_entrypoint.py +++ b/tests/test_install_mode_entrypoint.py @@ -49,11 +49,19 @@ @pytest.fixture() def corpus(tmp_path: Path) -> Path: - """A fresh repository, as F13.1's preamble makes one.""" + """A fresh repository, as F13.1's preamble makes one. + + Made APART FROM the user's own git configuration (`GIT_CONFIG_GLOBAL`, + `GIT_CONFIG_NOSYSTEM`), as `tests/test_checkout_head.py` makes its + repositories, so a global signing rule or hook cannot fail the setup + before the install-mode probe it exists for ever runs (Copilot review of + this PR). + """ root = tmp_path / "plain-documents" root.mkdir() (root / "note.md").write_text("# A note\n\nPlain text.\n", encoding="utf-8") env = {**os.environ, + "GIT_CONFIG_GLOBAL": os.devnull, "GIT_CONFIG_NOSYSTEM": "1", "GIT_AUTHOR_NAME": "fixture", "GIT_AUTHOR_EMAIL": "fixture@example.invalid", "GIT_COMMITTER_NAME": "fixture", "GIT_COMMITTER_EMAIL": "fixture@example.invalid"} From 525f61c31db5df8b5264bb6a4f485ea0325d76b4 Mon Sep 17 00:00:00 2001 From: Brett Heap <1513478+brettheap@users.noreply.github.com> Date: Wed, 30 Sep 2026 00:58:12 +0000 Subject: [PATCH 09/88] Fix round: runtime migrate and reset refuse what a local install cannot be (Copilot review) load_migration_settings recorded OPENDOX_INSTALL_MODE=local but never asked refuse_what_a_local_install_cannot_be. So `runtime migrate` and a confirmed `runtime reset` accepted OPENDOX_OIDC_ISSUER, OPENDOX_OIDC_AUDIENCE, OPENDOX_OIDC_JWKS_URL or a non-loopback OPENDOX_BIND_HOST beside `local`, which load_settings and generate-and-open both refuse. They now refuse them at configuration, before any database is reached. Seven new cases: - the three broker settings x {migrate, reset}; - the bind. All seven fail at 32683e8 and pass here. Full suite: 2538 selected, 2527 passed, 11 skipped, 0 failed. Arc: neutral-product-standalone-operability Lane: openxfactory-4 (openXfactory-4-openDox_extraction) Co-Authored-By: Claude Opus 5.5 (1M context) --- src/opendox/runtime/config.py | 19 +++++++++++++----- tests_runtime/test_install_mode.py | 31 ++++++++++++++++++++++++++++++ 2 files changed, 45 insertions(+), 5 deletions(-) diff --git a/src/opendox/runtime/config.py b/src/opendox/runtime/config.py index 6edd5c88..326144e4 100644 --- a/src/opendox/runtime/config.py +++ b/src/opendox/runtime/config.py @@ -1801,6 +1801,16 @@ def load_migration_settings(env: Mapping[str, str] | None = None) -> RuntimeSett not, because those are the served runtime and must have the real thing. """ env = os.environ if env is None else env + # THE ONE READING OF THE SELECTOR, AND OF WHAT A LOCAL INSTALL CANNOT BE + # (plan 034 T070; Copilot review of openDox-code#67). A migration run is + # part of the same install as the served one, so `runtime migrate` and + # `runtime reset` refuse what `load_settings` and `generate-and-open` + # refuse beside `local` — a broker setting, or a non-loopback + # `OPENDOX_BIND_HOST` — rather than accepting it in the one loader that + # never reads it. + mode = install_mode(env) + if mode == INSTALL_MODE_LOCAL: + refuse_what_a_local_install_cannot_be(env) dsn = env.get(PREFIX + "MIGRATION_DATABASE_URL", "").strip() if not dsn: raise ConfigurationError( @@ -1818,11 +1828,10 @@ def load_migration_settings(env: Mapping[str, str] | None = None) -> RuntimeSett return RuntimeSettings( database_url=dsn, migration_database_url=dsn, - # READ, SO AN UNRECOGNISED VALUE IS REFUSED HERE TOO (plan 034 T070): - # a migration run is part of the same install and one reading of the - # selector serves every verb. It changes nothing else a migration run - # does; the broker fields below are sentinels in either shape. - install_mode=install_mode(env), + # READ ABOVE, SO AN UNRECOGNISED VALUE IS REFUSED HERE TOO (plan 034 + # T070). It changes nothing else a migration run does; the broker + # fields below are sentinels in either shape. + install_mode=mode, oidc_issuer=MIGRATION_SENTINEL_ISSUER, oidc_audience=MIGRATION_SENTINEL_AUDIENCE, oidc_jwks_url=None, diff --git a/tests_runtime/test_install_mode.py b/tests_runtime/test_install_mode.py index de9ca524..28932ab3 100644 --- a/tests_runtime/test_install_mode.py +++ b/tests_runtime/test_install_mode.py @@ -321,6 +321,37 @@ def _no_broker(_settings): assert code == 1 +@pytest.mark.parametrize("verb", [["runtime", "migrate"], + ["runtime", "reset", "--confirm", + cli.RESET_CONFIRMATION]]) +@pytest.mark.parametrize("name", [PREFIX + "OIDC_ISSUER", + PREFIX + "OIDC_AUDIENCE", + PREFIX + "OIDC_JWKS_URL"]) +def test_migrate_and_reset_refuse_a_broker_setting_beside_the_local_mode( + scrubbed, verb: list[str], name: str) -> None: + """The migration loader asks what every other loader asks of `local` + (Copilot review of openDox-code#67): refused at CONFIGURATION, before any + database is reached — `reset` included, confirmation and all.""" + scrubbed.setenv(PREFIX + "MIGRATION_DATABASE_URL", DSNS[PREFIX + "MIGRATION_DATABASE_URL"]) + scrubbed.setenv(MODE, "local") + scrubbed.setenv(name, "https://issuer.example.invalid/realms/x" + if name != PREFIX + "OIDC_AUDIENCE" else "fixture") + code, evidence = _run(verb) + assert code == 1 + assert evidence["refusal"] == "configuration", evidence + assert name in evidence["message"], evidence + + +def test_migrate_refuses_a_non_loopback_bind_beside_the_local_mode( + scrubbed) -> None: + scrubbed.setenv(PREFIX + "MIGRATION_DATABASE_URL", DSNS[PREFIX + "MIGRATION_DATABASE_URL"]) + scrubbed.setenv(MODE, "local") + scrubbed.setenv(PREFIX + "BIND_HOST", "0.0.0.0") + code, evidence = _run(["runtime", "migrate"]) + assert code == 1 and evidence["refusal"] == "configuration", evidence + assert "loopback" in evidence["message"].lower(), evidence + + def test_runtime_status_reports_the_hosted_mode_it_loaded(scrubbed) -> None: for name, value in HOSTED.items(): scrubbed.setenv(name, value) From 02dadc5590dafe69a2ef156dc3d77bf275e24d67 Mon Sep 17 00:00:00 2001 From: Brett Heap <1513478+brettheap@users.noreply.github.com> Date: Wed, 30 Sep 2026 01:12:23 +0000 Subject: [PATCH 10/88] Fix round: status reports a local install's broker on its early return too (Copilot review) When the runtime extra is absent, `runtime status` returns early, and that return said `broker_keys: "not probed"` for every install. A local install's broker is not configured whether or not the extra is present. That answer comes from the configuration, not from a probe, so the early return now gives the local install the answer the full report gives: `"not configured (local mode)"`, with `broker_discovery: null`. Both returns write it through one helper, so the two cannot drift. A hosted install's early return still reads "not probed", as before (13.6). The branch is covered now, so its `pragma: no cover` goes. A new case runs both shapes with `opendox.runtime.db` absent from `sys.modules`. Before (`525f61c`'s runtime/cli.py): local 1 failed and hosted passed. After: both pass. Four mutants of the fix are killed. Full suite: 2540 selected, 2529 passed, 11 skipped, 0 failed. Lane: openxfactory-4 (openXfactory-4-openDox_extraction) Arc: neutral-product-standalone-operability Co-Authored-By: Claude Opus 5.5 (1M context) --- src/opendox/runtime/cli.py | 28 ++++++++++++++++++++++++---- tests_runtime/test_install_mode.py | 28 ++++++++++++++++++++++++++++ 2 files changed, 52 insertions(+), 4 deletions(-) diff --git a/src/opendox/runtime/cli.py b/src/opendox/runtime/cli.py index af0ac390..a69bfd41 100644 --- a/src/opendox/runtime/cli.py +++ b/src/opendox/runtime/cli.py @@ -664,6 +664,19 @@ def cmd_serve(args: argparse.Namespace) -> int: "exiting normally"}, ok=True) +def _report_the_local_broker(report: dict[str, Any]) -> None: + """What `status` says of a LOCAL install's broker, on every path. + + Its broker is NOT CONFIGURED (plan 034 T070; #1144 13.4): a statement + about the install's configuration rather than a probe's result, so there + is no discovery URL to report and nothing counts against `ok`. `status` + returns from two places, and both write it here, so the two answers + cannot drift apart (Copilot review of openDox-code#67). + """ + report["broker_keys"] = "not configured (local mode)" + report["broker_discovery"] = None + + def cmd_status(args: argparse.Namespace) -> int: """Report, never change: configuration, the schema pin, the ledger, the broker. @@ -693,10 +706,18 @@ def cmd_status(args: argparse.Namespace) -> int: try: from opendox.runtime.db import Database - except ImportError as exc: # pragma: no cover - the extra is absent + except ImportError as exc: report["runtime_extra"] = f"absent: {_safe_message(exc)}" report["database"] = "not probed" - report["broker_keys"] = "not probed" + # A LOCAL INSTALL'S BROKER IS NOT CONFIGURED WHETHER OR NOT THE EXTRA + # IS PRESENT (plan 034 T070; Copilot review of openDox-code#67). That + # answer comes from its configuration, not from a probe, so this early + # return gives the same one the full report gives below. A hosted + # install's broker was never probed, and says so, as before. + if settings.install_mode == INSTALL_MODE_LOCAL: + _report_the_local_broker(report) + else: + report["broker_keys"] = "not probed" return _emit(report, ok=False) report["runtime_extra"] = "present" @@ -778,8 +799,7 @@ def cmd_status(args: argparse.Namespace) -> int: # — F13.1 runs it under `set -e`, and a verdict of "unhealthy" for a # broker the install was never meant to have would be false. if settings.install_mode == INSTALL_MODE_LOCAL: - report["broker_keys"] = "not configured (local mode)" - report["broker_discovery"] = None + _report_the_local_broker(report) return _emit(report, ok=ok) try: from opendox.runtime.oidc import build_verifier diff --git a/tests_runtime/test_install_mode.py b/tests_runtime/test_install_mode.py index 28932ab3..f9c4bf9e 100644 --- a/tests_runtime/test_install_mode.py +++ b/tests_runtime/test_install_mode.py @@ -352,6 +352,34 @@ def test_migrate_refuses_a_non_loopback_bind_beside_the_local_mode( assert "loopback" in evidence["message"].lower(), evidence +@pytest.mark.parametrize("mode", [INSTALL_MODE_LOCAL, INSTALL_MODE_HOSTED]) +def test_runtime_status_without_the_runtime_extra_reports_the_broker_by_mode( + scrubbed, mode: str) -> None: + """`status` returns early when the runtime extra is absent, and that + return gives the local broker the same answer the full report gives + (Copilot review of openDox-code#67). A hosted install's broker reads + "not probed", unchanged (13.6). `None` in `sys.modules` is how an absent + module is simulated: the import raises `ImportError`, as it would without + the extra.""" + import sys + + scrubbed.setitem(sys.modules, "opendox.runtime.db", None) + for name, value in (HOSTED if mode == INSTALL_MODE_HOSTED else DSNS).items(): + scrubbed.setenv(name, value) + scrubbed.setenv(MODE, mode) + code, evidence = _run(["runtime", "status", "--probe-timeout", "0.2"]) + assert evidence["runtime_extra"].startswith("absent"), evidence + assert evidence["database"] == "not probed", evidence + assert code == 1 + if mode == INSTALL_MODE_LOCAL: + assert evidence["broker_keys"] == "not configured (local mode)", evidence + assert "broker_discovery" in evidence, evidence + assert evidence["broker_discovery"] is None + else: + assert evidence["broker_keys"] == "not probed", evidence + assert "broker_discovery" not in evidence, evidence + + def test_runtime_status_reports_the_hosted_mode_it_loaded(scrubbed) -> None: for name, value in HOSTED.items(): scrubbed.setenv(name, value) From 859b37b60a6c70818bcfdd8749ce232dcdfb7741 Mon Sep 17 00:00:00 2001 From: Brett Heap <1513478+brettheap@users.noreply.github.com> Date: Wed, 30 Sep 2026 15:04:37 +0000 Subject: [PATCH 11/88] Fix round: a healthy local status is proven to exit 0, and RuntimeSettings' broker invariants are scoped (Copilot review) A local `status` returns `ok` on the database's verdict alone. Both earlier local cases forced a database fault and asserted exit 1, so a regression that also counted the absent broker as a fault would still have passed. A DB-backed case now runs `status` for a local install against a migrated schema on the suite's own server (`database` and `postgres_dsn`, with the schema selected in the DSN). It asserts `ok` true, exit 0, the database reachable with nothing pending and no drift, and the broker reported as not configured and never probed. Measured: with the local return mutated to `ok=False`, this case fails and the other 40 in the module pass. `RuntimeSettings`' docstring said that a local install's issuer and audience are empty and that a hosted one always carries a real issuer. That is true of `load_settings` alone. `load_migration_settings` carries the migration sentinels in either shape. The docstring now scopes each statement to its loader. Full suite: 2541 selected, 2530 passed, 11 skipped, 0 failed. Lane: openxfactory-4 (openXfactory-4-openDox_extraction) Arc: neutral-product-standalone-operability Co-Authored-By: Claude Opus 5.5 (1M context) --- src/opendox/runtime/config.py | 22 +++++++++++++++------ tests_runtime/test_install_mode.py | 31 ++++++++++++++++++++++++++++++ 2 files changed, 47 insertions(+), 6 deletions(-) diff --git a/src/opendox/runtime/config.py b/src/opendox/runtime/config.py index 326144e4..ad8c518d 100644 --- a/src/opendox/runtime/config.py +++ b/src/opendox/runtime/config.py @@ -203,12 +203,22 @@ class RuntimeSettings: requires one (unaffected by this: it already refused to load without one). `install_mode` is `INSTALL_MODE_HOSTED` or `INSTALL_MODE_LOCAL` (plan 034 - T070; #1144 13.4). A LOCAL install has no broker, so its `oidc_issuer` and - `oidc_audience` are EMPTY and its `oidc_jwks_url` is `None` — never a - placeholder that looks like an endpoint — and `jwks_url()` and - `discovery_url()` answer the empty string for it rather than a path glued - onto nothing. A hosted install always carries a real issuer, because - `load_settings` refuses one without it. + T070; #1144 13.4), whichever loader built the object. + + THE BROKER FIELDS DEPEND ON THE LOADER, and what follows holds for + `load_settings` only (Copilot review of openDox-code#67): + * From `load_settings`, a LOCAL install has no broker. Its + `oidc_issuer` and `oidc_audience` are EMPTY and its `oidc_jwks_url` + is `None`, never a placeholder that looks like an endpoint, and + `jwks_url()` and `discovery_url()` answer the empty string rather + than a path glued onto nothing. A HOSTED one always carries a real + issuer, because `load_settings` refuses one without it. + * From `load_migration_settings`, in EITHER shape, the issuer and + audience are `MIGRATION_SENTINEL_ISSUER` and + `MIGRATION_SENTINEL_AUDIENCE`. A migration run reaches no broker at + all, and anything that tried to with those values would fail naming + them. So neither statement above applies to it. `install_mode` there + records the shape the run belongs to, and nothing else. """ database_url: str diff --git a/tests_runtime/test_install_mode.py b/tests_runtime/test_install_mode.py index f9c4bf9e..8fcc2a7b 100644 --- a/tests_runtime/test_install_mode.py +++ b/tests_runtime/test_install_mode.py @@ -352,6 +352,37 @@ def test_migrate_refuses_a_non_loopback_bind_beside_the_local_mode( assert "loopback" in evidence["message"].lower(), evidence +def test_runtime_status_of_a_healthy_local_install_exits_zero( + scrubbed, monkeypatch: pytest.MonkeyPatch, postgres_dsn: str, + database) -> None: + """THE EXIT CODE IS THE DATABASE'S VERDICT ALONE (Copilot review of + openDox-code#67). A migrated, reachable database is the only thing a + local install's `status` needs, and with it the verb exits 0. The broker + is reported as not configured and is never probed, so F13.1's `set -e` + survives. The other local cases force a database fault and exit 1, so + they cannot tell a broker counted as a fault from a database that failed. + """ + from opendox.runtime import oidc + + def _no_broker(_settings): + raise AssertionError("status built a broker verifier for a LOCAL " + "install, which has no broker") + + monkeypatch.setattr(oidc, "build_verifier", _no_broker) + joiner = "&" if "?" in postgres_dsn else "?" + scrubbed.setenv(PREFIX + "DATABASE_URL", + f"{postgres_dsn}{joiner}options=-c%20search_path%3D" + f"{database.schema}%2Cpublic") + scrubbed.setenv(MODE, "local") + code, evidence = _run(["runtime", "status", "--probe-timeout", "5"]) + assert evidence["database"] == "reachable", evidence + assert evidence["pending_migrations"] == [], evidence + assert not evidence["migration_drift"], evidence + assert evidence["broker_keys"] == "not configured (local mode)", evidence + assert evidence["broker_discovery"] is None + assert evidence["ok"] is True and code == 0, evidence + + @pytest.mark.parametrize("mode", [INSTALL_MODE_LOCAL, INSTALL_MODE_HOSTED]) def test_runtime_status_without_the_runtime_extra_reports_the_broker_by_mode( scrubbed, mode: str) -> None: From 5e528729aaa1097e4ff5009f7d2a50bfd78fc394 Mon Sep 17 00:00:00 2001 From: Brett Heap <1513478+brettheap@users.noreply.github.com> Date: Wed, 30 Sep 2026 15:11:40 +0000 Subject: [PATCH 12/88] Fix round: the local install's migrations, pid, initdb, start and interrupts hardened (Copilot review) Copilot's reviews at 95fe16f and 32db5d8 opened ten threads. Eight are fixed here. The two about the server package (PostgreSQL 16.2, no wheel for 3.13) wait on the holder. - Migrations (r4139811473, r4139880241). An explicit OPENDOX_MIGRATIONS_DIR is used as given. Unset, a LOCAL install uses only the copy its own installation carries, and never the working directory's: its entry point runs every migration as the bundle's owner, and the canonical gate pins 0001 alone. Where the installation carries none, it is refused, naming the setting. The installation's copy is the source tree the module was imported from (src/ beside a pyproject.toml naming opendox), then the RECORD of the distribution that holds the running module, and never another one found by name. A HOSTED install's unset default is unchanged (13.6). - The pid (r4139811555). A postmaster.pid is believed only for this data directory's postmaster, as the kernel reports it: an executable named postgres whose working directory is the data directory. Another user's process is never believed. A lock that /proc proves stale is removed before the launch, so a recycled pid no longer holds the bundle. - initdb (r4139880213). It runs into an attempt directory beside the data directory, which is renamed into place only on success. An attempt whose process is gone is removed. A non-empty data directory that holds no cluster is refused and left untouched. - start() (r4139880279). Directories, initialize, launch, wait, bootstrap and migrate are one guarded operation, and every failure is the one named refusal (phase and class name), with anything started stopped. - Interrupts (r4139880267). SIGTERM or Ctrl-C anywhere in the local lifecycle is a clean stop: no traceback, the bundle stopped, the handler restored first. Nothing was served, so the exit is 128 + the signal number. A served run ended by SIGTERM still exits 0. - Refusal wording (r4139880298). Broker settings and operator DSNs are two classes, and each is refused with its own reason. - The test helper (r4139811584). The launch helper is bounded by its deadline, through a selector. Measured with a silent 8 s child and a 1 s deadline: the old loop returned after 8.0 s, the new one after 1.0 s. tests_runtime/test_local_lifecycle.py (new, hermetic) holds these cases, plus a real-server stale-lock case and the helper's own case in test_bundled_postgres.py. Against ac61596's source, 17 of the module's first 18 cases fail. The one that passes is the hosted default, which is unchanged on purpose. All 23 mutants of the fixes are killed. Lane: openxfactory-4 (openXfactory-4-openDox_extraction) Arc: neutral-product-standalone-operability Co-Authored-By: Claude Opus 5.5 (1M context) --- src/opendox/cli.py | 34 +- src/opendox/runtime/bundle.py | 185 ++++++++-- src/opendox/runtime/config.py | 195 +++++++--- tests_runtime/test_bundled_postgres.py | 96 ++++- tests_runtime/test_local_lifecycle.py | 489 +++++++++++++++++++++++++ 5 files changed, 913 insertions(+), 86 deletions(-) create mode 100644 tests_runtime/test_local_lifecycle.py diff --git a/src/opendox/cli.py b/src/opendox/cli.py index 11c3b203..5e0073ee 100644 --- a/src/opendox/cli.py +++ b/src/opendox/cli.py @@ -459,9 +459,19 @@ def _resolve_install_shape(args: argparse.Namespace, return runtime_config.load_settings(env, local_flag=local_flag) -def _terminate_as_interrupt(signum, frame): # pragma: no cover - a signal +class _Terminated(KeyboardInterrupt): + """SIGTERM, raised as the interrupt the serve loop already stops cleanly on. + + A subclass, so the serve loop's own `except KeyboardInterrupt` still ends + a served run with 0 (F13.1's `kill "$SERVER"; wait "$SERVER"`). An + interrupt that arrives BEFORE the serve loop can still say which signal + it was. + """ + + +def _terminate_as_interrupt(signum, frame): """SIGTERM, read as the Ctrl-C the serve loop already stops cleanly on.""" - raise KeyboardInterrupt + raise _Terminated def cmd_generate_and_open(args: argparse.Namespace, *, opener=webbrowser.open) -> int: @@ -513,10 +523,28 @@ def cmd_generate_and_open(args: argparse.Namespace, *, opener=webbrowser.open) - print(f" database {report['socket_dir']} (bundled, pid {report['pid']}, " f"migrations applied now: {server.applied or 'none pending'})") return _generate_and_open(args, opener=opener) + except KeyboardInterrupt as interrupt: + # AN INTERRUPT ANYWHERE IN THE LOCAL LIFECYCLE IS A CLEAN STOP (Copilot + # review of openDox-code#69). SIGTERM, or Ctrl-C, can arrive while the + # server is initializing or migrating, or while the snapshot is being + # generated, all before the serve loop's own handler. It is a stop + # that was asked for, so it is not a traceback: the `finally` below + # stops the bundled server and restores the handler. Nothing was + # served, so the exit is the signal's conventional status (128 + its + # number) and not 0. + signum = (signal.SIGTERM if isinstance(interrupt, _Terminated) + else signal.SIGINT) + print(f"generate-and-open interrupted ({signum.name}) before it " + "served; its bundled PostgreSQL server stops with it", + file=sys.stderr) + return 128 + int(signum) finally: - server.stop() + # THE HANDLER FIRST, so a second SIGTERM during the stop takes the + # default action at once; the parent-death signal still stops the + # server if this process goes before `stop()` has finished. if previous is not None: signal.signal(signal.SIGTERM, previous) + server.stop() def _generate_and_open(args: argparse.Namespace, *, opener) -> int: diff --git a/src/opendox/runtime/bundle.py b/src/opendox/runtime/bundle.py index 701aa1bf..633dc3a3 100644 --- a/src/opendox/runtime/bundle.py +++ b/src/opendox/runtime/bundle.py @@ -29,7 +29,9 @@ THE LIFECYCLE, IN FULL: - * `initdb` once per data directory: the MIGRATION identity + * `initdb` once per data directory, into an attempt directory that is + renamed into place only when it has succeeded, so an interrupted first + start never leaves a half-built cluster: the MIGRATION identity (`config.BUNDLE_OWNER_ROLE`) is the bootstrap superuser, local connections are `trust` and host connections are `reject`, UTF-8 in the `C` locale. Trust is safe BECAUSE of the socket: its directory is 0700, owned by the @@ -60,9 +62,11 @@ import ctypes import importlib.util import os +import shutil import signal import subprocess import sys +import tempfile import time from pathlib import Path from typing import Any @@ -129,33 +133,84 @@ def server_binaries() -> Path: return binaries +def _lock_file_pid(bundle: DatabaseBundle) -> int | None: + """The pid on the first line of the server's `postmaster.pid`, or `None`.""" + try: + first = (bundle.data_dir / "postmaster.pid").read_text( + encoding="utf-8").splitlines()[0] + return int(first.strip()) + except (OSError, IndexError, ValueError): + return None + + +def _serves(pid: int, bundle: DatabaseBundle) -> bool | None: + """Whether `pid` is the postmaster of `bundle`'s data directory. + + Asked of the KERNEL, as the pair this module launches: an executable named + `postgres` whose working directory IS the data directory. The postmaster + changes into its data directory at startup, and neither of the two can be + rewritten by its process title, so a recycled pid given to anything else, + even another `postgres` serving another directory, is not it (Copilot + review of openDox-code#69). + + `False` also when the kernel will not say: another user's process cannot + be this bundle's server, because the server runs as the owner of a 0700 + data directory, the user this runs as. `None` only where there is no + `/proc` to ask at all. + """ + if not Path("/proc/self").exists(): + return None + try: + executable = os.readlink(f"/proc/{pid}/exe") + cwd = os.readlink(f"/proc/{pid}/cwd") + except OSError: + return False + if Path(executable.removesuffix(" (deleted)")).name != "postgres": + return False + try: + return Path(cwd).resolve() == bundle.data_dir.resolve() + except OSError: + return False + + def running_pid(bundle: DatabaseBundle) -> int | None: """The pid of a live server on `bundle`'s data directory, or `None`. Read from the server's own `postmaster.pid` (its first line), and only - believed while that process exists: a file left by a server that did not - stop cleanly names a pid that is gone, or that the kernel has since given - to something else, and PostgreSQL itself treats such a file as stale. + believed while that process exists and IS this bundle's server: a file + left by a server that did not stop cleanly names a pid that is gone, or + that the kernel has since given to something else (`_serves`). """ - try: - first = (bundle.data_dir / "postmaster.pid").read_text( - encoding="utf-8").splitlines()[0] - pid = int(first.strip()) - except (OSError, IndexError, ValueError): + pid = _lock_file_pid(bundle) + if pid is None: return None try: os.kill(pid, 0) - except ProcessLookupError: + except (ProcessLookupError, PermissionError): + # gone; or alive and another user's, which cannot be this server return None - except PermissionError: - return pid # alive, and not ours to signal - # A REUSED PID IS NOT A SERVER: where the kernel says what runs there, it - # must be a postgres, or the file is stale whatever its first line says. + return pid if _serves(pid, bundle) is not False else None + + +def _remove_a_proven_stale_lock(bundle: DatabaseBundle) -> None: + """Remove a `postmaster.pid` the kernel PROVES is not this server's. + + PostgreSQL removes a lock file whose pid is gone. It refuses to start, + however, over one whose pid the kernel has given to another live process + of the same user, and that refusal would last as long as the unrelated + process does. Where `/proc` shows that process is not this data + directory's postmaster, the lock is stale by proof and is removed. Where + nothing can be proven, it is left for PostgreSQL to judge. + """ + pid = _lock_file_pid(bundle) + if pid is None: + return try: - command = Path(f"/proc/{pid}/cmdline").read_bytes() - except OSError: - return pid # no /proc to ask; believe the live pid - return pid if b"postgres" in command else None + os.kill(pid, 0) + except (ProcessLookupError, PermissionError): + return # PostgreSQL's own rule covers both + if _serves(pid, bundle) is False: + (bundle.data_dir / "postmaster.pid").unlink(missing_ok=True) def report(bundle: DatabaseBundle) -> dict[str, Any]: @@ -245,25 +300,37 @@ def start(self) -> BundledServer: f"(pid {already}). One local install's database belongs to one " "entry point at a time: stop the other `generate-and-open " f"{LOCAL_FLAG}`, or give this one its own {PREFIX}STATE_DIR") - self._prepare_directories() - if not (self.bundle.data_dir / "PG_VERSION").is_file(): - self._initdb(binaries) - self._launch(binaries) + # THE WHOLE START IS ONE GUARDED OPERATION (Copilot review of + # openDox-code#69). The directories, `initdb` and the launch fail as + # plainly as the connection does: a timeout, a permission, a missing + # file. Each one comes out as the one named refusal the entry point + # prints, with whatever was started stopped, never as a traceback. + phase = "preparing its directories" try: + self._prepare_directories() + phase = "initializing its data directory" + self._initialize(binaries) + phase = "launching it" + _remove_a_proven_stale_lock(self.bundle) + self._launch(binaries) + phase = "waiting for it to accept a connection" self._wait_until_ready() + phase = "bootstrapping its database and served role" self._bootstrap() + phase = "migrating it" self._migrate() except (BundleRefused, migrations.MigrationError): self.stop() raise - except Exception as exc: # noqa: BLE001 - the driver's, named not quoted + except Exception as exc: # noqa: BLE001 - named, never quoted # THE DRIVER'S TEXT IS NOT REPEATED, for the reason `runtime/cli.py` - # gives: it quotes the connection string it was handed. The class - # names what went wrong and the server's own log says the rest. + # gives: it quotes the connection string it was handed. The phase + # and the class name say what went wrong, and the server's own log + # says the rest. self.stop() raise BundleRefused( - f"the bundled PostgreSQL server started but could not be " - f"prepared ({type(exc).__name__}); its log is " + f"the bundled PostgreSQL server could not be started " + f"({phase}: {type(exc).__name__}); its log is " f"{self.log_path}") from None except BaseException: self.stop() @@ -284,9 +351,69 @@ def _prepare_directories(self) -> None: self.bundle.socket_dir.mkdir(mode=0o700, parents=True, exist_ok=True) os.chmod(self.bundle.socket_dir, 0o700) - def _initdb(self, binaries: Path) -> None: + #: The prefix an initialization attempt's directory carries, beside the + #: data directory, followed by the pid of the process making it. + ATTEMPT_PREFIX = "data.initdb-" + + def _initialize(self, binaries: Path) -> None: + """A complete cluster at `data_dir`, or a refusal. Never a partial one. + + `initdb` runs into an ATTEMPT directory beside the data directory, and + the attempt is renamed into place only once `initdb` has succeeded. So + a data directory exists only as a finished cluster, and a first start + that dies midway (a kill, a full disk, a power cut) leaves an attempt + and not a half-built data directory that the next start would either + launch or fail to re-initialize for ever (Copilot review of + openDox-code#69). The next start removes an attempt whose process is + gone, and starts again. + + A data directory that exists and holds no cluster is NOT this + install's to remove, unless it is empty: it is refused, named, and + left as it is. + """ + data = self.bundle.data_dir + if (data / "PG_VERSION").is_file(): + return + if data.exists(): + try: + data.rmdir() # an empty directory holds nothing + except OSError: + raise BundleRefused( + f"{data} exists and holds no PostgreSQL cluster, so it is " + "not this install's database, and it is left untouched: " + "move it aside, or give this install its own " + f"{PREFIX}STATE_DIR") from None + self._remove_abandoned_attempts() + attempt = Path(tempfile.mkdtemp( + prefix=f"{self.ATTEMPT_PREFIX}{os.getpid()}-", dir=data.parent)) + try: + self._initdb(binaries, attempt) + try: + os.rename(attempt, data) + except OSError: + if not (data / "PG_VERSION").is_file(): + raise + # another start finished first; its cluster is the one used + shutil.rmtree(attempt, ignore_errors=True) + except BaseException: + shutil.rmtree(attempt, ignore_errors=True) + raise + + def _remove_abandoned_attempts(self) -> None: + """Every initialization attempt whose process no longer exists.""" + for candidate in self.bundle.data_dir.parent.glob( + f"{self.ATTEMPT_PREFIX}*"): + owner = candidate.name[len(self.ATTEMPT_PREFIX):].split("-", 1)[0] + try: + os.kill(int(owner), 0) + except ProcessLookupError: + shutil.rmtree(candidate, ignore_errors=True) + except (ValueError, OSError): + continue # alive, or not ours to judge + + def _initdb(self, binaries: Path, target: Path) -> None: done = subprocess.run( - [str(binaries / "initdb"), "-D", str(self.bundle.data_dir), + [str(binaries / "initdb"), "-D", str(target), "-U", BUNDLE_OWNER_ROLE, "--auth-local=trust", "--auth-host=reject", "--encoding=UTF8", "--locale=C", "--no-instructions"], env=_child_environment(), capture_output=True, text=True, diff --git a/src/opendox/runtime/config.py b/src/opendox/runtime/config.py index 229b3e75..f6257bbe 100644 --- a/src/opendox/runtime/config.py +++ b/src/opendox/runtime/config.py @@ -32,10 +32,12 @@ import re import shlex import sys +import tomllib import urllib.parse from collections.abc import Mapping from dataclasses import dataclass from pathlib import Path, PurePath +from typing import Any #: The environment prefix. One string, so a rename is one edit. PREFIX = "OPENDOX_" @@ -186,11 +188,13 @@ class Setting: ), Setting( PREFIX + "MIGRATIONS_DIR", "migrations", False, False, - "the ordered-SQL directory, repository-root-relative. Unset, it is " - "`migrations` where the working directory holds one (a checkout, or " - "the image's /app), and otherwise the copy the installed package " - "ships (plan 034 T072), so an install run from anywhere else still " - "has the migrations it applies", + "the ordered-SQL directory, repository-root-relative. Unset, a HOSTED " + "install uses `migrations` where the working directory holds one (a " + "checkout, or the image's /app), as it always has, and otherwise the " + "copy this installation carries; a LOCAL install uses ONLY the copy " + "this installation carries and never the working directory's (plan " + "034 T072), so launching it from a checkout of somebody else's " + "repository cannot run that repository's SQL", ), Setting( PREFIX + "PROJECT_REPOSITORY_ROOT", "var/projects", False, False, @@ -1579,13 +1583,21 @@ def _refuse_two_dsns_that_select_different_schemas( #: pre-existing service can stand in for it"), so an operator's DSN beside #: `local` is either about to be silently overridden or is another server #: trying to stand in for the bundled one. Neither is accepted. -HOSTED_ONLY_SETTINGS: tuple[str, ...] = ( +#: +#: TWO CLASSES, TWO REASONS (Copilot review of openDox-code#69): the broker +#: settings are refused because honouring `local` would drop an +#: authentication, and the DSNs because the local install supplies its own +#: database. Each refusal says its own reason. +BROKER_SETTINGS: tuple[str, ...] = ( PREFIX + "OIDC_ISSUER", PREFIX + "OIDC_AUDIENCE", PREFIX + "OIDC_JWKS_URL", +) +OPERATOR_DATABASE_SETTINGS: tuple[str, ...] = ( PREFIX + "DATABASE_URL", PREFIX + "MIGRATION_DATABASE_URL", ) +HOSTED_ONLY_SETTINGS: tuple[str, ...] = BROKER_SETTINGS + OPERATOR_DATABASE_SETTINGS #: THE BUNDLED SERVER'S IDENTITY (plan 034 T072; #1144 13.1, as T007 batch H's #: addendum reads). The data directory and the socket directory live under the @@ -1712,49 +1724,121 @@ def database_bundle(state: Path) -> DatabaseBundle: PACKAGED_MIGRATIONS = PurePath("share", "opendox", "migrations") -def packaged_migrations_dir() -> Path | None: - """The migrations this INSTALLATION carries, or `None` where it carries none. +def _source_tree_migrations(module_file: Path) -> Path | None: + """The `migrations/` of the SOURCE TREE `module_file` was imported from. - Two places, in order. The installed distribution's own data files — a - wheel install, whose `RECORD` lists them wherever the install scheme put - them. Then the source tree this module was imported from — an editable - install, which installs no data files, or `src/` on the path — whose - `migrations/` sits beside `src/`. + `module_file` is this module (`src/opendox/runtime/config.py`), so the + tree is three directories up: a checkout run with `src/` on the path, or + an editable install, neither of which installs data files. It counts only + where it really is that tree: `src/` is the directory the package sits + in, and the root's `pyproject.toml` names THIS project. A wheel's + `site-packages`, or a `--target` directory that happens to sit inside + some other checkout, is neither. """ + here = module_file.resolve() + package_parent, root = here.parents[2], here.parents[3] + if package_parent.name != "src": + return None try: - files = importlib.metadata.distribution("opendox").files or () - except importlib.metadata.PackageNotFoundError: - files = () + project = tomllib.loads( + (root / "pyproject.toml").read_text(encoding="utf-8")) + except (OSError, tomllib.TOMLDecodeError): + return None + if project.get("project", {}).get("name") != "opendox": + return None + candidate = root / "migrations" + return candidate if candidate.is_dir() else None + + +def _distribution_migrations(module_file: Path, + distribution: Any | None = None) -> Path | None: + """The migrations the INSTALLED DISTRIBUTION of `module_file` carries. + + From the distribution's own `RECORD`, wherever its install scheme put the + data files — and ONLY when that distribution is the one `module_file` was + loaded from. A name lookup alone is not: with a checkout's `src/` on the + path, `distribution("opendox")` can find an older wheel installed beside + it, and that wheel's migrations are another version's (Copilot review of + openDox-code#69). + """ + if distribution is None: + try: + distribution = importlib.metadata.distribution("opendox") + except importlib.metadata.PackageNotFoundError: + return None + files = list(distribution.files or ()) + here = module_file.resolve() + tail = here.parts[-3:] # ("opendox", "runtime", "config.py") + if not any(PurePath(entry).parts[-3:] == tail + and Path(entry.locate()).resolve() == here for entry in files): + return None for entry in files: parts = PurePath(entry).parts if (entry.name.endswith(".sql") and tuple(parts[-4:-1]) == PACKAGED_MIGRATIONS.parts): return Path(entry.locate()).resolve().parent - source = Path(__file__).resolve().parents[3] - if (source / "pyproject.toml").is_file() and (source / "migrations").is_dir(): - return source / "migrations" return None -def migrations_dir(env: Mapping[str, str] | None = None) -> Path: +def installation_migrations_dir() -> Path | None: + """The migrations THIS INSTALLATION carries, or `None` where it carries none. + + Tied to the code that is running, never to a name or to the working + directory. First the source tree this module was imported from (a + checkout, or an editable install). Then the installed distribution that + this module belongs to, from its `RECORD` (a wheel install). A real wheel + install has no `pyproject.toml` beside its package, so it falls through to + its own `RECORD` (Copilot review of openDox-code#69). + """ + module_file = Path(__file__) + return (_source_tree_migrations(module_file) + or _distribution_migrations(module_file)) + + +def migrations_dir(env: Mapping[str, str] | None = None, *, + local: bool = False) -> Path: """`OPENDOX_MIGRATIONS_DIR`, or where this install's migrations are. - Set, it is used as given, as it always was. Unset, it is `migrations` - wherever the working directory holds one — a checkout, or the image's - `/app` — which is today's default, unchanged; and OTHERWISE the copy the - installed package carries (`packaged_migrations_dir`), so `pip install - "opendox[local]"` run from a user's home directory migrates its bundled - server instead of refusing for a missing directory. Where neither exists - it is still `migrations`, and the canonical gate refuses it by name. + SET, it is used as given, in either shape: executing another directory's + SQL is something an operator says, not something a directory implies. + + UNSET, the shape decides. + * A LOCAL install uses ONLY the copy this installation carries + (`installation_migrations_dir`), never the working directory's. Its + entry point runs every migration it finds as the bundled server's + OWNER, and the canonical gate pins `0001` alone. So a `migrations/` + in whatever directory a user launches it from, holding the pinned + `0001` and SQL of its own, would otherwise be executed (Copilot + review of openDox-code#69). An installation that carries none is + refused, naming the setting. + * A HOSTED install keeps today's default, unchanged (13.6): `migrations` + where the working directory holds one (the image's `/app`, a + checkout), and otherwise the copy this installation carries. Where + neither exists, it is still `migrations`, and the canonical gate + refuses it by name. The compose file and the Kubernetes manifests + set the variable explicitly, so they read nothing implicit. """ env = os.environ if env is None else env - raw = env.get(PREFIX + "MIGRATIONS_DIR", "").strip() + setting = PREFIX + "MIGRATIONS_DIR" + raw = env.get(setting, "").strip() if raw: return Path(raw) + if local: + found = installation_migrations_dir() + if found is None: + raise ConfigurationError( + f"{setting} is unset, and this installation carries no " + "migrations of its own: a wheel install has them under " + f"`{PACKAGED_MIGRATIONS}` in its data directory, and a " + "checkout has `migrations/` beside `src/`. A LOCAL install " + "never reads the working directory's `migrations/`, because " + "its entry point runs them as the bundled server's owner. " + f"Reinstall the package, or name the directory in {setting}") + return found here = Path("migrations") if here.is_dir(): return here - return packaged_migrations_dir() or here + return installation_migrations_dir() or here def install_mode(env: Mapping[str, str] | None = None, *, @@ -1837,21 +1921,46 @@ def refuse_what_a_local_install_cannot_be(env: Mapping[str, str]) -> None: _optional(env, _by_name(PREFIX + "BIND_HOST")) or "127.0.0.1") +def _named(given: list[str]) -> str: + return f"{' and '.join(given)} {'are' if len(given) > 1 else 'is'} set" + + def _refuse_hosted_only_settings(env: Mapping[str, str]) -> None: - """Every setting in `HOSTED_ONLY_SETTINGS` given beside `local`, named.""" - given = [name for name in HOSTED_ONLY_SETTINGS - if env.get(name, "").strip()] - if given: - raise ConfigurationError( - f"{' and '.join(given)} {'are' if len(given) > 1 else 'is'} set, " - "and this is a LOCAL install, which has no broker and reads " - f"{'none of them' if len(given) > 1 else 'none'}. A broker " + """Every setting in `HOSTED_ONLY_SETTINGS` given beside `local`, named. + + Each CLASS carries its own reason (Copilot review of openDox-code#69). + A broker setting says a hosted install was meant, and a DSN says another + database was meant. A local run with only `OPENDOX_DATABASE_URL` is told + about its database, not about an authentication it never configured. + The values are never repeated: a DSN carries a password. + """ + def given(names: tuple[str, ...]) -> list[str]: + return [name for name in names if env.get(name, "").strip()] + + brokers, databases = given(BROKER_SETTINGS), given(OPERATOR_DATABASE_SETTINGS) + if not (brokers or databases): + return + reasons = [] + if brokers: + reasons.append( + f"{_named(brokers)}, and a LOCAL install has no broker and reads " + f"{'none of them' if len(brokers) > 1 else 'none'}. A broker " "setting beside the local mode says a HOSTED install was meant, " "and honouring `local` over it would silently drop that " - f"authentication: unset {'them' if len(given) > 1 else 'it'} for " - f"a local install, or drop {LOCAL_FLAG} / " - f"{PREFIX}INSTALL_MODE=local for a hosted one (the values are not " - "repeated here)") + "authentication") + if databases: + reasons.append( + f"{_named(databases)}, and a LOCAL install supplies BOTH of its " + "DSNs itself, from the PostgreSQL server it bundles under " + f"{PREFIX}STATE_DIR (13.1). An operator's DSN beside the local " + "mode would either be silently overridden or be another server " + "standing in for the bundled one, and neither is accepted") + every = brokers + databases + raise ConfigurationError( + "; and ".join(reasons) + f". Unset {'them' if len(every) > 1 else 'it'} " + f"for a local install, or drop {LOCAL_FLAG} / " + f"{PREFIX}INSTALL_MODE=local for a hosted one (the values are not " + "repeated here)") def require_the_hosted_issuer(env: Mapping[str, str] | None = None) -> None: @@ -1998,7 +2107,7 @@ def load_settings(env: Mapping[str, str] | None = None, *, served_schema=_served_schema(env), served_database=_served_database(env) or (BUNDLE_DATABASE if local else None), publish_openapi=_boolean(env, _by_name(PREFIX + "PUBLISH_OPENAPI")), - migrations_dir=migrations_dir(env), + migrations_dir=migrations_dir(env, local=local), project_repository_root=Path( _optional(env, _by_name(PREFIX + "PROJECT_REPOSITORY_ROOT")) or "var/projects" ), @@ -2081,7 +2190,7 @@ def load_migration_settings(env: Mapping[str, str] | None = None) -> RuntimeSett served_schema=_served_schema(env), served_database=_served_database(env) or (BUNDLE_DATABASE if local else None), publish_openapi=False, - migrations_dir=migrations_dir(env), + migrations_dir=migrations_dir(env, local=local), project_repository_root=Path( _optional(env, _by_name(PREFIX + "PROJECT_REPOSITORY_ROOT")) or "var/projects"), diff --git a/tests_runtime/test_bundled_postgres.py b/tests_runtime/test_bundled_postgres.py index 895bcfb9..acf6e369 100644 --- a/tests_runtime/test_bundled_postgres.py +++ b/tests_runtime/test_bundled_postgres.py @@ -33,6 +33,7 @@ import json import os import re +import selectors import shutil import signal import subprocess @@ -141,6 +142,33 @@ def _wait_gone(pid: int, seconds: float = 30.0) -> bool: return False +def _first_url(child: subprocess.Popen, seconds: float) -> str | None: + """The first `http://` line `child` prints within `seconds`, or `None`. + + BOUNDED BY THE DEADLINE, not by the child: a blocking `readline()` waits + for as long as a child that is alive and silent stays so, which is exactly + the startup failure this has to diagnose (Copilot review of + openDox-code#69). So the pipe is polled with a selector, only up to the + time left, and read in whatever pieces arrive. + """ + deadline = time.monotonic() + seconds + pending = b"" + with selectors.DefaultSelector() as selector: + selector.register(child.stdout, selectors.EVENT_READ) + while (remaining := deadline - time.monotonic()) > 0: + if not selector.select(timeout=remaining): + return None # the deadline + chunk = os.read(child.stdout.fileno(), 65536) + if not chunk: + return None # the child closed it + pending += chunk + *lines, pending = pending.split(b"\n") + for line in lines: + if line.startswith(b"http://"): + return line.strip().decode() + return None + + def _launch(corpus: Path, state: Path, run_dir: Path) -> tuple[subprocess.Popen, str]: """`generate-and-open --local` in the BACKGROUND, and the URL it serves.""" child = subprocess.Popen( @@ -149,22 +177,40 @@ def _launch(corpus: Path, state: Path, run_dir: Path) -> tuple[subprocess.Popen, "--run-dir", str(run_dir), "--no-open", "--no-validate", "--port", "0"], env=_clean_env(**{STATE: str(state)}), cwd=ROOT, - stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) - deadline = time.monotonic() + 90 - url = None - while time.monotonic() < deadline and url is None: - line = child.stdout.readline() - if not line: - break - if line.startswith("http://"): - url = line.strip() + stdout=subprocess.PIPE, stderr=subprocess.PIPE) + url = _first_url(child, 90) if url is None: child.kill() - out, err = child.communicate(timeout=30) - raise AssertionError(f"the entry point never served: {err[-2000:]}") + _out, err = child.communicate(timeout=30) + raise AssertionError( + f"the entry point never served: {err.decode(errors='replace')[-2000:]}") return child, url +def test_the_launch_helper_is_bounded_by_its_deadline_not_by_the_child() -> None: + """A child that is alive and silent does not hold the helper past its + deadline; one that prints the URL is read, however the pieces arrive.""" + silent = subprocess.Popen([sys.executable, "-c", "import time; time.sleep(60)"], + stdout=subprocess.PIPE) + try: + began = time.monotonic() + assert _first_url(silent, 1.0) is None + assert time.monotonic() - began < 10 + finally: + silent.kill() + silent.communicate(timeout=10) + talker = subprocess.Popen( + [sys.executable, "-c", "import sys, time; sys.stdout.write(' serving '); " + "sys.stdout.flush(); time.sleep(0.2); print('x'); " + "print('http://127.0.0.1:1/index.html', flush=True); time.sleep(60)"], + stdout=subprocess.PIPE) + try: + assert _first_url(talker, 30) == "http://127.0.0.1:1/index.html" + finally: + talker.kill() + talker.communicate(timeout=10) + + def _status(state: Path) -> tuple[int, dict]: """`OPENDOX_INSTALL_MODE=local opendox-runtime runtime status`, a SECOND process.""" done = subprocess.run( @@ -325,6 +371,34 @@ def test_a_second_entry_point_on_the_same_state_dir_is_refused( again.stop() +@pytest.mark.skipif(not Path("/proc/self").exists(), reason="asks /proc") +def test_a_stale_lock_naming_a_recycled_pid_does_not_hold_the_bundle( + state_dir: Path) -> None: + """A server that did not stop cleanly leaves its `postmaster.pid`, and the + kernel gives its pid to something else, here a process with `postgres` in + its argv. `status` does not report that process, and the next start + starts, with the proven-stale lock removed (Copilot review of + openDox-code#69).""" + settings = config.load_settings({MODE: "local", STATE: str(state_dir)}) + bundle_mod.BundledServer(settings).start().stop() # a real cluster + bundle = config.DatabaseBundle(state_dir) + decoy = subprocess.Popen( + [sys.executable, "-c", "import time; time.sleep(60)", "postgres"], + cwd=bundle.data_dir, stdout=subprocess.DEVNULL) + try: + (bundle.data_dir / "postmaster.pid").write_text( + f"{decoy.pid}\n{bundle.data_dir}\n", encoding="utf-8") + assert bundle_mod.report(bundle)["pid"] is None + server = bundle_mod.BundledServer(settings).start() + try: + assert bundle_mod.running_pid(bundle) == server.report()["pid"] + finally: + server.stop() + finally: + decoy.kill() + decoy.wait(timeout=10) + + def test_migrate_under_the_local_mode_uses_the_bundle_and_refuses_a_dsn( state_dir: Path) -> None: """`runtime migrate` is part of the same install: it reads the bundle's diff --git a/tests_runtime/test_local_lifecycle.py b/tests_runtime/test_local_lifecycle.py new file mode 100644 index 00000000..334deb49 --- /dev/null +++ b/tests_runtime/test_local_lifecycle.py @@ -0,0 +1,489 @@ +"""The LOCAL install's lifecycle, hardened (plan 034 T072; Copilot review of +openDox-code#69). + +Hermetic: nothing here starts a real PostgreSQL server. The bundle's binaries +are stood in by small scripts where a phase must fail, and the entry point's +bundle is stood in where the case is the entry point's own handling. The real +server's cases are `test_bundled_postgres.py`'s. + + * which migrations a local install runs: this installation's own, never the + working directory's, and never another installation's found by name; + * a stale `postmaster.pid` is believed only for this data directory's + postmaster, asked of the kernel; + * `initdb` never leaves a half-built data directory; + * every phase of a start fails as the one named refusal, never a traceback; + * an interrupt anywhere in the local lifecycle is a clean stop. +""" + +from __future__ import annotations + +import os +import shutil +import signal +import subprocess +import sys +import tempfile +import time +from pathlib import Path + +import pytest + +from opendox import cli as cli_mod +from opendox.runtime import bundle as bundle_mod +from opendox.runtime import config +from opendox.runtime.config import PREFIX + +ROOT = Path(__file__).resolve().parents[1] +MODE = PREFIX + "INSTALL_MODE" +STATE = PREFIX + "STATE_DIR" +MIGRATIONS = PREFIX + "MIGRATIONS_DIR" + + +@pytest.fixture() +def scrubbed(monkeypatch: pytest.MonkeyPatch) -> pytest.MonkeyPatch: + for name in config.SETTING_NAMES: + monkeypatch.delenv(name, raising=False) + return monkeypatch + + +@pytest.fixture() +def short_state(): + """A state directory with a SHORT path (the socket's path is bounded).""" + path = Path(tempfile.mkdtemp(prefix="odx-l-", dir="/tmp")) + yield path + for directory in path.rglob("*"): + if directory.is_dir(): + directory.chmod(0o700) + shutil.rmtree(path, ignore_errors=True) + + +def _local(state: Path) -> config.RuntimeSettings: + return config.load_settings({MODE: "local", STATE: str(state)}) + + +# -- which migrations a local install runs ------------------------------------ + + +@pytest.fixture() +def hostile_cwd(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Path: + """A working directory whose `migrations/` holds the pinned `0001` AND SQL + of its own: what a checkout of somebody else's repository can hold.""" + here = tmp_path / "somebody-elses-checkout" + (here / "migrations").mkdir(parents=True) + for sql in (ROOT / "migrations").glob("0001*.sql"): + shutil.copy2(sql, here / "migrations" / sql.name) + (here / "migrations" / "0099_not_this_products.sql").write_text( + "create table stolen (secret text);\n", encoding="utf-8") + monkeypatch.chdir(here) + return here + + +def test_a_local_install_never_runs_the_working_directorys_migrations( + hostile_cwd: Path, short_state: Path) -> None: + settings = _local(short_state) + found = Path(settings.migrations_dir).resolve() + assert not found.is_relative_to(hostile_cwd.resolve()), found + assert found == (ROOT / "migrations").resolve(), found + migrate = config.load_migration_settings({MODE: "local", STATE: str(short_state)}) + assert Path(migrate.migrations_dir).resolve() == found + + +def test_a_hosted_install_keeps_its_working_directory_default( + hostile_cwd: Path) -> None: + """13.6: the hosted default is unchanged, and the deploy files set the + variable explicitly.""" + assert config.migrations_dir({}) == Path("migrations") + + +def test_an_explicit_migrations_dir_is_used_as_given_in_either_shape( + tmp_path: Path) -> None: + chosen = str(tmp_path / "chosen") + assert config.migrations_dir({MIGRATIONS: chosen}, local=True) == Path(chosen) + assert config.migrations_dir({MIGRATIONS: chosen}) == Path(chosen) + + +def test_a_local_installation_that_carries_none_is_refused_naming_the_setting( + monkeypatch: pytest.MonkeyPatch, hostile_cwd: Path) -> None: + monkeypatch.setattr(config, "installation_migrations_dir", lambda: None) + with pytest.raises(config.ConfigurationError) as caught: + config.migrations_dir({}, local=True) + assert MIGRATIONS in str(caught.value), caught.value + + +class _FakeFile: + """One `importlib.metadata` RECORD entry, located under `base`.""" + + def __init__(self, base: Path, relative: str) -> None: + self.base, self.relative = base, relative + self.name = Path(relative).name + self.parts = Path(relative).parts + + def __fspath__(self) -> str: + return self.relative + + def locate(self) -> Path: + return self.base / self.relative + + +class _FakeDistribution: + def __init__(self, base: Path) -> None: + site = "lib/python3/site-packages" + self.files = [ + _FakeFile(base, f"{site}/opendox/runtime/config.py"), + _FakeFile(base, "share/opendox/migrations/0001_older.sql"), + ] + + +def test_another_installations_record_is_not_this_ones(tmp_path: Path) -> None: + """A distribution found by NAME whose RECORD does not hold the running + module (an older wheel beside a checkout) is not this installation.""" + older = _FakeDistribution(tmp_path / "older-wheel") + assert config._distribution_migrations(Path(config.__file__), older) is None + + +def test_the_record_of_the_distribution_that_holds_the_module_is_used( + tmp_path: Path) -> None: + base = tmp_path / "wheel" + dist = _FakeDistribution(base) + module = dist.files[0].locate() + module.parent.mkdir(parents=True) + module.write_text("# stand-in\n", encoding="utf-8") + assert config._distribution_migrations(module, dist) == \ + (base / "share/opendox/migrations").resolve() + + +def test_the_source_tree_wins_over_a_wheel_found_by_name( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + """A checkout run with `src/` on the path, beside an older wheel: the + checkout's own migrations, not the wheel's (Copilot review of + openDox-code#69).""" + older = _FakeDistribution(tmp_path / "older-wheel") + monkeypatch.setattr(config.importlib.metadata, "distribution", + lambda name: older) + assert config.installation_migrations_dir() == ROOT / "migrations" + + +def test_the_source_tree_is_asked_before_any_record( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + """The documented ORDER, pinned: even a RECORD that does hold the running + module is asked only after the source tree it was imported from.""" + class _Holding: + files = [_FakeFile(ROOT / "src", "opendox/runtime/config.py"), + _FakeFile(tmp_path, "share/opendox/migrations/0001_other.sql")] + + assert config._distribution_migrations(Path(config.__file__), _Holding()) == \ + (tmp_path / "share/opendox/migrations").resolve() + monkeypatch.setattr(config.importlib.metadata, "distribution", + lambda name: _Holding()) + assert config.installation_migrations_dir() == ROOT / "migrations" + + +def test_a_package_outside_src_is_not_a_source_tree(tmp_path: Path) -> None: + """A `--target` install inside some other checkout: its parent has a + `pyproject.toml` and a `migrations/`, and it is still not this tree.""" + target = tmp_path / "checkout" / "vendor" / "opendox" / "runtime" + target.mkdir(parents=True) + (tmp_path / "checkout" / "pyproject.toml").write_text( + '[project]\nname = "opendox"\n', encoding="utf-8") + (tmp_path / "checkout" / "migrations").mkdir() + assert config._source_tree_migrations(target / "config.py") is None + + +# -- a stale postmaster.pid --------------------------------------------------- + + +def _decoy(executable: str, cwd: Path, *argv: str) -> subprocess.Popen: + return subprocess.Popen([executable, *argv], cwd=cwd, + stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + + +def _lock(bundle: config.DatabaseBundle, pid: int) -> None: + bundle.data_dir.mkdir(parents=True, exist_ok=True) + (bundle.data_dir / "postmaster.pid").write_text( + f"{pid}\n{bundle.data_dir}\n", encoding="utf-8") + + +@pytest.mark.skipif(not Path("/proc/self").exists(), reason="asks /proc") +def test_a_recycled_pid_is_not_believed_even_with_postgres_in_its_argv( + short_state: Path) -> None: + """Even IN the data directory, and with `postgres -D ` in its + argv: the executable is not `postgres`, so it is not this server.""" + bundle = config.DatabaseBundle(short_state) + bundle.data_dir.mkdir(parents=True) + decoy = _decoy(sys.executable, bundle.data_dir, "-c", + "import time; time.sleep(60)", "postgres", "-D", + str(bundle.data_dir)) + try: + _lock(bundle, decoy.pid) + assert bundle_mod.running_pid(bundle) is None + assert bundle_mod.report(bundle)["pid"] is None + # and the start is not refused over it: the stale lock is removed + bundle_mod._remove_a_proven_stale_lock(bundle) + assert not (bundle.data_dir / "postmaster.pid").exists() + finally: + decoy.kill() + decoy.wait(timeout=10) + + +@pytest.mark.skipif(not Path("/proc/self").exists(), reason="asks /proc") +def test_a_postgres_serving_another_directory_is_not_this_server( + short_state: Path, tmp_path: Path) -> None: + """The exact pair: an executable NAMED `postgres` is not enough, it must + be running in THIS data directory. And the positive control: the same + decoy, run in the data directory, is believed.""" + sleeper = shutil.which("sleep") + assert sleeper + named = tmp_path / "bin" / "postgres" + named.parent.mkdir() + shutil.copy2(sleeper, named) + bundle = config.DatabaseBundle(short_state) + bundle.data_dir.mkdir(parents=True) + elsewhere = tmp_path / "another-data-dir" + elsewhere.mkdir() + for cwd, believed in ((elsewhere, False), (bundle.data_dir, True)): + decoy = _decoy(str(named), cwd, "60") + try: + _lock(bundle, decoy.pid) + deadline = time.monotonic() + 10 + while (Path(os.readlink(f"/proc/{decoy.pid}/exe")).name != "postgres" + and time.monotonic() < deadline): + time.sleep(0.02) # the exec is complete + got = bundle_mod.running_pid(bundle) + assert got == (decoy.pid if believed else None), (cwd, got) + finally: + decoy.kill() + decoy.wait(timeout=10) + + +def test_another_users_process_is_not_this_server(short_state: Path) -> None: + """A pid the kernel will not let this user signal belongs to another user, + and this bundle's server runs as the owner of its 0700 data directory. + Pid 1 is another user's on an ordinary host and in CI. Where the suite + runs as pid 1's owner, the pid is still not a postgres in this directory. + Either way it is not believed.""" + bundle = config.DatabaseBundle(short_state) + _lock(bundle, 1) + assert bundle_mod.running_pid(bundle) is None + + +# -- the two refusal classes, each with its own reason -------------------------- + + +def test_a_dsn_beside_local_is_refused_for_the_database_not_a_broker() -> None: + with pytest.raises(config.ConfigurationError) as caught: + config.load_settings({MODE: "local", + PREFIX + "DATABASE_URL": "postgresql://s@h.invalid/x"}) + message = str(caught.value) + assert PREFIX + "DATABASE_URL" in message and "bundles" in message, message + assert "broker" not in message and "authentication" not in message, message + + +def test_a_broker_setting_beside_local_is_refused_for_the_broker() -> None: + with pytest.raises(config.ConfigurationError) as caught: + config.load_settings({MODE: "local", + PREFIX + "OIDC_AUDIENCE": "fixture"}) + message = str(caught.value) + assert PREFIX + "OIDC_AUDIENCE" in message, message + assert "broker" in message and "authentication" in message, message + assert "bundles" not in message, message + + +def test_both_classes_together_name_both_reasons() -> None: + with pytest.raises(config.ConfigurationError) as caught: + config.load_settings({ + MODE: "local", PREFIX + "OIDC_ISSUER": "https://i.invalid/r/x", + PREFIX + "MIGRATION_DATABASE_URL": "postgresql://m:hunter2@h.invalid/x"}) + message = str(caught.value) + for fragment in (PREFIX + "OIDC_ISSUER", PREFIX + "MIGRATION_DATABASE_URL", + "authentication", "bundles"): + assert fragment in message, (fragment, message) + assert "hunter2" not in message + + +# -- initdb, and every phase of a start --------------------------------------- + + +def _binaries(tmp_path: Path, *, initdb: str, postgres: str = "exit 0") -> Path: + """Stand-in `initdb` and `postgres` scripts, executable.""" + directory = tmp_path / "fake-bin" + directory.mkdir() + for name, body in (("initdb", initdb), ("postgres", postgres)): + script = directory / name + script.write_text(body if body.startswith("#!") else + f"#!/bin/sh\n{body}\n", encoding="utf-8") + script.chmod(0o755) + return directory + + +def _server(monkeypatch, tmp_path: Path, state: Path, **scripts) -> bundle_mod.BundledServer: + binaries = _binaries(tmp_path, **scripts) + monkeypatch.setattr(bundle_mod, "server_binaries", lambda: binaries) + return bundle_mod.BundledServer(_local(state)) + + +def _target_of_initdb() -> str: + """Shell: the directory `initdb -D ` was told to initialize.""" + return 'while [ "$1" != "-D" ]; do shift; done; T="$2"' + + +def test_an_initdb_that_dies_midway_leaves_no_data_directory( + monkeypatch, tmp_path: Path, short_state: Path) -> None: + """The half-built cluster: `PG_VERSION` written, then the run fails. The + data directory must not exist afterwards, so the next start initializes + again instead of launching the remains (Copilot review of #69).""" + server = _server(monkeypatch, tmp_path, short_state, initdb=( + f'{_target_of_initdb()}\necho 16 > "$T/PG_VERSION"\n' + 'mkdir -p "$T/base"\necho "initdb: killed" >&2\nexit 1')) + with pytest.raises(bundle_mod.BundleRefused) as caught: + server.start() + assert "initdb" in str(caught.value) + data = server.bundle.data_dir + assert not data.exists(), sorted(p.name for p in data.iterdir()) + leftovers = list(data.parent.glob(f"{bundle_mod.BundledServer.ATTEMPT_PREFIX}*")) + assert leftovers == [], leftovers + + +def test_an_abandoned_attempt_is_removed_and_a_live_ones_is_kept( + monkeypatch, tmp_path: Path, short_state: Path) -> None: + server = _server(monkeypatch, tmp_path, short_state, initdb="exit 1") + parent = server.bundle.data_dir.parent + parent.mkdir(parents=True) + dead = subprocess.Popen(["true"]) + dead.wait(timeout=10) + abandoned = parent / f"{server.ATTEMPT_PREFIX}{dead.pid}-x" + live = parent / f"{server.ATTEMPT_PREFIX}{os.getpid()}-y" + for attempt in (abandoned, live): + attempt.mkdir() + (attempt / "PG_VERSION").write_text("16\n", encoding="utf-8") + with pytest.raises(bundle_mod.BundleRefused): + server.start() + assert not abandoned.exists() + assert live.exists(), "a concurrent start's attempt was removed" + + +def test_a_data_directory_that_is_not_a_cluster_is_refused_and_left_alone( + monkeypatch, tmp_path: Path, short_state: Path) -> None: + server = _server(monkeypatch, tmp_path, short_state, initdb="exit 0") + data = server.bundle.data_dir + data.mkdir(parents=True) + (data / "somebody-elses-file").write_text("keep me\n", encoding="utf-8") + with pytest.raises(bundle_mod.BundleRefused) as caught: + server.start() + assert str(data) in str(caught.value) and STATE in str(caught.value) + assert (data / "somebody-elses-file").read_text(encoding="utf-8") == "keep me\n" + + +def test_an_initdb_that_hangs_is_the_named_refusal_not_a_traceback( + monkeypatch, tmp_path: Path, short_state: Path) -> None: + monkeypatch.setattr(bundle_mod, "START_TIMEOUT_SECONDS", 0.5) + server = _server(monkeypatch, tmp_path, short_state, initdb="exec sleep 30") + with pytest.raises(bundle_mod.BundleRefused) as caught: + server.start() + assert "initializing its data directory: TimeoutExpired" in str(caught.value) + assert not server.bundle.data_dir.exists() + + +def test_a_launch_that_cannot_exec_is_the_named_refusal( + monkeypatch, tmp_path: Path, short_state: Path) -> None: + server = _server(monkeypatch, tmp_path, short_state, + initdb=f'{_target_of_initdb()}\necho 16 > "$T/PG_VERSION"', + postgres="#!/nonexistent/interpreter\n") + with pytest.raises(bundle_mod.BundleRefused) as caught: + server.start() + assert "launching it: FileNotFoundError" in str(caught.value), caught.value + assert server.process is None + + +def test_directories_it_cannot_make_are_the_named_refusal( + monkeypatch, tmp_path: Path, short_state: Path) -> None: + if hasattr(os, "geteuid") and os.geteuid() == 0: # pragma: no cover + pytest.skip("root ignores directory modes") + locked = short_state / "locked" + locked.mkdir(mode=0o500) + binaries = _binaries(tmp_path, initdb="exit 0") + monkeypatch.setattr(bundle_mod, "server_binaries", lambda: binaries) + server = bundle_mod.BundledServer(_local(locked / "state")) + with pytest.raises(bundle_mod.BundleRefused) as caught: + server.start() + assert "preparing its directories: PermissionError" in str(caught.value) + + +# -- an interrupt anywhere in the local lifecycle ------------------------------- + + +@pytest.fixture() +def local_entry(scrubbed, monkeypatch, tmp_path: Path): + """`cmd_generate_and_open --local` with its bundle stood in. The corpus + checks are not under test here and pass whatever the root.""" + scrubbed.setenv(MODE, "local") + scrubbed.setenv(STATE, "/tmp/odx-never-created") + monkeypatch.setattr(cli_mod, "_refuse_non_corpus_repo_root", lambda args: None) + monkeypatch.setattr(cli_mod, "_refuse_malformed_generated_at", lambda args: None) + stopped: list[int] = [] + behaviour: dict = {} + + class _StandIn: + applied: list = [] + + def __init__(self, settings) -> None: + pass + + def start(self): + behaviour.get("start", lambda: None)() + return self + + def stop(self) -> None: + stopped.append(1) + + def report(self) -> dict: + return {"data_dir": None, "socket_dir": "(stood in)", "pid": None} + + monkeypatch.setattr(cli_mod.bundle_mod, "BundledServer", _StandIn) + args = cli_mod.build_parser().parse_args( + ["generate-and-open", config.LOCAL_FLAG, "--repo-root", str(tmp_path), + "--repository", "fixture", "--no-open", "--no-serve"]) + return args, behaviour, stopped + + +def _entry(args) -> int: + """The entry point, with an ESCAPING interrupt turned into a failure: one + that reached pytest would abort the whole session, not fail this case.""" + try: + return cli_mod.cmd_generate_and_open(args, opener=lambda url: None) + except KeyboardInterrupt as escaped: + raise AssertionError( + f"the interrupt escaped the entry point ({type(escaped).__name__}), " + "which is a traceback and not a clean stop") from None + + +def _sigterm_to_self() -> None: + os.kill(os.getpid(), signal.SIGTERM) + for _ in range(500): # the handler runs between two bytecodes + time.sleep(0.01) + raise AssertionError("SIGTERM was not delivered") # pragma: no cover + + +def test_a_sigterm_while_the_bundle_starts_is_a_clean_stop( + local_entry, capsys) -> None: + args, behaviour, stopped = local_entry + before = signal.getsignal(signal.SIGTERM) + behaviour["start"] = _sigterm_to_self + rc = _entry(args) + assert rc == 128 + signal.SIGTERM + assert "interrupted (SIGTERM)" in capsys.readouterr().err + assert stopped == [1], "the bundle was not stopped" + assert signal.getsignal(signal.SIGTERM) == before, "the handler was not restored" + + +def test_a_ctrl_c_while_the_snapshot_is_generated_is_a_clean_stop( + local_entry, monkeypatch, capsys) -> None: + args, _behaviour, stopped = local_entry + + def _interrupted(args, *, opener): + raise KeyboardInterrupt + + monkeypatch.setattr(cli_mod, "_generate_and_open", _interrupted) + rc = _entry(args) + assert rc == 128 + signal.SIGINT + assert "interrupted (SIGINT)" in capsys.readouterr().err + assert stopped == [1] From 96b2699fd511b53b0a4b8fd89c200ac35c711490 Mon Sep 17 00:00:00 2001 From: Brett Heap <1513478+brettheap@users.noreply.github.com> Date: Wed, 30 Sep 2026 15:48:41 +0000 Subject: [PATCH 13/88] Fix round: an unprovable pid is not believed, and the state dir refuses by name (Copilot review) Copilot's review at ac61596a opened two more threads, both real. - r4139938402: the old fallback believed any pid it could not inspect. So a process that exited between the signal check and the /proc read, or any pid on a platform without /proc, counted as the server. Round 4 already treated a vanished process as gone on the /proc path. This round makes the rule total. `running_pid` believes a pid only when the kernel proves it is this data directory's postmaster. Where nothing can be asked (no /proc: macOS, the BSDs), it believes nothing, and this module does not refuse a start over it. PostgreSQL's own interlocks, the lock file's live-pid check and the shared-memory check, still refuse a second postmaster, so this never yields two servers, and never a refusal over a process that is not one. A lock that cannot be proven stale is left for PostgreSQL to judge. The price on such a platform is a `status` with no pid. That is recorded, not hidden: the standard library has no portable way to ask, and a third-party module here would be an undeclared runtime dependency (test_consumer_reach). Measured before: round 4's source with no /proc, and a python decoy in the data dir, reported the decoy's pid. ac61596's source reported a pid that had already exited. - r4139938444: `Path.expanduser()` raises RuntimeError for an unknown `~user`, and `Path.home()` does the same where there is no home. Both now refuse by name, as ConfigurationError naming OPENDOX_STATE_DIR. A hosted install still never reads the setting and is not refused over it (13.6). Five new cases fail against 28bdccd's source and pass here. Five mutants of the fixes are killed. Full suite: 2584 selected, 2573 passed, 11 skipped, 0 failed. Lane: openxfactory-4 (openXfactory-4-openDox_extraction) Arc: neutral-product-standalone-operability Co-Authored-By: Claude Opus 5.5 (1M context) --- src/opendox/runtime/bundle.py | 68 +++++++++++++++++------ src/opendox/runtime/config.py | 24 ++++++-- tests_runtime/test_local_lifecycle.py | 80 +++++++++++++++++++++++++-- 3 files changed, 144 insertions(+), 28 deletions(-) diff --git a/src/opendox/runtime/bundle.py b/src/opendox/runtime/bundle.py index 633dc3a3..9014332e 100644 --- a/src/opendox/runtime/bundle.py +++ b/src/opendox/runtime/bundle.py @@ -143,29 +143,54 @@ def _lock_file_pid(bundle: DatabaseBundle) -> int | None: return None +#: Where the kernel answers what a pid is, on Linux. A module constant so a +#: case can take it away and exercise a platform without it. +PROC = Path("/proc") + + +def _identity(pid: int) -> tuple[str, str] | None: + """`(executable, working directory)` of `pid`, from the kernel's `/proc`. + + `None` where the process is gone or is not this user's to inspect. A + process that exits between the lock file's read and this one is GONE, + never proof of anything (Copilot review of openDox-code#69). Raises + `LookupError` where there is no `/proc` at all (macOS, the BSDs): the + standard library has no portable way to ask, and `running_pid` then + believes nothing it cannot prove. + """ + if not PROC.joinpath("self").exists(): + raise LookupError("no /proc to ask") + try: + return (os.readlink(PROC / str(pid) / "exe"), + os.readlink(PROC / str(pid) / "cwd")) + except OSError: # gone, or another user's: not inspectable + return None + + def _serves(pid: int, bundle: DatabaseBundle) -> bool | None: """Whether `pid` is the postmaster of `bundle`'s data directory. - Asked of the KERNEL, as the pair this module launches: an executable named - `postgres` whose working directory IS the data directory. The postmaster - changes into its data directory at startup, and neither of the two can be - rewritten by its process title, so a recycled pid given to anything else, - even another `postgres` serving another directory, is not it (Copilot - review of openDox-code#69). - - `False` also when the kernel will not say: another user's process cannot - be this bundle's server, because the server runs as the owner of a 0700 - data directory, the user this runs as. `None` only where there is no - `/proc` to ask at all. + Asked of the platform, as the pair this module launches: an executable + named `postgres` whose working directory IS the data directory. The + postmaster changes into its data directory at startup, and neither of the + two can be rewritten by its process title. So a recycled pid given to + anything else is not it, even another `postgres` serving another directory + (Copilot review of openDox-code#69). + + `False` also for a process that is gone, or that the platform will not + describe: another user's process cannot be this bundle's server, because + the server runs as the owner of a 0700 data directory, which is the user + this runs as. `None` only where nothing can be asked at all. """ - if not Path("/proc/self").exists(): - return None try: - executable = os.readlink(f"/proc/{pid}/exe") - cwd = os.readlink(f"/proc/{pid}/cwd") - except OSError: + identity = _identity(pid) + except LookupError: + return None + if identity is None: return False - if Path(executable.removesuffix(" (deleted)")).name != "postgres": + executable, cwd = identity + if Path(executable.removesuffix(" (deleted)")).name not in {"postgres", + "postgres.exe"}: return False try: return Path(cwd).resolve() == bundle.data_dir.resolve() @@ -189,7 +214,14 @@ def running_pid(bundle: DatabaseBundle) -> int | None: except (ProcessLookupError, PermissionError): # gone; or alive and another user's, which cannot be this server return None - return pid if _serves(pid, bundle) is not False else None + # BELIEVED ONLY WHEN PROVEN. Where nothing can say what the pid is (no + # `/proc`), it is not reported as this server, and this module does not + # refuse a start over it. PostgreSQL's own interlocks, the lock file's + # live-pid check and the shared-memory check, still refuse a second + # postmaster on one data directory. So an unverifiable pid never yields + # two servers, and never a refusal over a process that is not one. The + # price on such a platform is a `status` that reports no pid. + return pid if _serves(pid, bundle) is True else None def _remove_a_proven_stale_lock(bundle: DatabaseBundle) -> None: diff --git a/src/opendox/runtime/config.py b/src/opendox/runtime/config.py index ce84233b..c9d7b73b 100644 --- a/src/opendox/runtime/config.py +++ b/src/opendox/runtime/config.py @@ -1696,7 +1696,16 @@ def state_dir(env: Mapping[str, str] | None = None) -> Path: setting = PREFIX + "STATE_DIR" raw = env.get(setting, "").strip() if raw: - path = Path(raw).expanduser() + try: + path = Path(raw).expanduser() + except RuntimeError: + # `~nosuchuser/...`: `expanduser` raises rather than answering, and + # a setting is refused by name, never by a traceback (Copilot + # review of openDox-code#69). + raise ConfigurationError( + f"{setting} is {raw!r}, whose `~` names no user this system " + "knows, so it expands to no directory. Name the state " + "directory absolutely") from None if not path.is_absolute(): raise ConfigurationError( f"{setting} is {raw!r}, which is not an absolute path. The " @@ -1705,9 +1714,16 @@ def state_dir(env: Mapping[str, str] | None = None) -> Path: "the same socket, so the state directory is named absolutely") return path xdg = env.get("XDG_STATE_HOME", "").strip() - base = Path(xdg) if xdg and Path(xdg).is_absolute() else ( - Path.home() / ".local" / "state") - return base / "opendox" + if xdg and Path(xdg).is_absolute(): + return Path(xdg) / "opendox" + try: + home = Path.home() + except RuntimeError: + raise ConfigurationError( + f"{setting} is unset and this process has no home directory to " + "put the default under (no HOME, and no password entry for the " + f"user). Set {setting} to an absolute path") from None + return home / ".local" / "state" / "opendox" def database_bundle(state: Path) -> DatabaseBundle: diff --git a/tests_runtime/test_local_lifecycle.py b/tests_runtime/test_local_lifecycle.py index 334deb49..6abb41c7 100644 --- a/tests_runtime/test_local_lifecycle.py +++ b/tests_runtime/test_local_lifecycle.py @@ -203,7 +203,11 @@ def _lock(bundle: config.DatabaseBundle, pid: int) -> None: f"{pid}\n{bundle.data_dir}\n", encoding="utf-8") -@pytest.mark.skipif(not Path("/proc/self").exists(), reason="asks /proc") +needs_proc = pytest.mark.skipif(not bundle_mod.PROC.joinpath("self").exists(), + reason="asks /proc") + + +@needs_proc def test_a_recycled_pid_is_not_believed_even_with_postgres_in_its_argv( short_state: Path) -> None: """Even IN the data directory, and with `postgres -D ` in its @@ -225,7 +229,7 @@ def test_a_recycled_pid_is_not_believed_even_with_postgres_in_its_argv( decoy.wait(timeout=10) -@pytest.mark.skipif(not Path("/proc/self").exists(), reason="asks /proc") +@needs_proc def test_a_postgres_serving_another_directory_is_not_this_server( short_state: Path, tmp_path: Path) -> None: """The exact pair: an executable NAMED `postgres` is not enough, it must @@ -244,10 +248,6 @@ def test_a_postgres_serving_another_directory_is_not_this_server( decoy = _decoy(str(named), cwd, "60") try: _lock(bundle, decoy.pid) - deadline = time.monotonic() + 10 - while (Path(os.readlink(f"/proc/{decoy.pid}/exe")).name != "postgres" - and time.monotonic() < deadline): - time.sleep(0.02) # the exec is complete got = bundle_mod.running_pid(bundle) assert got == (decoy.pid if believed else None), (cwd, got) finally: @@ -255,6 +255,45 @@ def test_a_postgres_serving_another_directory_is_not_this_server( decoy.wait(timeout=10) +def test_a_pid_nothing_can_describe_is_not_believed_and_its_lock_is_kept( + short_state: Path, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + """No `/proc` (macOS, the BSDs): nothing can say what the pid is, so it + is not reported as this server, and its lock is NOT removed, since + nothing proved it stale. PostgreSQL's own interlock is left to judge a + start. The decoy would be believed if it could be described (the case + above, where it is).""" + monkeypatch.setattr(bundle_mod, "PROC", tmp_path / "no-proc") + sleeper = shutil.which("sleep") + assert sleeper + named = tmp_path / "bin" / "postgres" + named.parent.mkdir() + shutil.copy2(sleeper, named) + bundle = config.DatabaseBundle(short_state) + bundle.data_dir.mkdir(parents=True) + decoy = _decoy(str(named), bundle.data_dir, "60") + try: + _lock(bundle, decoy.pid) + assert bundle_mod.running_pid(bundle) is None + bundle_mod._remove_a_proven_stale_lock(bundle) + assert (bundle.data_dir / "postmaster.pid").exists() + finally: + decoy.kill() + decoy.wait(timeout=10) + + +@needs_proc +def test_a_process_that_exits_before_it_is_described_is_not_believed( + short_state: Path, monkeypatch: pytest.MonkeyPatch) -> None: + """Alive at the signal check, gone by the time it is described: that is + a stale lock, never proof of a server.""" + bundle = config.DatabaseBundle(short_state) + gone = subprocess.Popen(["true"]) + gone.wait(timeout=10) + _lock(bundle, gone.pid) + monkeypatch.setattr(bundle_mod.os, "kill", lambda pid, sig: None) + assert bundle_mod.running_pid(bundle) is None + + def test_another_users_process_is_not_this_server(short_state: Path) -> None: """A pid the kernel will not let this user signal belongs to another user, and this bundle's server runs as the owner of its 0700 data directory. @@ -266,6 +305,35 @@ def test_another_users_process_is_not_this_server(short_state: Path) -> None: assert bundle_mod.running_pid(bundle) is None +# -- the state directory, refused by name -------------------------------------- + + +def test_a_state_dir_naming_an_unknown_user_is_refused_by_name() -> None: + raw = "~no-such-user-odx-8f3a/state" + with pytest.raises(config.ConfigurationError) as caught: + config.state_dir({STATE: raw}) + assert STATE in str(caught.value), caught.value + with pytest.raises(config.ConfigurationError): + config.load_settings({MODE: "local", STATE: raw}) + # a HOSTED install never reads it, and is not refused over it (13.6) + hosted = config.load_settings({ + STATE: raw, PREFIX + "DATABASE_URL": "postgresql://s@127.0.0.1:1/x", + PREFIX + "OIDC_ISSUER": "https://issuer.example.invalid/realms/x", + PREFIX + "OIDC_AUDIENCE": "fixture"}) + assert hosted.install_mode == config.INSTALL_MODE_HOSTED + + +def test_no_home_for_the_default_state_dir_is_refused_by_name( + monkeypatch: pytest.MonkeyPatch) -> None: + def _no_home(): + raise RuntimeError("Could not determine home directory.") + + monkeypatch.setattr(config.Path, "home", staticmethod(_no_home)) + with pytest.raises(config.ConfigurationError) as caught: + config.state_dir({}) + assert STATE in str(caught.value), caught.value + + # -- the two refusal classes, each with its own reason -------------------------- From 026f00ea7ce8eda3dfc513e3cff848c9a08a472d Mon Sep 17 00:00:00 2001 From: Brett Heap <1513478+brettheap@users.noreply.github.com> Date: Wed, 30 Sep 2026 15:51:48 +0000 Subject: [PATCH 14/88] Fix round: the install-mode module says which of its cases is DB-backed (Copilot review) The module docstring called every case hermetic. Since the fourth fix round, one is not: `test_runtime_status_of_a_healthy_local_install_exits_zero` takes the suite's `postgres_dsn` and `database` fixtures, because a healthy local `status` exits 0 only against a database that answers. The docstring now names that case and says it is skipped without Postgres and fails under CI, like every DB-backed case. It says the rest stay hermetic. Docstring only: the module runs 41 passed. Lane: openxfactory-4 (openXfactory-4-openDox_extraction) Arc: neutral-product-standalone-operability Co-Authored-By: Claude Opus 5.5 (1M context) --- tests_runtime/test_install_mode.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/tests_runtime/test_install_mode.py b/tests_runtime/test_install_mode.py index 8fcc2a7b..24d44eb6 100644 --- a/tests_runtime/test_install_mode.py +++ b/tests_runtime/test_install_mode.py @@ -1,12 +1,20 @@ """`OPENDOX_INSTALL_MODE`: the local single-user install, and the hosted one it cannot be reached from by omission (plan 034 T070; #1144 13.4, 13.5, 13.6). -HERMETIC: standard library plus `opendox.runtime.config` and the runtime CLI, -both stdlib-only at import. No database is reached: every DSN below is a -well-formed PostgreSQL URI aimed at port 1 of the loopback, so a refusal here -is always a CONFIGURATION refusal, which is the whole of what 13.4-13.6 ask of +HERMETIC, WITH ONE EXCEPTION. Every case but one uses the standard library +plus `opendox.runtime.config` and the runtime CLI, both stdlib-only at import, +and reaches no database. Each DSN in those cases is a well-formed PostgreSQL +URI aimed at port 1 of the loopback, so a refusal there is always a +CONFIGURATION refusal, which is the whole of what 13.4-13.6 ask of `load_settings`. +The exception is `test_runtime_status_of_a_healthy_local_install_exits_zero`, +which is DB-BACKED (Copilot review of openDox-code#67). A healthy local +`status` exits 0 only against a database that answers, so that case takes the +suite's `postgres_dsn` and `database` fixtures (`tests_runtime/conftest.py`). +Like every DB-backed case, it is skipped where there is no Postgres, and it +fails under CI. + WHAT IS RULED AND WHAT IS READ, so a reviewer can tell them apart: * RULED: the selector, its two values and its hosted default (#1144 13.4); From a0fb7c8d4320108a408f6ece93a09adc69c4e408 Mon Sep 17 00:00:00 2001 From: Brett Heap <1513478+brettheap@users.noreply.github.com> Date: Wed, 30 Sep 2026 15:58:55 +0000 Subject: [PATCH 15/88] Fix round: pyproject's packaging note names the resolver that exists (Copilot review) The data-files note pointed readers at `opendox.runtime.config.packaged_migrations_dir`, which fix round 4 replaced with `installation_migrations_dir`, the source tree first and then the RECORD of the distribution that holds the running module. The note now names that function and says what it asks. The packaging case now also checks that every `opendox.runtime.config.` pyproject.toml names exists, so a stale pointer cannot come back. Against 4aed6278's pyproject.toml it fails, naming `packaged_migrations_dir`. Here it passes. Full suite: 2584 selected, 2573 passed, 11 skipped, 0 failed. Lane: openxfactory-4 (openXfactory-4-openDox_extraction) Arc: neutral-product-standalone-operability Co-Authored-By: Claude Opus 5.5 (1M context) --- pyproject.toml | 7 ++++--- tests_runtime/test_bundled_postgres.py | 5 +++++ 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 7be64df0..15f8ca53 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -246,9 +246,10 @@ where = ["src"] # `COPY migrations ./migrations`, run from `/app`) and every checkout run finds # it; this maps the same files into the wheel's data directory, so an install # run OUTSIDE a checkout still has the migrations its bundled server applies. -# `opendox.runtime.config.packaged_migrations_dir` finds them through the -# installed distribution's own record, and the canonical digest gate is what -# proves the copy found is the pinned one. A data directory and not package +# `opendox.runtime.config.installation_migrations_dir` finds them through the +# own record of the installed distribution that holds the running module +# (after the source tree, for a checkout), and the canonical digest gate is +# what proves the copy found is the pinned one. A data directory and not package # data because package data must live inside the package, and the one tree is # not moved. [tool.setuptools.data-files] diff --git a/tests_runtime/test_bundled_postgres.py b/tests_runtime/test_bundled_postgres.py index acf6e369..1062c809 100644 --- a/tests_runtime/test_bundled_postgres.py +++ b/tests_runtime/test_bundled_postgres.py @@ -245,6 +245,11 @@ def test_the_local_extra_carries_the_runtime_and_the_server_and_test_joins_it( assert re.search(r"(?m)^pgserver==", lock), "the lock does not pin pgserver" files = project["tool"]["setuptools"]["data-files"] assert files == {"share/opendox/migrations": ["migrations/*.sql"]} + # and the packaging notes name functions that exist (Copilot review of #69) + for name in re.findall(r"opendox\.runtime\.config\.(\w+)", + (ROOT / "pyproject.toml").read_text()): + assert hasattr(config, name), ( + f"pyproject.toml points readers at config.{name}, which does not exist") # -- the layout, before anything starts ---------------------------------------- From 0f77d5c1a6dc68ea8314b2b2177c90f1147f569c Mon Sep 17 00:00:00 2001 From: Brett Heap <1513478+brettheap@users.noreply.github.com> Date: Wed, 30 Sep 2026 16:33:22 +0000 Subject: [PATCH 16/88] Fix round: libpq's environment, the socket's path and a relative HOME (Copilot and SonarCloud review) Copilot's review at a0fb7c8d opened three threads, and SonarCloud raised a reliability finding. All four are fixed here. - PG* defaults (r4146787926). libpq fills every parameter a DSN leaves unset from the environment. PGHOSTADDR outranks the socket `host` and sends the connection to TCP, PGSERVICE fills parameters from a service file, and PGOPTIONS sets the session's parameters. No DSN can name every parameter, and an explicitly empty `service` is itself an error. So `bundle.isolated_from_libpq_environment` lifts every PG* variable out of os.environ for the duration and puts it back afterwards. It wraps `generate-and-open --local`'s whole lifecycle and the runtime CLI's verbs under `local`. A hosted install's libpq is untouched (13.6). With PGHOSTADDR=192.0.2.1, PGSERVICE=no-such-service and PGOPTIONS=-c search_path=nowhere set, the real entry point still starts, migrates and serves its own server, and `runtime status` still finds it. - The socket's path (r4146787852). Before the socket directory is chmodded (a chmod follows a symlink), the resolved path is checked. The state dir, postgres/ and run/ must be real directories owned by this user and writable by no one else. Every ancestor must be owned by this user or by root, and must be sticky if every user can write it, or if a group other than this user's own can write it. Anything else is refused by name. - A relative HOME (r4146659876). It is refused for the default state directory, which would otherwise depend on the working directory. - SonarCloud S6466. server_binaries no longer indexes a list. It takes the first search location or none, and both refusal shapes have a case. Against a0fb7c8d's source, 8 of the new cases fail and the positive control passes. 11 mutants are killed. Full suite: 2595 selected, 2584 passed, 11 skipped, 0 failed. Lane: openxfactory-4 (openXfactory-4-openDox_extraction) Arc: neutral-product-standalone-operability Co-Authored-By: Claude Opus 5.5 (1M context) --- src/opendox/cli.py | 11 ++ src/opendox/runtime/bundle.py | 112 ++++++++++++++++++- src/opendox/runtime/cli.py | 26 ++++- src/opendox/runtime/config.py | 9 ++ tests_runtime/test_bundled_postgres.py | 36 ++++++- tests_runtime/test_local_lifecycle.py | 142 +++++++++++++++++++++++++ 6 files changed, 326 insertions(+), 10 deletions(-) diff --git a/src/opendox/cli.py b/src/opendox/cli.py index 5e0073ee..fd2da4e9 100644 --- a/src/opendox/cli.py +++ b/src/opendox/cli.py @@ -508,6 +508,17 @@ def cmd_generate_and_open(args: argparse.Namespace, *, opener=webbrowser.open) - _refuse_malformed_generated_at(args) server = bundle_mod.BundledServer(settings) args.database_bundle = server + # NO `PG*` DEFAULT REACHES THE BUNDLE'S CONNECTIONS while this process + # runs its database (Copilot review of openDox-code#69): see + # `bundle.isolated_from_libpq_environment`. + with bundle_mod.isolated_from_libpq_environment(): + return _run_the_local_lifecycle(args, server, opener=opener) + + +def _run_the_local_lifecycle(args: argparse.Namespace, server, *, opener) -> int: + """Start the bundled server, generate and serve, and stop it, however + this ends: a served run ended by Ctrl-C or SIGTERM, a `--no-serve` run, a + refusal, a failure, or an interrupt before anything was served.""" try: previous = signal.signal(signal.SIGTERM, _terminate_as_interrupt) except ValueError: # not the main thread: no handler to own diff --git a/src/opendox/runtime/bundle.py b/src/opendox/runtime/bundle.py index 9014332e..0994f446 100644 --- a/src/opendox/runtime/bundle.py +++ b/src/opendox/runtime/bundle.py @@ -59,15 +59,18 @@ from __future__ import annotations +import contextlib import ctypes import importlib.util import os import shutil import signal +import stat import subprocess import sys import tempfile import time +from collections.abc import Iterator from pathlib import Path from typing import Any @@ -115,14 +118,17 @@ def server_binaries() -> Path: object under the user's runtime directory as a side effect. """ spec = importlib.util.find_spec(SERVER_DISTRIBUTION) - locations = list(spec.submodule_search_locations or ()) if spec else [] - if not locations: + # THE FIRST LOCATION, OR NONE, read without an index, so no path reaches + # a subscript that could raise (SonarCloud S6466 on openDox-code#69). + location = next(iter(spec.submodule_search_locations or ()), None) \ + if spec else None + if location is None: raise BundleRefused( "the local install's PostgreSQL server is not installed: it " "arrives with the `local` extra, `pip install \"opendox[local]\"` " "(R1Q16 (iii)). A local install brings its own database and never " "borrows one") - binaries = Path(locations[0]) / "pginstall" / "bin" + binaries = Path(location) / "pginstall" / "bin" missing = [name for name in ("initdb", "postgres") if not os.access(binaries / name, os.X_OK)] if missing: @@ -263,6 +269,35 @@ def _child_environment() -> dict[str, str]: if not name.startswith("PG")} +@contextlib.contextmanager +def isolated_from_libpq_environment() -> Iterator[None]: + """Run a LOCAL install's client side with libpq's `PG*` defaults out of reach. + + libpq fills every connection parameter a DSN leaves unset from the + process environment. Some of those parameters move the connection + somewhere else. `PGHOSTADDR` outranks the DSN's socket `host` and sends + it to a TCP server. `PGSERVICE` fills parameters from a service file. + `PGOPTIONS` sets session parameters, a `search_path` among them. The + bundle's DSNs name their socket, port, user and database, but they + cannot name every parameter libpq has, and an explicitly empty `service` + is itself an error. So a local install's process reads NONE of them while + it runs its database: they are lifted out of `os.environ` for the + duration and put back afterwards (Copilot review of openDox-code#69). + `_child_environment` already does the same for `initdb` and the server. + + For the process's own entry points only: `generate-and-open --local` + around its whole lifecycle, and the runtime CLI's verbs under `local`. + Nothing else in those processes speaks libpq. + """ + lifted = {name: os.environ.pop(name) for name in + [name for name in os.environ if name.startswith("PG")]} + try: + yield + finally: + for name, value in lifted.items(): + os.environ.setdefault(name, value) + + def _die_with_parent(): """A `preexec_fn` that signals the server when its parent goes away (iv). @@ -291,6 +326,33 @@ def _preexec() -> None: # pragma: no cover - runs in the child return _preexec +def _unsafe_because(info: os.stat_result, *, uid: int, gid: int, + own: bool) -> str | None: + """Why one directory of the socket's path is unsafe, or `None`.""" + mode = info.st_mode + if stat.S_ISLNK(mode): + return "is a symbolic link" + if not stat.S_ISDIR(mode): + return "is not a directory" + if own: + if info.st_uid != uid: + return f"is owned by uid {info.st_uid}, not by this user" + if mode & 0o022: + return (f"is writable by {'every user' if mode & 0o002 else 'its group'}" + f" (mode {stat.S_IMODE(mode):o})") + return None + if info.st_uid not in (uid, 0): + return f"is owned by uid {info.st_uid}, neither this user nor root" + sticky = bool(mode & stat.S_ISVTX) + if mode & 0o002 and not sticky: + return (f"is writable by every user and is not sticky " + f"(mode {stat.S_IMODE(mode):o})") + if mode & 0o020 and info.st_gid != gid and not sticky: + return (f"is writable by group {info.st_gid}, which is not this " + f"user's own, and is not sticky (mode {stat.S_IMODE(mode):o})") + return None + + class BundledServer: """One local install's PostgreSQL server: started as this process's child. @@ -381,8 +443,47 @@ def _prepare_directories(self) -> None: for directory in (self.bundle.state_dir, self.bundle.data_dir.parent): directory.mkdir(mode=0o700, parents=True, exist_ok=True) self.bundle.socket_dir.mkdir(mode=0o700, parents=True, exist_ok=True) + # CHECKED BEFORE THE CHMOD, which follows a symbolic link: a `run` + # placed there as a link would otherwise have its TARGET re-moded. + self._refuse_an_unsafe_tree() os.chmod(self.bundle.socket_dir, 0o700) + def _refuse_an_unsafe_tree(self) -> None: + """The socket's whole path is this user's to change, or it is refused. + + `trust` makes reaching the socket the credential, so the 0700 on the + socket directory is worth only what the directories above it are + worth. A directory entry is controlled by its PARENT. A parent that + another user can write lets them rename `run` away, or put a symbolic + link in its place, after the mode is set (Copilot review of + openDox-code#69). So, over the RESOLVED path, which a user's own + symbolic link in `OPENDOX_STATE_DIR` may lead to: + + * this install's own tree, the state directory, `postgres/` and + `run/`, must be real directories, owned by this user and writable + by no one else; + * every directory above it must be owned by this user or by root. + One that every user can write must be sticky, as `/tmp` is, so + nobody can rename what is not theirs. One that its group can + write must be sticky too, unless the group is this user's own, + which is how a umask-002 system creates the user's directories. + """ + uid, gid = os.getuid(), os.getgid() + state = self.bundle.state_dir.resolve() + own = [state, state / self.bundle.socket_dir.parent.name, + state / self.bundle.socket_dir.parent.name / self.bundle.socket_dir.name] + for directory, mine in [(path, True) for path in own] + \ + [(path, False) for path in state.parents]: + info = os.lstat(directory) + reason = _unsafe_because(info, uid=uid, gid=gid, own=mine) + if reason is not None: + raise BundleRefused( + f"{directory} {reason}, so another user could replace the " + "bundled server's socket directory, and reaching that " + "socket is the only credential the server asks for. Use a " + f"state directory only this user can change ({PREFIX}" + "STATE_DIR)") + #: The prefix an initialization attempt's directory carries, beside the #: data directory, followed by the pid of the process making it. ATTEMPT_PREFIX = "data.initdb-" @@ -599,5 +700,6 @@ def __exit__(self, *exc: object) -> None: self.stop() -__all__ = ["BUNDLE_PORT", "BundleRefused", "BundledServer", "report", - "running_pid", "server_binaries"] +__all__ = ["BUNDLE_PORT", "BundleRefused", "BundledServer", + "isolated_from_libpq_environment", "report", "running_pid", + "server_binaries"] diff --git a/src/opendox/runtime/cli.py b/src/opendox/runtime/cli.py index d8b5fc20..b5d22207 100644 --- a/src/opendox/runtime/cli.py +++ b/src/opendox/runtime/cli.py @@ -80,6 +80,7 @@ import contextlib import json import logging +import os import re import stat import sys @@ -97,6 +98,7 @@ ConfigurationError, redacted_url, RuntimeSettings, + install_mode, load_migration_settings, load_settings, migration_database_url, @@ -1225,6 +1227,27 @@ def build_parser() -> argparse.ArgumentParser: return parser +def _isolated_when_local() -> contextlib.AbstractContextManager[None]: + """A LOCAL install's verbs run with libpq's `PG*` defaults out of reach. + + The bundle's DSNs name their socket, but libpq fills everything else from + the environment, and `PGHOSTADDR` alone would send `status` or `migrate` + to a TCP server instead (Copilot review of openDox-code#69; see + `bundle.isolated_from_libpq_environment`). A hosted install's operator + configures libpq as they please, as before (13.6). A selector that cannot + be read isolates nothing; the verb refuses it by name. + """ + try: + local = install_mode(os.environ) == INSTALL_MODE_LOCAL + except ConfigurationError: + local = False + if not local: + return contextlib.nullcontext() + from opendox.runtime import bundle as bundle_mod + + return bundle_mod.isolated_from_libpq_environment() + + def main(argv: list[str] | None = None) -> int: """Parse and dispatch, and NEVER let a traceback be the whole answer. @@ -1240,7 +1263,8 @@ def main(argv: list[str] | None = None) -> int: """ args = build_parser().parse_args(argv) try: - return int(args.func(args)) + with _isolated_when_local(): + return int(args.func(args)) except SystemExit: raise except Exception as exc: # noqa: BLE001 diff --git a/src/opendox/runtime/config.py b/src/opendox/runtime/config.py index c9d7b73b..56725995 100644 --- a/src/opendox/runtime/config.py +++ b/src/opendox/runtime/config.py @@ -1723,6 +1723,15 @@ def state_dir(env: Mapping[str, str] | None = None) -> Path: f"{setting} is unset and this process has no home directory to " "put the default under (no HOME, and no password entry for the " f"user). Set {setting} to an absolute path") from None + if not home.is_absolute(): + # `Path.home()` returns HOME as given, and a relative one would give + # the serving process and a `runtime status` run from another + # directory two different sockets (Copilot review of openDox-code#69). + raise ConfigurationError( + f"{setting} is unset and HOME is {str(home)!r}, which is not an " + "absolute path, so the default state directory would depend on " + f"the working directory. Set {setting} to an absolute path, or " + "HOME to one") return home / ".local" / "state" / "opendox" diff --git a/tests_runtime/test_bundled_postgres.py b/tests_runtime/test_bundled_postgres.py index 1062c809..56e1f844 100644 --- a/tests_runtime/test_bundled_postgres.py +++ b/tests_runtime/test_bundled_postgres.py @@ -169,14 +169,15 @@ def _first_url(child: subprocess.Popen, seconds: float) -> str | None: return None -def _launch(corpus: Path, state: Path, run_dir: Path) -> tuple[subprocess.Popen, str]: +def _launch(corpus: Path, state: Path, run_dir: Path, + **extra: str) -> tuple[subprocess.Popen, str]: """`generate-and-open --local` in the BACKGROUND, and the URL it serves.""" child = subprocess.Popen( [sys.executable, str(DRIVER), "generate-and-open", config.LOCAL_FLAG, "--repo-root", str(corpus), "--repository", "fixture", "--run-dir", str(run_dir), "--no-open", "--no-validate", "--port", "0"], - env=_clean_env(**{STATE: str(state)}), cwd=ROOT, + env=_clean_env(**{STATE: str(state)}, **extra), cwd=ROOT, stdout=subprocess.PIPE, stderr=subprocess.PIPE) url = _first_url(child, 90) if url is None: @@ -211,12 +212,12 @@ def test_the_launch_helper_is_bounded_by_its_deadline_not_by_the_child() -> None talker.communicate(timeout=10) -def _status(state: Path) -> tuple[int, dict]: +def _status(state: Path, **extra: str) -> tuple[int, dict]: """`OPENDOX_INSTALL_MODE=local opendox-runtime runtime status`, a SECOND process.""" done = subprocess.run( [sys.executable, "-m", "opendox.runtime.cli", "runtime", "status", "--probe-timeout", "10"], - env=_clean_env(**{MODE: "local", STATE: str(state)}), cwd=ROOT, + env=_clean_env(**{MODE: "local", STATE: str(state)}, **extra), cwd=ROOT, capture_output=True, text=True, timeout=60) return done.returncode, json.loads(done.stdout) @@ -343,6 +344,33 @@ def test_the_entry_point_owns_a_migrated_server_with_no_tcp_listener( "the data directory must survive a stop: it is the install's database" +#: libpq defaults that, READ, would move the bundle's connections: to an +#: unroutable TCP address (TEST-NET-1, RFC 5737), through a service that does +#: not exist, and into a schema that does not either. +HOSTILE_LIBPQ = {"PGHOSTADDR": "192.0.2.1", "PGSERVICE": "no-such-service-odx", + "PGOPTIONS": "-c search_path=nowhere"} + + +def test_libpq_defaults_in_the_environment_never_reach_the_bundle( + corpus: Path, state_dir: Path, tmp_path: Path) -> None: + """`PGHOSTADDR` outranks a DSN's socket `host`, `PGSERVICE` fills + parameters from a service file, and `PGOPTIONS` sets the session's + parameters. With all three set, the entry point still starts, migrates + and serves ITS OWN server, and `runtime status` in a second process still + finds it (Copilot review of openDox-code#69).""" + server, url = _launch(corpus, state_dir, tmp_path / "run", **HOSTILE_LIBPQ) + try: + code, status = _status(state_dir, **HOSTILE_LIBPQ) + assert status.get("database") == "reachable", status + assert status.get("applied_migrations") and \ + not status.get("pending_migrations"), status + assert code == 0 and status["ok"] is True, status + finally: + server.send_signal(signal.SIGTERM) + server.communicate(timeout=60) + assert server.returncode == 0, server.returncode + + @pytest.mark.skipif(not sys.platform.startswith("linux"), reason="PR_SET_PDEATHSIG is Linux's") def test_the_server_stops_even_when_the_entry_point_is_killed_outright( diff --git a/tests_runtime/test_local_lifecycle.py b/tests_runtime/test_local_lifecycle.py index 6abb41c7..5bec3e8b 100644 --- a/tests_runtime/test_local_lifecycle.py +++ b/tests_runtime/test_local_lifecycle.py @@ -20,6 +20,7 @@ import os import shutil import signal +import stat import subprocess import sys import tempfile @@ -334,6 +335,129 @@ def _no_home(): assert STATE in str(caught.value), caught.value +def test_a_relative_home_for_the_default_state_dir_is_refused_by_name( + monkeypatch: pytest.MonkeyPatch) -> None: + """`Path.home()` returns HOME as given; a relative one would put the + socket wherever each process happens to run (Copilot review of #69).""" + monkeypatch.setenv("HOME", "relative-home") + with pytest.raises(config.ConfigurationError) as caught: + config.state_dir({}) + assert STATE in str(caught.value) and "HOME" in str(caught.value) + # an absolute XDG_STATE_HOME still answers without HOME at all + assert config.state_dir({"XDG_STATE_HOME": "/srv/state"}) == \ + Path("/srv/state/opendox") + + +# -- libpq's environment, out of reach --------------------------------------- + + +def test_libpq_defaults_are_lifted_for_the_duration_and_put_back( + monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("PGHOSTADDR", "192.0.2.1") + monkeypatch.setenv("PGSERVICE", "no-such-service-odx") + monkeypatch.setenv("OPENDOX_NOT_LIBPQ", "kept") + with bundle_mod.isolated_from_libpq_environment(): + assert not [name for name in os.environ if name.startswith("PG")] + assert os.environ["OPENDOX_NOT_LIBPQ"] == "kept" + assert os.environ["PGHOSTADDR"] == "192.0.2.1" + assert os.environ["PGSERVICE"] == "no-such-service-odx" + + +def test_the_runtime_cli_isolates_only_a_local_install( + monkeypatch: pytest.MonkeyPatch, scrubbed) -> None: + from opendox.runtime import cli as runtime_cli + + seen: dict = {} + + def _verb(args) -> int: + seen["PGHOSTADDR"] = os.environ.get("PGHOSTADDR") + return 0 + + monkeypatch.setenv("PGHOSTADDR", "192.0.2.1") + parser = runtime_cli.build_parser() + monkeypatch.setattr(runtime_cli, "build_parser", lambda: parser) + real_parse = parser.parse_args + + def _parse(argv=None): + args = real_parse(argv) + args.func = _verb + return args + + monkeypatch.setattr(parser, "parse_args", _parse) + scrubbed.setenv(MODE, "local") + assert runtime_cli.main(["runtime", "status"]) == 0 + assert seen["PGHOSTADDR"] is None, "a local verb saw PGHOSTADDR" + scrubbed.setenv(MODE, "hosted") + assert runtime_cli.main(["runtime", "status"]) == 0 + assert seen["PGHOSTADDR"] == "192.0.2.1", "a hosted verb lost its libpq setting" + assert os.environ["PGHOSTADDR"] == "192.0.2.1" + + +# -- the socket's path, this user's to change ----------------------------------- + + +def _prepared(monkeypatch, tmp_path: Path, state: Path) -> bundle_mod.BundledServer: + binaries = _binaries(tmp_path, initdb='echo "initdb reached" >&2; exit 1') + monkeypatch.setattr(bundle_mod, "server_binaries", lambda: binaries) + return bundle_mod.BundledServer(_local(state)) + + +def test_a_state_dir_others_can_write_is_refused( + monkeypatch, tmp_path: Path, short_state: Path) -> None: + short_state.chmod(0o777) + server = _prepared(monkeypatch, tmp_path, short_state) + with pytest.raises(bundle_mod.BundleRefused) as caught: + server.start() + assert "writable by every user" in str(caught.value), caught.value + assert str(short_state) in str(caught.value) + + +@pytest.mark.parametrize("which", ["postgres", "run"]) +def test_a_symlink_inside_the_state_tree_is_refused_and_its_target_untouched( + monkeypatch, tmp_path: Path, short_state: Path, which: str) -> None: + target = tmp_path / "somewhere-else" + target.mkdir(mode=0o755) + target.chmod(0o755) + if which == "postgres": + (short_state / "postgres").symlink_to(target) + else: + (short_state / "postgres").mkdir(mode=0o700) + (short_state / "postgres" / "run").symlink_to(target) + server = _prepared(monkeypatch, tmp_path, short_state) + with pytest.raises(bundle_mod.BundleRefused) as caught: + server.start() + assert "symbolic link" in str(caught.value), caught.value + assert stat.S_IMODE(target.stat().st_mode) == 0o755, "the link's target was re-moded" + + +def test_an_ancestor_every_user_can_write_without_the_sticky_bit_is_refused( + monkeypatch, tmp_path: Path, short_state: Path) -> None: + open_dir = short_state / "open" + open_dir.mkdir() + open_dir.chmod(0o777) + server = _prepared(monkeypatch, tmp_path, open_dir / "state") + with pytest.raises(bundle_mod.BundleRefused) as caught: + server.start() + assert str(open_dir) in str(caught.value) and "not sticky" in str(caught.value) + + +def test_a_sticky_or_own_group_ancestor_is_accepted( + monkeypatch, tmp_path: Path, short_state: Path) -> None: + """The positive controls: `/tmp`'s shape (every user, sticky) and a + umask-002 system's shape (this user's own group). The start gets past + the tree and reaches `initdb`, which the stand-in fails on purpose.""" + sticky = short_state / "sticky" + sticky.mkdir() + sticky.chmod(0o1777) + group = sticky / "group" + group.mkdir() + group.chmod(0o775) + server = _prepared(monkeypatch, tmp_path, group / "state") + with pytest.raises(bundle_mod.BundleRefused) as caught: + server.start() + assert "initdb" in str(caught.value), caught.value + + # -- the two refusal classes, each with its own reason -------------------------- @@ -368,6 +492,24 @@ def test_both_classes_together_name_both_reasons() -> None: assert "hunter2" not in message +@pytest.mark.parametrize("found", ["absent", "no-locations"]) +def test_a_missing_server_package_is_the_named_refusal( + monkeypatch: pytest.MonkeyPatch, found: str) -> None: + """No `pgserver` at all, or a spec with no location: both are the one + refusal naming the `local` extra, never an `IndexError`.""" + import importlib.machinery + + spec = None + if found == "no-locations": + spec = importlib.machinery.ModuleSpec("pgserver", None, is_package=True) + spec.submodule_search_locations = [] + monkeypatch.setattr(bundle_mod.importlib.util, "find_spec", + lambda name: spec) + with pytest.raises(bundle_mod.BundleRefused) as caught: + bundle_mod.server_binaries() + assert 'opendox[local]' in str(caught.value), caught.value + + # -- initdb, and every phase of a start --------------------------------------- From 379fbb14539c9069dc1fd737403ef46f8d1ea950 Mon Sep 17 00:00:00 2001 From: Brett Heap <1513478+brettheap@users.noreply.github.com> Date: Wed, 30 Sep 2026 16:48:28 +0000 Subject: [PATCH 17/88] Fix round: the socket's path trusts no group and no link it cannot vouch for (Copilot review) Copilot's review at 0f77d5c1 opened two threads. Both were real. - r4147004990: a user's primary group can have other members, so a 0775 ancestor is not private. Every ancestor that anyone else can write, a group included, must now be sticky. The round-7 allowance for the user's own group is gone, and its positive control is now a refusal case. The sticky shape (/tmp) is still the control. - r4147005063: resolving the configured path before checking it discarded the path that was actually configured. A link on that path could be repointed afterwards, while the bundle kept using the unresolved paths. Now: - the ancestors of BOTH the configured path and the resolved one are checked; - every symbolic link on the configured path must be owned by this user or by root; - `..` is refused in OPENDOX_STATE_DIR and XDG_STATE_HOME (and in a derived HOME), so the configured components are the ones the kernel walks. A user's own link to a private directory is still accepted. Against 0f77d5c1's source, 5 of the new cases fail and the 4 controls pass. 5 mutants are killed. Full suite: 2601 selected, 2590 passed, 11 skipped, 0 failed. Lane: openxfactory-4 (openXfactory-4-openDox_extraction) Arc: neutral-product-standalone-operability Co-Authored-By: Claude Opus 5.5 (1M context) --- src/opendox/runtime/bundle.py | 94 ++++++++++++++++----------- src/opendox/runtime/config.py | 17 +++++ tests_runtime/test_local_lifecycle.py | 84 ++++++++++++++++++++++-- 3 files changed, 152 insertions(+), 43 deletions(-) diff --git a/src/opendox/runtime/bundle.py b/src/opendox/runtime/bundle.py index 0994f446..d38338b4 100644 --- a/src/opendox/runtime/bundle.py +++ b/src/opendox/runtime/bundle.py @@ -80,6 +80,7 @@ BUNDLE_OWNER_ROLE, BUNDLE_PORT, BUNDLE_SERVED_ROLE, + BUNDLE_SOCKET_DIR, LOCAL_FLAG, PREFIX, DatabaseBundle, @@ -326,8 +327,12 @@ def _preexec() -> None: # pragma: no cover - runs in the child return _preexec -def _unsafe_because(info: os.stat_result, *, uid: int, gid: int, - own: bool) -> str | None: +#: The install's own two directories under its state directory, the socket's +#: parent and the socket directory (`config.BUNDLE_SOCKET_DIR`). +BUNDLE_TREE = BUNDLE_SOCKET_DIR.parts + + +def _unsafe_because(info: os.stat_result, *, uid: int, own: bool) -> str | None: """Why one directory of the socket's path is unsafe, or `None`.""" mode = info.st_mode if stat.S_ISLNK(mode): @@ -343,13 +348,11 @@ def _unsafe_because(info: os.stat_result, *, uid: int, gid: int, return None if info.st_uid not in (uid, 0): return f"is owned by uid {info.st_uid}, neither this user nor root" - sticky = bool(mode & stat.S_ISVTX) - if mode & 0o002 and not sticky: - return (f"is writable by every user and is not sticky " - f"(mode {stat.S_IMODE(mode):o})") - if mode & 0o020 and info.st_gid != gid and not sticky: - return (f"is writable by group {info.st_gid}, which is not this " - f"user's own, and is not sticky (mode {stat.S_IMODE(mode):o})") + # A GROUP IS OTHER USERS, the user's own primary group included: it can + # have other members (Copilot review of openDox-code#69). + if mode & 0o022 and not mode & stat.S_ISVTX: + return (f"is writable by {'every user' if mode & 0o002 else 'its group'}" + f" and is not sticky (mode {stat.S_IMODE(mode):o})") return None @@ -452,37 +455,52 @@ def _refuse_an_unsafe_tree(self) -> None: """The socket's whole path is this user's to change, or it is refused. `trust` makes reaching the socket the credential, so the 0700 on the - socket directory is worth only what the directories above it are - worth. A directory entry is controlled by its PARENT. A parent that - another user can write lets them rename `run` away, or put a symbolic - link in its place, after the mode is set (Copilot review of - openDox-code#69). So, over the RESOLVED path, which a user's own - symbolic link in `OPENDOX_STATE_DIR` may lead to: - - * this install's own tree, the state directory, `postgres/` and - `run/`, must be real directories, owned by this user and writable - by no one else; - * every directory above it must be owned by this user or by root. - One that every user can write must be sticky, as `/tmp` is, so - nobody can rename what is not theirs. One that its group can - write must be sticky too, unless the group is this user's own, - which is how a umask-002 system creates the user's directories. + socket directory is worth only what the path above it is worth. A + directory entry is controlled by its PARENT: a parent that another + user can write lets them rename `run` away, or put a symbolic link in + its place, after the mode is set. A symbolic link on the way there + can be pointed elsewhere by whoever owns it, or by whoever can write + the directory it sits in (Copilot review of openDox-code#69). So: + + * THE INSTALL'S OWN TREE, as the configured path resolves: the + state directory, `postgres/` and `run/` must be real directories, + owned by this user and writable by no one else; + * EVERY DIRECTORY ABOVE IT, on the configured path and on the path + it resolves to, must be owned by this user or by root. One that + anyone else can write, a group included, must be sticky, as + `/tmp` is, so nobody can rename what is not theirs; + * EVERY SYMBOLIC LINK on the configured path must be this user's or + root's. + + `OPENDOX_STATE_DIR` never holds `..` (`config.state_dir` refuses it), + so the configured path's components are the ones the kernel walks. """ - uid, gid = os.getuid(), os.getgid() - state = self.bundle.state_dir.resolve() - own = [state, state / self.bundle.socket_dir.parent.name, - state / self.bundle.socket_dir.parent.name / self.bundle.socket_dir.name] - for directory, mine in [(path, True) for path in own] + \ - [(path, False) for path in state.parents]: - info = os.lstat(directory) - reason = _unsafe_because(info, uid=uid, gid=gid, own=mine) + uid = os.getuid() + configured = self.bundle.state_dir + state = configured.resolve() + tree = [state, state / BUNDLE_TREE[0], state / BUNDLE_TREE[0] / BUNDLE_TREE[1]] + checks = [(path, True) for path in tree] + [ + (path, False) for path in dict.fromkeys( + [*state.parents, *configured.parents])] + for directory, mine in checks: + info = os.lstat(directory) if mine else os.stat(directory) + reason = _unsafe_because(info, uid=uid, own=mine) if reason is not None: - raise BundleRefused( - f"{directory} {reason}, so another user could replace the " - "bundled server's socket directory, and reaching that " - "socket is the only credential the server asks for. Use a " - f"state directory only this user can change ({PREFIX}" - "STATE_DIR)") + raise self._unsafe(directory, reason) + for component in (configured, *configured.parents): + info = os.lstat(component) + if stat.S_ISLNK(info.st_mode) and info.st_uid not in (uid, 0): + raise self._unsafe( + component, f"is a symbolic link owned by uid {info.st_uid}, " + "neither this user nor root, who could point it elsewhere") + + @staticmethod + def _unsafe(directory: Path, reason: str) -> BundleRefused: + return BundleRefused( + f"{directory} {reason}, so another user could replace the bundled " + "server's socket directory, and reaching that socket is the only " + "credential the server asks for. Use a state directory only this " + f"user can change ({PREFIX}STATE_DIR)") #: The prefix an initialization attempt's directory carries, beside the #: data directory, followed by the pid of the process making it. diff --git a/src/opendox/runtime/config.py b/src/opendox/runtime/config.py index 56725995..36cd6272 100644 --- a/src/opendox/runtime/config.py +++ b/src/opendox/runtime/config.py @@ -1706,6 +1706,14 @@ def state_dir(env: Mapping[str, str] | None = None) -> Path: f"{setting} is {raw!r}, whose `~` names no user this system " "knows, so it expands to no directory. Name the state " "directory absolutely") from None + if ".." in path.parts: + # PARENT TRAVERSAL IS REFUSED, so the path the bundle checks is + # the one the kernel walks: `a/../b` names `b` lexically and + # something else wherever `a` is a symbolic link (Copilot review + # of openDox-code#69). + raise ConfigurationError( + f"{setting} is {raw!r}, which climbs out through `..`. Name " + "the state directory directly") if not path.is_absolute(): raise ConfigurationError( f"{setting} is {raw!r}, which is not an absolute path. The " @@ -1715,6 +1723,11 @@ def state_dir(env: Mapping[str, str] | None = None) -> Path: return path xdg = env.get("XDG_STATE_HOME", "").strip() if xdg and Path(xdg).is_absolute(): + if ".." in Path(xdg).parts: + raise ConfigurationError( + f"{setting} is unset and XDG_STATE_HOME is {xdg!r}, which " + f"climbs out through `..`. Set {setting}, or XDG_STATE_HOME, " + "to the directory itself") return Path(xdg) / "opendox" try: home = Path.home() @@ -1732,6 +1745,10 @@ def state_dir(env: Mapping[str, str] | None = None) -> Path: "absolute path, so the default state directory would depend on " f"the working directory. Set {setting} to an absolute path, or " "HOME to one") + if ".." in home.parts: + raise ConfigurationError( + f"{setting} is unset and HOME is {str(home)!r}, which climbs out " + f"through `..`. Set {setting} to the directory itself") return home / ".local" / "state" / "opendox" diff --git a/tests_runtime/test_local_lifecycle.py b/tests_runtime/test_local_lifecycle.py index 5bec3e8b..9285016d 100644 --- a/tests_runtime/test_local_lifecycle.py +++ b/tests_runtime/test_local_lifecycle.py @@ -441,23 +441,97 @@ def test_an_ancestor_every_user_can_write_without_the_sticky_bit_is_refused( assert str(open_dir) in str(caught.value) and "not sticky" in str(caught.value) -def test_a_sticky_or_own_group_ancestor_is_accepted( +def test_a_sticky_ancestor_is_accepted( monkeypatch, tmp_path: Path, short_state: Path) -> None: - """The positive controls: `/tmp`'s shape (every user, sticky) and a - umask-002 system's shape (this user's own group). The start gets past - the tree and reaches `initdb`, which the stand-in fails on purpose.""" + """The positive control, `/tmp`'s shape: every user can write it, and it + is sticky. The start gets past the tree and reaches `initdb`, which the + stand-in fails on purpose.""" sticky = short_state / "sticky" sticky.mkdir() sticky.chmod(0o1777) - group = sticky / "group" + server = _prepared(monkeypatch, tmp_path, sticky / "state") + with pytest.raises(bundle_mod.BundleRefused) as caught: + server.start() + assert "initdb" in str(caught.value), caught.value + + +def test_a_group_writable_ancestor_is_refused_even_for_this_users_group( + monkeypatch, tmp_path: Path, short_state: Path) -> None: + """A group is other users, the user's own primary group included (Copilot + review of #69): a 0775 ancestor is refused whatever its group.""" + group = short_state / "group" group.mkdir() group.chmod(0o775) + assert group.stat().st_gid == os.getgid() # this user's own group server = _prepared(monkeypatch, tmp_path, group / "state") + with pytest.raises(bundle_mod.BundleRefused) as caught: + server.start() + assert str(group) in str(caught.value) and "its group" in str(caught.value) + + +def test_a_link_in_a_directory_others_can_write_is_refused( + monkeypatch, tmp_path: Path, short_state: Path) -> None: + """The configured path goes through a symbolic link, and the link sits + in a directory every user can write. The link's TARGET is private, and + it is still refused, because anyone could replace the link (Copilot + review of #69).""" + private = short_state / "private" + private.mkdir(mode=0o700) + open_dir = short_state / "open" + open_dir.mkdir() + open_dir.chmod(0o777) + (open_dir / "link").symlink_to(private) + server = _prepared(monkeypatch, tmp_path, open_dir / "link" / "state") + with pytest.raises(bundle_mod.BundleRefused) as caught: + server.start() + assert str(open_dir) in str(caught.value) and "not sticky" in str(caught.value) + + +def test_this_users_own_link_to_a_private_directory_is_accepted( + monkeypatch, tmp_path: Path, short_state: Path) -> None: + private = short_state / "private" + private.mkdir(mode=0o700) + (short_state / "link").symlink_to(private) + server = _prepared(monkeypatch, tmp_path, short_state / "link" / "state") with pytest.raises(bundle_mod.BundleRefused) as caught: server.start() assert "initdb" in str(caught.value), caught.value +def test_a_link_another_user_owns_is_refused( + monkeypatch, tmp_path: Path, short_state: Path) -> None: + """Another user's link could be pointed elsewhere after the check. A + non-root suite cannot create one, so its `lstat` is stood in, for that + one path only.""" + private = short_state / "private" + private.mkdir(mode=0o700) + link = short_state / "link" + link.symlink_to(private) + real_lstat = os.lstat + + def _lstat(path, *args, **kwargs): + info = real_lstat(path, *args, **kwargs) + if Path(path) == link: + fields = list(info) + fields[4] = os.getuid() + 4242 # st_uid + return os.stat_result(fields) + return info + + monkeypatch.setattr(bundle_mod.os, "lstat", _lstat) + server = _prepared(monkeypatch, tmp_path, link / "state") + with pytest.raises(bundle_mod.BundleRefused) as caught: + server.start() + assert str(link) in str(caught.value) and "symbolic link owned by" in str(caught.value) + + +@pytest.mark.parametrize("variable", [STATE, "XDG_STATE_HOME"]) +def test_parent_traversal_in_the_state_path_is_refused(variable: str) -> None: + value = "/tmp/odx-a/../odx-b" + with pytest.raises(config.ConfigurationError) as caught: + config.state_dir({variable: value}) + assert "`..`" in str(caught.value), caught.value + + # -- the two refusal classes, each with its own reason -------------------------- From 84a6c04158e0d34b9f9d844a16aca4a691e9dfcc Mon Sep 17 00:00:00 2001 From: Brett Heap <1513478+brettheap@users.noreply.github.com> Date: Wed, 30 Sep 2026 17:47:27 +0000 Subject: [PATCH 18/88] T072: pixeltable-pgserver carries the server, and it authenticates by peer (RULED 5916000030 items 2, 3) Brett's rulings on openxFactory#656 (comment 5916000030) cover two things. Item 2, "pixeltable-pgserver (Recommended)". The `local` extra's carrier is now pixeltable-pgserver>=0.6.0, the maintained fork of pgserver. Only its binaries are used, found under pixeltable_pgserver/pginstall/bin. Measured on the installed 0.6.0 wheel: - initdb and postgres report PostgreSQL 16.14; - postgres links libz, libpthread, librt, libdl, libm and libc only, and initdb links the wheel's own vendored libpq through $ORIGIN; - the highest GLIBC symbol any binary or server module needs is 2.25, and the wheels are tagged manylinux_2_27/2_28; - the licence is Apache-2.0 (dist-info LICENSE and classifier); - the cp312 x86_64 wheel is 24,704,230 bytes; - wheels exist for cp310 to cp314. The lock was re-resolved in a clean environment under the existing pins less pgserver. The only line that moved is pgserver==0.1.4 -> pixeltable-pgserver==0.6.0. Item 3, "Peer auth + accept (Recommended)". - initdb now runs with --auth-local=peer --auth-host=reject. - Before every launch the bundle writes pg_hba.conf and pg_ident.conf atomically, mode 0600. pg_hba.conf holds one local rule, peer map=opendox, and host reject for IPv4 and IPv6. pg_ident.conf maps the running OS user (from the password database), and nobody else, to opendox and opendox_runtime. - listen_addresses stays empty. - An OS user name the map cannot hold plainly is refused, as is a uid with no password entry. - A cluster that an older build left as trust is put back to peer on its next start. The server's own reading proves it. pg_hba_file_rules has exactly those three rules and pg_ident_file_mappings exactly those two mappings, and system_user is peer: for both roles. The same OS user asking for a role outside the map is refused ("peer authentication failed"). Against 379fbb14's source and packaging, 13 of the new cases fail. Nine mutants of the carrier and the authentication are killed. The auth mutants are also killed by the real-server cases alone. Full suite: 2613 selected, 2602 passed, 11 skipped, 0 failed. Lane: openxfactory-4 (openXfactory-4-openDox_extraction) Arc: neutral-product-standalone-operability Co-Authored-By: Claude Opus 5.5 (1M context) --- constraints-cpython312-linux.txt | 7 +- pyproject.toml | 31 +++-- src/opendox/runtime/bundle.py | 158 +++++++++++++++++++------ src/opendox/runtime/config.py | 13 +- tests_runtime/test_bundled_postgres.py | 93 ++++++++++++++- tests_runtime/test_local_lifecycle.py | 75 +++++++++++- 6 files changed, 323 insertions(+), 54 deletions(-) diff --git a/constraints-cpython312-linux.txt b/constraints-cpython312-linux.txt index 4772dbe1..36437c08 100644 --- a/constraints-cpython312-linux.txt +++ b/constraints-cpython312-linux.txt @@ -34,6 +34,11 @@ # (`-c` this file), for plan 034 T072's `local` extra and the `test` extra's # `setuptools`: pgserver, its psutil, platformdirs and fasteners, and # setuptools are new; no earlier pin moved. +# Re-resolved 2026-09-30 on cpython 3.12.3 / linux x86_64, in a clean +# environment under the pins above less `pgserver`, when the carrier became +# `pixeltable-pgserver` (RULED, openxFactory#656 `5916000030` item 2): the one +# line that moved is `pgserver==0.1.4` -> `pixeltable-pgserver==0.6.0`, whose +# own requirements were all pinned already. PyJWT==2.14.0 PyYAML==6.0.3 Pygments==2.21.0 @@ -53,7 +58,7 @@ httpx==0.28.1 idna==3.20 iniconfig==2.3.0 packaging==26.3 -pgserver==0.1.4 +pixeltable-pgserver==0.6.0 platformdirs==4.12.2 pluggy==1.6.0 psutil==7.2.2 diff --git a/pyproject.toml b/pyproject.toml index 15f8ca53..3a870b31 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -119,18 +119,29 @@ test = [ # `opendox generate-and-open --local …`. It carries the runtime's packages and # the bundled server's own, and nothing else. # -# `pgserver` IS THE SERVER'S CARRIER, and only its binaries are used (see -# `opendox/runtime/bundle.py` for why its own manager is not): it ships -# PostgreSQL 16 as `initdb` and `postgres` inside the wheel, built to link only -# libc and libz, so a manylinux2014 host needs nothing else installed — no -# system PostgreSQL, no ICU. Apache-2.0; the server it carries is under the -# PostgreSQL License. `>=0.1.4` is the release this package has been exercised -# against (measured 2026-09-30, python 3.12.3: PostgreSQL 16.2), the same rule -# every floor in this file is set by. Its own three requirements (psutil, -# platformdirs, fasteners) arrive with it and are imported by nothing here. +# `pixeltable-pgserver` IS THE SERVER'S CARRIER (RULED, openxFactory#656 +# comment `5916000030` item 2, Brett Heap 2026-09-30: "pixeltable-pgserver +# (Recommended)"), and only its binaries are used (see `opendox/runtime/ +# bundle.py` for why its own manager is not). It is the maintained fork of +# `pgserver`, and it ships PostgreSQL 16 as `initdb` and `postgres` inside the +# wheel, built to link only libc and libz, so a host needs no system +# PostgreSQL and no ICU. Measured 2026-09-30 on its 0.6.0 cp312 manylinux +# wheel: +# * PostgreSQL 16.14 (`pginstall/bin/postgres --version`); the wheel also +# carries an 18.4 under `pginstall18/`, which this package does not use; +# * wheels for cp310 to cp314, so every interpreter `requires-python` +# admits today has one; `pgserver` 0.1.4 stopped at cp312 and 16.2; +# * linux wheels tagged manylinux_2_27/2_28 (glibc 2.27 and later), on +# x86_64 and aarch64, plus macOS and Windows; +# * about 24.7 MB per wheel, because it carries the two server majors; +# * Apache-2.0 for the package; the PostgreSQL License for the server. +# `>=0.6.0` is the release this package has been exercised against, the same +# rule every floor in this file is set by. Its own requirements (fasteners, +# platformdirs, psutil, typing-extensions) arrive with it and are imported by +# nothing here. local = [ "opendox[runtime]", - "pgserver>=0.1.4", + "pixeltable-pgserver>=0.6.0", ] # THE RUNTIME EXTRA — `split-opendox-two-layer-product` § 3.5, RULED Q2 diff --git a/src/opendox/runtime/bundle.py b/src/opendox/runtime/bundle.py index d38338b4..53a73e06 100644 --- a/src/opendox/runtime/bundle.py +++ b/src/opendox/runtime/bundle.py @@ -16,16 +16,18 @@ packages and the server's own; (iv) it stops with the entry point. -THE SERVER'S OWN PACKAGE IS `pgserver` (pyproject.toml's `local` extra), and -only its BINARIES are used: `initdb` and `postgres` from the wheel's -`pginstall/bin`, found by `importlib.util.find_spec` without importing -`pgserver` at all. Its Python manager is deliberately not used — it daemonizes -the server through `pg_ctl`, which re-parents it away from this process -(against (i)), shares one server between processes by reference count and -stops it from `atexit` (which a SIGTERM never runs, against (iv)), and may put -the socket under the user's runtime directory instead of the state directory -(against 13.1). The binaries themselves link only libc and libz, so they run on -any manylinux2014 host, which a wheel that links the system's ICU does not. +THE SERVER'S OWN PACKAGE IS `pixeltable-pgserver` (pyproject.toml's `local` +extra; RULED openxFactory#656 `5916000030` item 2), and only its BINARIES are +used: PostgreSQL 16's `initdb` and `postgres` from the wheel's `pginstall/bin`, +found by `importlib.util.find_spec` without importing `pixeltable_pgserver` at +all. Its Python manager is deliberately not used. It daemonizes the server +through `pg_ctl`, which re-parents it away from this process (against (i)). It +shares one server between processes by reference count and stops it from +`atexit`, which a SIGTERM never runs (against (iv)). And it may put the socket +under the user's runtime directory, opened to 0777, instead of the state +directory (against 13.1). The binaries link only the C library (libc, libm, +libpthread, librt, libdl) and libz from the system, plus the wheel's own +vendored libpq, so they need no system PostgreSQL and no ICU. THE LIFECYCLE, IN FULL: @@ -33,10 +35,15 @@ renamed into place only when it has succeeded, so an interrupted first start never leaves a half-built cluster: the MIGRATION identity (`config.BUNDLE_OWNER_ROLE`) is the bootstrap superuser, local connections - are `trust` and host connections are `reject`, UTF-8 in the `C` locale. - Trust is safe BECAUSE of the socket: its directory is 0700, owned by the - user running the install, and the server opens no TCP port at all, so - reaching the socket is the credential and nothing else can. + are `peer` and host connections are `reject`, UTF-8 in the `C` locale. + * PEER AUTHENTICATION, re-asserted before every launch (RULED + openxFactory#656 `5916000030` item 3). `pg_hba.conf` admits Unix-socket + connections through the `opendox` map only, and `pg_ident.conf`'s map + admits THIS install's OS user as the two roles and nobody else. The kernel + reports the connecting process's uid (`SO_PEERCRED`), so no password + exists to leak or to store, and a process of any other user is refused + even where it could reach the socket. The socket's directory is 0700 + besides, and the server opens no TCP port at all. * `postgres` started as a DIRECT CHILD of this process (`subprocess.Popen`, never `pg_ctl`), with `listen_addresses` empty and the socket directory given. On Linux it also carries `PR_SET_PDEATHSIG`, so an entry point @@ -88,8 +95,13 @@ database_bundle, ) -#: The distribution the `local` extra installs for the server's binaries. -SERVER_DISTRIBUTION = "pgserver" +#: The distribution the `local` extra installs for the server's binaries, and +#: the package it installs them under. +SERVER_DISTRIBUTION = "pixeltable-pgserver" +SERVER_PACKAGE = "pixeltable_pgserver" + +#: The `pg_ident.conf` map `pg_hba.conf`'s one local line authenticates through. +IDENT_MAP = "opendox" #: How long a start may take before it is a failure: `initdb` on a slow disk, #: plus the server's own recovery on a data directory an earlier run did not @@ -114,11 +126,11 @@ class BundleRefused(Exception): def server_binaries() -> Path: """The directory holding the bundled `initdb` and `postgres`, or a refusal. - Found WITHOUT importing `pgserver`: its package initializer imports its - manager, which this module does not use and whose import creates a lock - object under the user's runtime directory as a side effect. + Found WITHOUT importing `pixeltable_pgserver`: its package initializer + imports its manager, which this module does not use and which registers an + `atexit` handler and reaches for the user's runtime directory. """ - spec = importlib.util.find_spec(SERVER_DISTRIBUTION) + spec = importlib.util.find_spec(SERVER_PACKAGE) # THE FIRST LOCATION, OR NONE, read without an index, so no path reaches # a subscript that could raise (SonarCloud S6466 on openDox-code#69). location = next(iter(spec.submodule_search_locations or ()), None) \ @@ -332,6 +344,78 @@ def _preexec() -> None: # pragma: no cover - runs in the child BUNDLE_TREE = BUNDLE_SOCKET_DIR.parts +def os_user() -> str: + """The name of the OS user this runs as, which peer authentication maps. + + The SERVER resolves a connecting uid to a name through the same password + database, so a uid with no entry could not be authenticated at all, and is + refused here, by name. So is a name that `pg_ident.conf` could read as more + than a name: a regular expression (a leading `/`), a quote, a comment + mark, or white space. + """ + import pwd + + uid = os.getuid() + try: + name = pwd.getpwuid(uid).pw_name + except KeyError: + raise BundleRefused( + f"uid {uid} has no entry in the password database, and the bundled " + "server's peer authentication maps the connecting user BY NAME, so " + "it could never admit this one. Run the local install as a user " + "the system knows") from None + if not name or name.startswith("/") or any( + ch in name for ch in '"#') or any(ch.isspace() for ch in name): + raise BundleRefused( + f"the OS user name {name!r} cannot be written into the bundled " + "server's pg_ident.conf as a plain name (it holds a quote, a `#`, " + "white space, or starts with `/`)") + return name + + +def authentication_files(user: str) -> dict[str, str]: + """`pg_hba.conf` and `pg_ident.conf` for a local install run by `user`. + + RULED openxFactory#656 `5916000030` item 3 ("Peer auth + accept"): + * ONE local line, PEER through the `opendox` map. The kernel reports the + connecting process's uid, and the map admits `user` as the owner role + and as the served role, and nobody else as anybody. + * Every HOST connection is rejected. The server also listens on no TCP + address at all (`listen_addresses` is empty), so these lines never + match. They are written so that the file says what the install is. + * No replication line, so a replication connection is refused. + """ + header = ("# Written by opendox.runtime.bundle before every start of this local\n" + "# install's bundled server (plan 034 T072). Changes here are replaced.\n") + hba = (header + + "# TYPE DATABASE USER ADDRESS METHOD\n" + f"local all all peer map={IDENT_MAP}\n" + "host all all 0.0.0.0/0 reject\n" + "host all all ::/0 reject\n") + ident = (header + + "# MAPNAME SYSTEM-USERNAME PG-USERNAME\n" + f'{IDENT_MAP} "{user}" {BUNDLE_OWNER_ROLE}\n' + f'{IDENT_MAP} "{user}" {BUNDLE_SERVED_ROLE}\n') + return {"pg_hba.conf": hba, "pg_ident.conf": ident} + + +def write_authentication(data_dir: Path, user: str) -> None: + """Write both files into `data_dir`, each replaced atomically, mode 0600. + + Before EVERY launch, not only after `initdb`: a data directory an earlier + build initialized, or a file edited by hand, is brought back to the one + configuration this install runs with. + """ + for name, content in authentication_files(user).items(): + target = data_dir / name + temporary = data_dir / f".{name}.opendox-{os.getpid()}" + descriptor = os.open(temporary, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, + 0o600) + with os.fdopen(descriptor, "w", encoding="utf-8") as handle: + handle.write(content) + os.replace(temporary, target) + + def _unsafe_because(info: os.stat_result, *, uid: int, own: bool) -> str | None: """Why one directory of the socket's path is unsafe, or `None`.""" mode = info.st_mode @@ -407,6 +491,8 @@ def start(self) -> BundledServer: self._prepare_directories() phase = "initializing its data directory" self._initialize(binaries) + phase = "configuring its authentication" + write_authentication(self.bundle.data_dir, os_user()) phase = "launching it" _remove_a_proven_stale_lock(self.bundle) self._launch(binaries) @@ -439,9 +525,9 @@ def _prepare_directories(self) -> None: A directory that already exists is NOT re-moded, on the rule the runtime's `init` keeps for an operator's own path — except the socket - directory, whose mode IS the access control of a `trust` server: it is - this install's own, under its own state directory, and it is narrowed - to 0700 whatever it was. + directory, which guards the only way in: it is this install's own, + under its own state directory, and it is narrowed to 0700 whatever it + was. """ for directory in (self.bundle.state_dir, self.bundle.data_dir.parent): directory.mkdir(mode=0o700, parents=True, exist_ok=True) @@ -454,11 +540,14 @@ def _prepare_directories(self) -> None: def _refuse_an_unsafe_tree(self) -> None: """The socket's whole path is this user's to change, or it is refused. - `trust` makes reaching the socket the credential, so the 0700 on the - socket directory is worth only what the path above it is worth. A - directory entry is controlled by its PARENT: a parent that another - user can write lets them rename `run` away, or put a symbolic link in - its place, after the mode is set. A symbolic link on the way there + The socket's directory is how this install's clients find ITS + server, so the 0700 on it is worth only what the path above it is + worth. Peer authentication keeps other users out of the server, but + not a substitute socket out of the path: whoever could replace `run` + could stand up a server of their own for this install's clients to + talk to. A directory entry is controlled by its PARENT: a parent that + another user can write lets them rename `run` away, or put a symbolic + link in its place, after the mode is set. A symbolic link on the way there can be pointed elsewhere by whoever owns it, or by whoever can write the directory it sits in (Copilot review of openDox-code#69). So: @@ -498,9 +587,9 @@ def _refuse_an_unsafe_tree(self) -> None: def _unsafe(directory: Path, reason: str) -> BundleRefused: return BundleRefused( f"{directory} {reason}, so another user could replace the bundled " - "server's socket directory, and reaching that socket is the only " - "credential the server asks for. Use a state directory only this " - f"user can change ({PREFIX}STATE_DIR)") + "server's socket directory and put a server of their own where " + "this install's clients look for it. Use a state directory only " + f"this user can change ({PREFIX}STATE_DIR)") #: The prefix an initialization attempt's directory carries, beside the #: data directory, followed by the pid of the process making it. @@ -565,7 +654,7 @@ def _remove_abandoned_attempts(self) -> None: def _initdb(self, binaries: Path, target: Path) -> None: done = subprocess.run( [str(binaries / "initdb"), "-D", str(target), - "-U", BUNDLE_OWNER_ROLE, "--auth-local=trust", "--auth-host=reject", + "-U", BUNDLE_OWNER_ROLE, "--auth-local=peer", "--auth-host=reject", "--encoding=UTF8", "--locale=C", "--no-instructions"], env=_child_environment(), capture_output=True, text=True, timeout=START_TIMEOUT_SECONDS) @@ -719,5 +808,6 @@ def __exit__(self, *exc: object) -> None: __all__ = ["BUNDLE_PORT", "BundleRefused", "BundledServer", - "isolated_from_libpq_environment", "report", "running_pid", - "server_binaries"] + "authentication_files", "isolated_from_libpq_environment", + "os_user", "report", "running_pid", "server_binaries", + "write_authentication"] diff --git a/src/opendox/runtime/config.py b/src/opendox/runtime/config.py index 36cd6272..42d99eb5 100644 --- a/src/opendox/runtime/config.py +++ b/src/opendox/runtime/config.py @@ -1665,10 +1665,15 @@ def dsn(self, role: str) -> str: `host` is the socket DIRECTORY (libpq's rule for a value that starts with `/`), percent-encoded so a state directory holding a space or a `&` is still one value; `port` is spelled so a `PGPORT` in the - environment cannot send libpq to a different socket file. No password: - the socket directory is 0700 and the server's own `pg_hba.conf` - trusts local connections only, so reaching the socket IS the - credential, and a host connection is rejected outright. + environment cannot send libpq to a different socket file. NO + PASSWORD, because there is none to give: the server authenticates a + Unix-socket connection by PEER (RULED openxFactory#656 `5916000030` + item 3). The kernel reports the connecting process's uid, and + `pg_ident.conf` maps this install's OS user, and nobody else, to the + two roles. The socket's directory is 0700, the server opens no TCP + port, and a host connection is rejected outright. SonarCloud's S2115 + ("add password protection") is ACCEPTED on this line for that reason, + with the same ruling as its authority. """ host = urllib.parse.quote(str(self.socket_dir), safe="/") return (f"postgresql://{role}@/{BUNDLE_DATABASE}" diff --git a/tests_runtime/test_bundled_postgres.py b/tests_runtime/test_bundled_postgres.py index 56e1f844..5fb6f5d2 100644 --- a/tests_runtime/test_bundled_postgres.py +++ b/tests_runtime/test_bundled_postgres.py @@ -36,6 +36,7 @@ import selectors import shutil import signal +import stat import subprocess import sys import sysconfig @@ -238,12 +239,16 @@ def test_the_local_extra_carries_the_runtime_and_the_server_and_test_joins_it( project = tomllib.loads((ROOT / "pyproject.toml").read_text()) extras = project["project"]["optional-dependencies"] assert "opendox[runtime]" in extras["local"] - assert any(req.startswith("pgserver") for req in extras["local"]) + # the carrier RULED on openxFactory#656 `5916000030` item 2 + assert any(req.startswith("pixeltable-pgserver") for req in extras["local"]) + assert not any(req.startswith("pgserver") for req in extras["local"]) assert "opendox[local]" in extras["test"], ( "F9.1 installs `.[test]` alone; without the local extra there, this " "suite could not start the server it tests") lock = (ROOT / "constraints-cpython312-linux.txt").read_text() - assert re.search(r"(?m)^pgserver==", lock), "the lock does not pin pgserver" + assert re.search(r"(?m)^pixeltable-pgserver==", lock), \ + "the lock does not pin pixeltable-pgserver" + assert not re.search(r"(?m)^pgserver==", lock), "the lock still pins pgserver" files = project["tool"]["setuptools"]["data-files"] assert files == {"share/opendox/migrations": ["migrations/*.sql"]} # and the packaging notes name functions that exist (Copilot review of #69) @@ -432,6 +437,90 @@ def test_a_stale_lock_naming_a_recycled_pid_does_not_hold_the_bundle( decoy.wait(timeout=10) +# -- peer authentication (RULED openxFactory#656 `5916000030` item 3) ---------- + + +def _owner(server) -> "object": + import psycopg + + return psycopg.connect(server.bundle.migration_dsn, autocommit=True) + + +def test_the_bundle_authenticates_by_peer_through_the_one_map( + state_dir: Path) -> None: + """The server's OWN reading of its two files, from `pg_hba_file_rules` + and `pg_ident_file_mappings`, and the method each connection really + used, from `system_user`. There is one local rule, peer through the + `opendox` map. Host is rejected. There is no `trust` anywhere. The map + admits this OS user as the two roles and names no other OS user.""" + user = bundle_mod.os_user() + settings = config.load_settings({MODE: "local", STATE: str(state_dir)}) + with bundle_mod.BundledServer(settings) as server: + with _owner(server) as conn: + rules = conn.execute( + "select type, database, user_name, auth_method, options, error " + "from pg_hba_file_rules order by rule_number").fetchall() + mappings = conn.execute( + "select map_name, sys_name, pg_username, error " + "from pg_ident_file_mappings order by map_number").fetchall() + for dsn, role in ((server.bundle.migration_dsn, config.BUNDLE_OWNER_ROLE), + (server.bundle.served_dsn, config.BUNDLE_SERVED_ROLE)): + import psycopg + + with psycopg.connect(dsn) as conn: + assert conn.execute("select current_user, system_user").fetchone() \ + == (role, f"peer:{user}") + assert rules == [ + ("local", ["all"], ["all"], "peer", [f"map={bundle_mod.IDENT_MAP}"], None), + ("host", ["all"], ["all"], "reject", None, None), + ("host", ["all"], ["all"], "reject", None, None)], rules + assert mappings == [ + (bundle_mod.IDENT_MAP, user, config.BUNDLE_OWNER_ROLE, None), + (bundle_mod.IDENT_MAP, user, config.BUNDLE_SERVED_ROLE, None)], mappings + + +def test_a_role_outside_the_map_is_refused_even_for_this_os_user( + state_dir: Path) -> None: + """The MAP decides, not the socket. The same OS user, over the same + 0700 socket, asking for a role the map does not name, is refused by + peer authentication. A suite that does not run as root cannot connect + as a second OS user. What stands for that case is the map itself, read + back above, which names this user and no other.""" + import psycopg + from psycopg import sql + + settings = config.load_settings({MODE: "local", STATE: str(state_dir)}) + with bundle_mod.BundledServer(settings) as server: + with _owner(server) as conn: + conn.execute(sql.SQL("create role {} login").format( + sql.Identifier("odx_stranger"))) + with pytest.raises(psycopg.OperationalError) as caught: + psycopg.connect(server.bundle.dsn("odx_stranger")).close() + assert "peer authentication failed" in str(caught.value).lower(), caught.value + + +def test_an_older_trust_cluster_is_brought_back_to_peer_on_start( + state_dir: Path) -> None: + """A data directory an earlier build initialized with `trust` (or a file + edited by hand) is put back to the one configuration before the next + launch, so it never serves as trust.""" + settings = config.load_settings({MODE: "local", STATE: str(state_dir)}) + bundle_mod.BundledServer(settings).start().stop() + data = config.DatabaseBundle(state_dir).data_dir + (data / "pg_hba.conf").write_text("local all all trust\n", encoding="utf-8") + (data / "pg_ident.conf").write_text("", encoding="utf-8") + with bundle_mod.BundledServer(settings) as server: + import psycopg + + with psycopg.connect(server.bundle.served_dsn) as conn: + method = conn.execute("select system_user").fetchone()[0] + expected = bundle_mod.authentication_files(bundle_mod.os_user()) + for name, content in expected.items(): + assert (data / name).read_text(encoding="utf-8") == content, name + assert stat.S_IMODE((data / name).stat().st_mode) == 0o600, name + assert method == f"peer:{bundle_mod.os_user()}", method + + def test_migrate_under_the_local_mode_uses_the_bundle_and_refuses_a_dsn( state_dir: Path) -> None: """`runtime migrate` is part of the same install: it reads the bundle's diff --git a/tests_runtime/test_local_lifecycle.py b/tests_runtime/test_local_lifecycle.py index 9285016d..baee0695 100644 --- a/tests_runtime/test_local_lifecycle.py +++ b/tests_runtime/test_local_lifecycle.py @@ -569,13 +569,14 @@ def test_both_classes_together_name_both_reasons() -> None: @pytest.mark.parametrize("found", ["absent", "no-locations"]) def test_a_missing_server_package_is_the_named_refusal( monkeypatch: pytest.MonkeyPatch, found: str) -> None: - """No `pgserver` at all, or a spec with no location: both are the one - refusal naming the `local` extra, never an `IndexError`.""" + """No `pixeltable_pgserver` at all, or a spec with no location: both are + the one refusal naming the `local` extra, never an `IndexError`.""" import importlib.machinery spec = None if found == "no-locations": - spec = importlib.machinery.ModuleSpec("pgserver", None, is_package=True) + spec = importlib.machinery.ModuleSpec(bundle_mod.SERVER_PACKAGE, None, + is_package=True) spec.submodule_search_locations = [] monkeypatch.setattr(bundle_mod.importlib.util, "find_spec", lambda name: spec) @@ -584,6 +585,74 @@ def test_a_missing_server_package_is_the_named_refusal( assert 'opendox[local]' in str(caught.value), caught.value +# -- peer authentication, hermetic ---------------------------------------------- + + +def test_the_authentication_files_admit_one_os_user_as_the_two_roles() -> None: + files = bundle_mod.authentication_files("alice") + active = {name: [line.split() for line in text.splitlines() + if line.strip() and not line.startswith("#")] + for name, text in files.items()} + assert active["pg_hba.conf"] == [ + ["local", "all", "all", "peer", f"map={bundle_mod.IDENT_MAP}"], + ["host", "all", "all", "0.0.0.0/0", "reject"], + ["host", "all", "all", "::/0", "reject"]] + assert active["pg_ident.conf"] == [ + [bundle_mod.IDENT_MAP, '"alice"', config.BUNDLE_OWNER_ROLE], + [bundle_mod.IDENT_MAP, '"alice"', config.BUNDLE_SERVED_ROLE]] + assert "trust" not in " ".join(" ".join(r) for rows in active.values() for r in rows) + + +def test_the_files_are_written_0600_and_replace_what_was_there( + tmp_path: Path) -> None: + (tmp_path / "pg_hba.conf").write_text("local all all trust\n") + bundle_mod.write_authentication(tmp_path, "alice") + for name, content in bundle_mod.authentication_files("alice").items(): + assert (tmp_path / name).read_text(encoding="utf-8") == content + assert stat.S_IMODE((tmp_path / name).stat().st_mode) == 0o600 + assert sorted(p.name for p in tmp_path.iterdir()) == ["pg_hba.conf", "pg_ident.conf"] + + +@pytest.mark.parametrize("name", ["/regex", 'quo"te', "has space", "hash#tag", ""]) +def test_an_os_user_name_the_map_cannot_hold_plainly_is_refused( + monkeypatch: pytest.MonkeyPatch, name: str) -> None: + import pwd + + class _Entry: + pw_name = name + + monkeypatch.setattr(pwd, "getpwuid", lambda uid: _Entry()) + with pytest.raises(bundle_mod.BundleRefused): + bundle_mod.os_user() + + +def test_a_uid_with_no_password_entry_is_refused_by_name( + monkeypatch: pytest.MonkeyPatch) -> None: + import pwd + + def _missing(uid): + raise KeyError(uid) + + monkeypatch.setattr(pwd, "getpwuid", _missing) + with pytest.raises(bundle_mod.BundleRefused) as caught: + bundle_mod.os_user() + assert "password database" in str(caught.value) + + +def test_initdb_is_asked_for_peer_and_host_reject( + monkeypatch, tmp_path: Path, short_state: Path) -> None: + """The flags `initdb` receives, recorded by a stand-in: local is `peer`, + host is `reject`, and `trust` is not asked for anywhere.""" + record = tmp_path / "initdb-argv" + server = _server(monkeypatch, tmp_path, short_state, initdb=( + f'echo "$@" > "{record}"\nexit 1')) + with pytest.raises(bundle_mod.BundleRefused): + server.start() + argv = record.read_text(encoding="utf-8").split() + assert "--auth-local=peer" in argv and "--auth-host=reject" in argv, argv + assert not [a for a in argv if "trust" in a], argv + + # -- initdb, and every phase of a start --------------------------------------- From f8e6e9e9ef23566cbf6046c4167a272307581827 Mon Sep 17 00:00:00 2001 From: Brett Heap <1513478+brettheap@users.noreply.github.com> Date: Wed, 30 Sep 2026 18:08:40 +0000 Subject: [PATCH 19/88] Fix round: the data path, fresh directories and readiness belong to this install (Copilot review) Copilot's review at 84a6c041 made three points, all real. - r4147680113: the tree check left out postgres/data. An existing data directory, a broken link included, now joins the own-tree check: a real directory, owned by this user, writable by no one else, not a link. A link to a cluster elsewhere would otherwise have been given this install's authentication files and launched outside the state tree. A fresh data directory needs no check, because _initialize renames it into place. - Fresh directories and the umask (overview, previously missed). mkdir(parents=True) creates intermediate directories with the default mode less the umask. Under umask 0002, a fresh ~/.local/state/opendox would create group-writable parents, which the tree check then refused. Each missing component is now created on its own and set to exactly 0700, whatever the umask. - Readiness (overview, previously missed). A successful connection proves only that some server answered. Two entry points racing from an idle state both launch, and the loser's postgres lives a moment while the winner's socket answers. So readiness now also needs the data directory's lock file to name this child. Otherwise the wait goes on until this child exits and is refused. One check after the connection is enough, since the lock admits one postmaster per data directory and the socket directory belongs to exactly one data directory. A before-check was tried and dropped: no mutant distinguishes it. Against 84a6c041's bundle.py, all 6 new cases fail. 4 mutants are killed. Full suite: 2619 selected, 2608 passed, 11 skipped, 0 failed. Lane: openxfactory-4 (openXfactory-4-openDox_extraction) Arc: neutral-product-standalone-operability Co-Authored-By: Claude Opus 5.5 (1M context) --- src/opendox/runtime/bundle.py | 60 ++++++++++++++++++++++++-- tests_runtime/test_bundled_postgres.py | 24 +++++++++++ tests_runtime/test_local_lifecycle.py | 52 ++++++++++++++++++++++ 3 files changed, 132 insertions(+), 4 deletions(-) diff --git a/src/opendox/runtime/bundle.py b/src/opendox/runtime/bundle.py index 53a73e06..f5ff03c0 100644 --- a/src/opendox/runtime/bundle.py +++ b/src/opendox/runtime/bundle.py @@ -84,6 +84,7 @@ from opendox.runtime import migrations from opendox.runtime.config import ( BUNDLE_DATABASE, + BUNDLE_DATA_DIR, BUNDLE_OWNER_ROLE, BUNDLE_PORT, BUNDLE_SERVED_ROLE, @@ -344,6 +345,31 @@ def _preexec() -> None: # pragma: no cover - runs in the child BUNDLE_TREE = BUNDLE_SOCKET_DIR.parts +def _make_private_directories(leaf: Path) -> None: + """`leaf` and every missing directory above it, each created 0700. + + `Path.mkdir(parents=True)` gives the directories it creates on the way + the default mode less the umask, whatever mode the leaf is given. So + under a common umask of 0002 a fresh `~/.local/state/opendox/...` would + create `.local` and `state` group-writable, and the tree check would + then refuse the directories this install had just made (Copilot review + of openDox-code#69). Each missing component is created here, one at a + time, and set to exactly 0700, whatever the umask is. A directory that + already exists is left as it is, and the tree check judges it. + """ + missing = [] + for directory in (leaf, *leaf.parents): + if os.path.lexists(directory): + break + missing.append(directory) + for directory in reversed(missing): + try: + directory.mkdir(mode=0o700) + except FileExistsError: + continue # made by a concurrent start; judged below + os.chmod(directory, 0o700) + + def os_user() -> str: """The name of the OS user this runs as, which peer authentication maps. @@ -529,9 +555,7 @@ def _prepare_directories(self) -> None: under its own state directory, and it is narrowed to 0700 whatever it was. """ - for directory in (self.bundle.state_dir, self.bundle.data_dir.parent): - directory.mkdir(mode=0o700, parents=True, exist_ok=True) - self.bundle.socket_dir.mkdir(mode=0o700, parents=True, exist_ok=True) + _make_private_directories(self.bundle.socket_dir) # CHECKED BEFORE THE CHMOD, which follows a symbolic link: a `run` # placed there as a link would otherwise have its TARGET re-moded. self._refuse_an_unsafe_tree() @@ -568,6 +592,14 @@ def _refuse_an_unsafe_tree(self) -> None: configured = self.bundle.state_dir state = configured.resolve() tree = [state, state / BUNDLE_TREE[0], state / BUNDLE_TREE[0] / BUNDLE_TREE[1]] + # AND THE DATA DIRECTORY, where one exists already, a broken link + # included (Copilot review of openDox-code#69). A `data` placed there + # as a link to a cluster elsewhere would otherwise be launched, and + # given this install's authentication files, outside the state tree. + # A fresh one needs no check: `_initialize` renames it into place. + data = state / BUNDLE_DATA_DIR + if os.path.lexists(data): + tree.append(data) checks = [(path, True) for path in tree] + [ (path, False) for path in dict.fromkeys( [*state.parents, *configured.parents])] @@ -701,7 +733,22 @@ def _wait_until_ready(self) -> None: with psycopg.connect(self._dsn(BUNDLE_OWNER_ROLE, "postgres"), connect_timeout=2, autocommit=True) as conn: conn.execute("select 1") - return + # READY MEANS THIS CHILD IS SERVING, not merely that the socket + # answered (Copilot review of openDox-code#69). Two entry points + # racing from an idle state both launch. The loser's `postgres` + # lives a moment before it refuses the winner's lock, and the + # winner's socket already answers. So once a connection has + # answered, the data directory's lock file must name THIS + # child. Otherwise the wait goes on until this child exits and + # is refused. One check, AFTER the connection, is enough. The + # lock admits one postmaster per data directory, the socket + # directory belongs to exactly one data directory, and a + # lock naming this child therefore means the socket that + # answered is this child's. + if self._serving_is_this_child(): + return + last = "the socket answered, but not from this child" + time.sleep(0.1) except psycopg.OperationalError as exc: # THE CLASS NAME ONLY: a driver's message quotes the DSN it # could not reach, and this package never repeats one @@ -713,6 +760,11 @@ def _wait_until_ready(self) -> None: f"{START_TIMEOUT_SECONDS:.0f}s ({last}); its log is " f"{self.log_path}") + def _serving_is_this_child(self) -> bool: + """Whether the data directory's lock file names the child just launched.""" + return (self.process is not None + and _lock_file_pid(self.bundle) == self.process.pid) + def _dsn(self, role: str, database: str) -> str: """`DatabaseBundle.dsn`, aimed at a database other than the served one.""" return self.bundle.dsn(role).replace( diff --git a/tests_runtime/test_bundled_postgres.py b/tests_runtime/test_bundled_postgres.py index 5fb6f5d2..62b6b3b8 100644 --- a/tests_runtime/test_bundled_postgres.py +++ b/tests_runtime/test_bundled_postgres.py @@ -437,6 +437,30 @@ def test_a_stale_lock_naming_a_recycled_pid_does_not_hold_the_bundle( decoy.wait(timeout=10) +def test_readiness_is_this_childs_server_not_a_winners_socket( + state_dir: Path) -> None: + """Two entry points racing from an idle state both launch. The loser's + `postgres` lives a moment before it refuses the winner's lock, while the + winner's socket already answers. The loser must wait for ITS child, and + be refused when that child exits, not connect to the winner and carry on + as if it owned a database (Copilot review of openDox-code#69). The + loser's child is stood in by a process that lives three seconds.""" + settings = config.load_settings({MODE: "local", STATE: str(state_dir)}) + with bundle_mod.BundledServer(settings): + loser = bundle_mod.BundledServer(settings) + loser.process = subprocess.Popen( + [sys.executable, "-c", "import time; time.sleep(3)"]) + began = time.monotonic() + try: + with pytest.raises(bundle_mod.BundleRefused) as caught: + loser._wait_until_ready() + finally: + loser.process.kill() + loser.process.wait(timeout=10) + assert "exited during start" in str(caught.value), caught.value + assert time.monotonic() - began < bundle_mod.START_TIMEOUT_SECONDS + + # -- peer authentication (RULED openxFactory#656 `5916000030` item 3) ---------- diff --git a/tests_runtime/test_local_lifecycle.py b/tests_runtime/test_local_lifecycle.py index baee0695..58696b0b 100644 --- a/tests_runtime/test_local_lifecycle.py +++ b/tests_runtime/test_local_lifecycle.py @@ -524,6 +524,58 @@ def _lstat(path, *args, **kwargs): assert str(link) in str(caught.value) and "symbolic link owned by" in str(caught.value) +@pytest.mark.parametrize("umask", [0o002, 0o200]) +def test_missing_directories_are_created_0700_whatever_the_umask( + monkeypatch, tmp_path: Path, short_state: Path, umask: int) -> None: + """A fresh default path creates the directories above the state tree + too. Under umask 0002 `mkdir(parents=True)` would make them 0775, and + the tree check would then refuse what this install had just made + (Copilot review of #69). A umask that takes the owner's own bits would + leave them unusable. Each is created exactly 0700, and the start reaches + `initdb`, which the stand-in fails on purpose.""" + state = short_state / "a" / "b" / "state" + server = _prepared(monkeypatch, tmp_path, state) + previous = os.umask(umask) + try: + with pytest.raises(bundle_mod.BundleRefused) as caught: + server.start() + finally: + os.umask(previous) + assert "initdb" in str(caught.value), caught.value + for directory in (short_state / "a", short_state / "a" / "b", state, + state / "postgres", state / "postgres" / "run"): + assert stat.S_IMODE(directory.stat().st_mode) == 0o700, directory + + +@pytest.mark.parametrize("shape", ["link", "broken-link", "open"]) +def test_a_data_directory_that_is_not_this_installs_own_is_refused( + monkeypatch, tmp_path: Path, short_state: Path, shape: str) -> None: + """An existing `data` joins the tree check (Copilot review of #69): as a + link to a cluster elsewhere, as a broken link, or as a directory others + can write. It is refused before anything is written into it or launched + on it.""" + elsewhere = tmp_path / "cluster-elsewhere" + (short_state / "postgres").mkdir(mode=0o700) + data = short_state / "postgres" / "data" + if shape in {"link", "open"}: + target = elsewhere if shape == "link" else data + target.mkdir(mode=0o700) + (target / "PG_VERSION").write_text("16\n", encoding="utf-8") + if shape in {"link", "broken-link"}: + data.symlink_to(elsewhere) + if shape == "open": + data.chmod(0o777) + server = _prepared(monkeypatch, tmp_path, short_state) + with pytest.raises(bundle_mod.BundleRefused) as caught: + server.start() + message = str(caught.value) + assert str(data) in message, message + assert ("symbolic link" if shape != "open" else "writable by every user") \ + in message, message + if shape == "link": + assert not (elsewhere / "pg_hba.conf").exists(), "wrote into the link's target" + + @pytest.mark.parametrize("variable", [STATE, "XDG_STATE_HOME"]) def test_parent_traversal_in_the_state_path_is_refused(variable: str) -> None: value = "/tmp/odx-a/../odx-b" From c8fac05e328bd879a220e43bd8c985a81a43b653 Mon Sep 17 00:00:00 2001 From: Brett Heap <1513478+brettheap@users.noreply.github.com> Date: Fri, 2 Oct 2026 20:04:39 +0000 Subject: [PATCH 20/88] T070, owed at the merge round: the standalone generate-and-open runs say --local Since this PR, generate-and-open with neither --local nor OPENDOX_INSTALL_MODE=local is a HOSTED install, which refuses without its broker's issuer (#1144 13.4, 13.5). Four cases that landed on main with phase 2 run generate-and-open as the single-user install and relied on the old default, so each now says --local: - tests/test_standalone_generate_path.py (T056), case 3: the server starts, answers and stops. The module docstring names the change and moves F10.1's plain-install run to T077. - tests/test_post_render_validator.py (T058), test_generate_and_open_gives_the_same_verdicts, both fixtures. - tests/test_projection_seams.py (T055), test_generate_and_open_refuses_an_empty_source_option_before_its_run_dir. No case means hosted, so none takes a hosted fixture. Before this commit, all four fail on the merged tree with the hosted issuer refusal; after it they pass. Three mutants of the local path are killed, each failing all four cases: --local ignored, local refusing its own loopback default, and local also asking for the hosted issuer. Full suite: 3117 selected, 3106 passed, 11 skipped, 0 failed. Lane: openxfactory-4 (openXfactory-4-openDox_extraction) Arc: neutral-product-standalone-operability Co-Authored-By: Claude Opus 5.5 (1M context) --- tests/test_post_render_validator.py | 10 ++++++---- tests/test_projection_seams.py | 8 +++++--- tests/test_standalone_generate_path.py | 19 +++++++++++++------ 3 files changed, 24 insertions(+), 13 deletions(-) diff --git a/tests/test_post_render_validator.py b/tests/test_post_render_validator.py index f2772edf..a18ddd22 100644 --- a/tests/test_post_render_validator.py +++ b/tests/test_post_render_validator.py @@ -141,11 +141,13 @@ def test_F7_2_the_malformed_fixture_is_refused_without_strict_too(tmp_path) -> N @pytest.mark.parametrize("fixture,expected", [(PLAIN, 0), (MALFORMED, 1)]) def test_generate_and_open_gives_the_same_verdicts(tmp_path, fixture, expected) -> None: - """`generate-and-open --no-open --no-serve --strict`: the good fixture - builds its server and prints its URL, and the malformed one stops before - a server is built, naming the rule.""" + """`generate-and-open --local --no-open --no-serve --strict`: the good + fixture builds its server and prints its URL, and the malformed one stops + before a server is built, naming the rule. `--local` because this is the + single-user install: since plan 034 T070, an unflagged run is HOSTED and + refuses without its broker's issuer, before the validator is reached.""" repo = fresh_repository(fixture, tmp_path) - child, status = run_module(tmp_path, "opendox.cli", "generate-and-open", + child, status = run_module(tmp_path, "opendox.cli", "generate-and-open", "--local", "--repo-root", str(repo), "--repository", "fixture", "--no-open", "--no-serve", "--strict", "--run-dir", str(tmp_path / "run")) diff --git a/tests/test_projection_seams.py b/tests/test_projection_seams.py index 1d5c5883..766a848a 100644 --- a/tests/test_projection_seams.py +++ b/tests/test_projection_seams.py @@ -1397,9 +1397,11 @@ def test_generate_and_open_refuses_an_empty_source_option_before_its_run_dir( _declaring_generator(calls) repo = _repository(tmp_path) run_dir = tmp_path / "run" - rc = cli.main(["generate-and-open", "--repo-root", str(repo), "--repository", - "garden", "--run-dir", str(run_dir), "--no-open", "--no-serve", - "--possibles", ""]) + # `--local`: the single-user install. Since plan 034 T070 an unflagged run + # is HOSTED, and its issuer refusal would come first. + rc = cli.main(["generate-and-open", "--local", "--repo-root", str(repo), + "--repository", "garden", "--run-dir", str(run_dir), + "--no-open", "--no-serve", "--possibles", ""]) assert rc == 1 assert ("generate-and-open refused: --possibles was given an empty path" in capsys.readouterr().err) diff --git a/tests/test_standalone_generate_path.py b/tests/test_standalone_generate_path.py index 0a95efca..b71f3650 100644 --- a/tests/test_standalone_generate_path.py +++ b/tests/test_standalone_generate_path.py @@ -17,7 +17,7 @@ role keys, the verb reports it, naming the document, the value and the six keys, and the snapshot it writes reads that document as a source. T054 tests the projection's half in process. -3. `python -m opendox.cli generate-and-open --no-open` STARTS a server, which +3. `python -m opendox.cli generate-and-open --local --no-open` STARTS a server, which answers `/index.html`, `/snapshot.json`, `/capabilities` and `/source/`, refuses `/source/.git/config`, and stops on an interrupt with status 0. 4. `python -m opendox.serve`, the server's own entry point, starts and answers @@ -46,8 +46,14 @@ seconds). T056 flushes it in both entry points, and cases 3 and 4 fail without that. -NOT HERE: F10.1's run through a plain install, with the console script and no -`--local`, arrives in phase 3 (T070, and T077 as batch H amends it). +`--local` (plan 034 T070; #1144 13.4, 13.5): case 3 is the single-user install, +so it says so. Since T070, `generate-and-open` with neither `--local` nor +`OPENDOX_INSTALL_MODE=local` is a HOSTED install, which refuses without its +broker's issuer. That refusal is what an unflagged run of this case would now +hit, and it is T070's own subject, held in `tests/test_install_mode_entrypoint.py`. + +NOT HERE: F10.1's run through a plain install, with the console script, arrives +in phase 3 (T077, as batch H amends it). A CREATED FILE: no carve-manifest row (RULED OQ-C). """ @@ -256,9 +262,10 @@ def test_the_unedited_fixture_declares_that_document_a_candidate(tmp_path) -> No # --------------------------------------------------------------------------- def test_generate_and_open_starts_a_server_that_answers_with_no_sibling(tmp_path) -> None: - """`python -m opendox.cli generate-and-open --no-open`, with no + """`python -m opendox.cli generate-and-open --local --no-open`, with no `--no-serve`: the server starts, says where on a buffered pipe, answers - the core routes, and stops on an interrupt with status 0. + the core routes, and stops on an interrupt with status 0. `--local` + because this is the single-user install (T070). Both lines it prints before blocking in `serve_forever()`, the URL and "serving until interrupted", are read WHILE IT RUNS, before the @@ -267,7 +274,7 @@ def test_generate_and_open_starts_a_server_that_answers_with_no_sibling(tmp_path e3574774, r4146289331).""" repo = _fresh_repository(tmp_path) run_dir = tmp_path / "run" - child = Child(tmp_path, "opendox.cli", "generate-and-open", + child = Child(tmp_path, "opendox.cli", "generate-and-open", "--local", "--repo-root", str(repo), "--repository", "fixture", "--no-open", "--port", "0", "--run-dir", str(run_dir)) try: From 2210739b507e77dff2cf48444fe18d2b29c1a0ee Mon Sep 17 00:00:00 2001 From: Brett Heap <1513478+brettheap@users.noreply.github.com> Date: Fri, 2 Oct 2026 20:09:38 +0000 Subject: [PATCH 21/88] T084: the rejection report prints every broken rule once, with its count MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit cli._report_non_conformance printed the validator's last 20 lines. So a snapshot that broke one rule many times and a second rule once showed copies of the first and never named the second. Now each rule id the report names is printed once, on its own line, ` × [] : `, in the order found and with where it is first broken. The next places that rule is broken follow beneath it without the id, five in all, then "… and N more of this rule". One rule can be broken in different ways, and a count beside the first place alone would read as that place repeated. The validator's own summary follows, and a report that names no rule id prints its own last lines as before. RULED openxFactory#656 5920216845, item 3 ("Show every rule, grouped (Recommended)"). No #1144 line moves: F7.2 asserts the fixture's rule id is printed, which stays true. The new module tests/test_rejection_report.py sits clear of tests/test_post_render_validator.py, which is T085's. Before the change: 3 failed, 1 passed. After: 6 passed, with test_post_render_validator.py still 50 passed. tests/test_projection_seams.py's envelope-keys case now asserts the grouped line, that the id appears once, and that the `documents` key is still named. The always-1 and never-groups mutants each fail 5 cases, and the drops-places mutant fails 3. Arc: neutral-product-standalone-operability Lane: openxfactory-4 (openXfactory-4-openDox_extraction) Co-Authored-By: Claude Opus 5.5 (1M context) --- src/opendox/cli.py | 54 +++++++- tests/test_projection_seams.py | 7 +- tests/test_rejection_report.py | 218 +++++++++++++++++++++++++++++++++ 3 files changed, 276 insertions(+), 3 deletions(-) create mode 100644 tests/test_rejection_report.py diff --git a/src/opendox/cli.py b/src/opendox/cli.py index 8bd0fe08..cd1b7092 100644 --- a/src/opendox/cli.py +++ b/src/opendox/cli.py @@ -19,6 +19,7 @@ import argparse import json import os +import re import sys import tempfile import webbrowser @@ -478,16 +479,65 @@ def _warn_validator_could_not_run(result, validator) -> None: print(f" {sys.executable} -m {remedy}", file=sys.stderr) +#: One broken rule, as a validator's report names it: `[] : +#: `, the line `opendox.validator.Violation.line()` prints. +_RULE_LINE = re.compile(r"^\[(?P[^\[\]\s]+)\] (?P.+)$") + +#: How many of a report's other lines (its summary, or a validator's own +#: words where it names no rule) are printed. +_REPORT_TAIL = 20 + +#: How many places each broken rule is shown at: the first on the rule's own +#: line, and the next ones beneath it. A rule broken at more places than this +#: says how many more, so its count stays exact and the report stays short. +_PLACES_SHOWN = 5 + + def _report_non_conformance(written: Path, result) -> None: """NOT CONFORMANT — the validator ran, reached a verdict, and rejected the snapshot. The one thing this message must never be mistaken for is the warning above it, so it says whose fault it is out loud and prints the findings themselves; "1 error(s)" alone told a human nothing he could act - on.""" + on. + + EVERY BROKEN RULE, ONCE, WITH ITS COUNT (plan 034 T084; RULED + openxFactory#656 `5920216845`, item 3, *"Show every rule, grouped + (Recommended)"*). This printed the validator's LAST 20 LINES, so a + snapshot that broke one rule a hundred times and a second rule once + showed twenty copies of the first and never named the second. Now each + rule id the report names is printed ONCE, on a line of its own, + ` × [] : `, in the order the validator found + them, with where it is first broken. The next places it is broken follow + beneath it, without the id, up to `_PLACES_SHOWN` in all, because one rule + can be broken in different ways (a missing key, then another), and a + count beside the first place alone would read as that place repeated. + The report's other lines (the validator's summary) follow. A validator + whose output names no rule id has nothing to group, so its own last lines + are printed, as before.""" print(f" validation FAILED — the pinned validator REJECTED {written}. This " f"is the SNAPSHOT, not the environment: the validator ran fine and " f"found the data non-conformant.", file=sys.stderr) - for line in (result.stdout or result.stderr).strip().splitlines()[-20:]: + lines = (result.stdout or result.stderr).strip().splitlines() + places: dict[str, list[str]] = {} + others: list[str] = [] + for line in lines: + named = _RULE_LINE.match(line.strip()) + if named is None: + others.append(line) + continue + places.setdefault(named["rule"], []).append(named["rest"]) + if places: + total = sum(len(where) for where in places.values()) + print(f" {total} violation(s) of {len(places)} rule(s), each rule " + f"once, with its count and where it is broken:", file=sys.stderr) + for rule, where in places.items(): + print(f" {len(where)} × [{rule}] {where[0]}", file=sys.stderr) + for place in where[1:_PLACES_SHOWN]: + print(f" {place}", file=sys.stderr) + if len(where) > _PLACES_SHOWN: + print(f" … and {len(where) - _PLACES_SHOWN} more of " + "this rule", file=sys.stderr) + for line in others[-_REPORT_TAIL:]: print(f" {line}", file=sys.stderr) diff --git a/tests/test_projection_seams.py b/tests/test_projection_seams.py index 1d5c5883..b82d4ab9 100644 --- a/tests/test_projection_seams.py +++ b/tests/test_projection_seams.py @@ -1510,7 +1510,12 @@ def test_openDoxs_own_kind_meets_openDoxs_own_validator(tmp_path, capsys) -> Non assert cli._validate(written, _validate_args(tmp_path)) == 1 err = capsys.readouterr().err assert "REJECTED" in err and "This is the SNAPSHOT" in err - assert "[envelope-keys] : 'documents' is required" in err, err + # ONE line per broken rule, with its count (T084; RULED 5920216845 item + # 3), and each further place it is broken beneath it, so every missing + # key is still named. + assert "6 × [envelope-keys] : " in err, err + assert err.count("[envelope-keys]") == 1, err + assert ": 'documents' is required" in err, err assert "6 violation(s) of the opendox-snapshot contract, by opendox.validator" in err assert "validation SKIPPED" not in err diff --git a/tests/test_rejection_report.py b/tests/test_rejection_report.py new file mode 100644 index 00000000..fefecdc8 --- /dev/null +++ b/tests/test_rejection_report.py @@ -0,0 +1,218 @@ +"""The rejection report: every broken rule, once, with its count (plan 034 +T084; RULED openxFactory#656 `5920216845`, item 3, *"Show every rule, grouped +(Recommended)"*). + +`cli._report_non_conformance` is what the generate verbs print when the +validator registered for a snapshot's kind rejects it. It printed the LAST 20 +LINES of the validator's output, so a snapshot that broke one rule a hundred +times and a second rule once showed twenty copies of the first and never named +the second. Now each broken rule id is printed ONCE, with its exact count and +where it is first broken, in the order the validator found them, and the +validator's own summary line follows. + +The cases: + +1. THE RULING'S CASE. A rejected snapshot that breaks one rule several times + and a second rule once, run through `cli._validate`, the function both + generate verbs call, with openDox's own validator registered for the + snapshot's kind: each rule id appears ONCE, with its exact count. So an + implementation that always prints `1`, or never groups a repeated id, + fails it. The snapshot is the one `python -m opendox.cli generate` writes + over a corpus with four empty titles and summaries (rule A, four times), + with ONE more break of a second rule written into it, because openDox's + projection does not let a corpus break that second rule at all. +2. THE VERB ITSELF, in a child with neither sibling importable + (`tests/standalone_child.py`): `generate --strict` over that corpus exits 1 + and names its one rule once, with the count 4. +3. A VALIDATOR THAT NAMES NO RULE ID (a host's, say) still has its own last + lines printed, as before: there is nothing to group, and nothing is hidden. + +A module of its own, clear of `tests/test_post_render_validator.py` (which +T085 edits). F7.2's assertions there still hold: the fixture's rule id is +printed with where it is first broken, and so is the validator's summary. + +A CREATED FILE: no carve-manifest row (RULED OQ-C). +""" + +from __future__ import annotations + +import argparse +import json +import re +from pathlib import Path + +import pytest + +from opendox import cli +from opendox import projection_seams as ps +from standalone_child import fresh_repository, run_module + +ROOT = Path(__file__).resolve().parent.parent +PLAIN = ROOT / "tests" / "fixtures" / "plain-documents" + +#: The rule the corpus below breaks four times, and the rule the written +#: snapshot then breaks once more. +REPEATED = "title-and-summary-are-text" +ONCE = "repository-is-text" + +#: Three empty titles and one empty summary: four breaks of `REPEATED`. +_EMPTIED = { + "notes-rain-barrel-leak.md": "title", + "notes-rain-barrel-overflow.md": "title", + "notes-toolshed-inventory.md": "title", + "grouping-compost-corner.md": "summary", +} + +#: One reported rule: ` × [] : `. +_GROUPED = re.compile(r"^ (?P[0-9]+) × \[(?P[^\]]+)\] (?P.+)$") + + +def _emptied_corpus(parent: Path) -> Path: + edits = {} + for name, field in _EMPTIED.items(): + text = (PLAIN / name).read_text(encoding="utf-8") + edits[name] = re.sub(rf"(?m)^{field}: .*$", f"{field}:", text, count=1) + assert edits[name] != text, f"{name} carries no {field}: line to empty" + return fresh_repository(PLAIN, parent, edits=edits) + + +@pytest.fixture() +def seams(): + """openDox's own defaults at the projection seams, and nothing left + behind: the seams are put back exactly as each case found them.""" + held = {name: getattr(ps, name) for name in ("registry", "corpus_root", "writer")} + found = {name: (seam._registered, seam._is_default, seam._default_read) + for name, seam in held.items()} + kinds = dict(ps.validators._registered) + read = set(ps.validators._default_read) + for seam in held.values(): + seam.unregister() + ps.validators.unregister() + ps.register_defaults() + try: + yield + finally: + for name, seam in held.items(): + seam._registered, seam._is_default, seam._default_read = found[name] + ps.validators._registered.clear() + ps.validators._registered.update(kinds) + ps.validators._default_read.clear() + ps.validators._default_read.update(read) + + +def _written_snapshot(tmp_path: Path) -> Path: + """The snapshot the real verb writes over the emptied corpus, unvalidated.""" + repo = _emptied_corpus(tmp_path) + out = tmp_path / "snapshot.json" + child, status = run_module( + tmp_path, "opendox.cli", "generate", "--repo-root", str(repo), + "--repository", "fixture", "--output", str(out), "--no-validate") + assert status == 0, child.stderr_text() + assert child.refused() == [], child.refused() + return out + + +def _grouped(err: str) -> list[tuple[str, int, str]]: + return [(m["rule"], int(m["count"]), m["rest"]) + for m in map(_GROUPED.match, err.splitlines()) if m] + + +def test_each_broken_rule_is_printed_once_with_its_exact_count( + tmp_path, seams, capsys) -> None: + """The ruling's case: one rule broken four times, a second once.""" + written = _written_snapshot(tmp_path) + snapshot = json.loads(written.read_text(encoding="utf-8")) + snapshot["repository"] = 7 # ONE break of `ONCE` + written.write_text(json.dumps(snapshot), encoding="utf-8") + args = argparse.Namespace(no_validate=False, strict=True, + repo_root=str(tmp_path / PLAIN.name)) + assert cli._validate(written, args) == 1 + err = capsys.readouterr().err + grouped = _grouped(err) + assert sorted((rule, count) for rule, count, _ in grouped) == sorted([ + (ONCE, 1), (REPEATED, 4)]), err + # ONCE EACH: no rule id is named on any other line of the report. + for rule in (REPEATED, ONCE): + assert len(re.findall(re.escape(f"[{rule}]"), err)) == 1, err + # where each is FIRST broken, in the validator's own words + assert dict((rule, rest) for rule, _, rest in grouped)[REPEATED].startswith( + "/documents/"), err + assert dict((rule, rest) for rule, _, rest in grouped)[ONCE].startswith( + "/repository:"), err + assert "5 violation(s) of 2 rule(s)" in err, err + # the validator's own summary line still follows + assert "5 violation(s) of the opendox-snapshot contract" in err, err + assert "the pinned validator REJECTED" in err and "This is the SNAPSHOT" in err + + +def test_the_verb_names_its_one_rule_once_with_its_count(tmp_path) -> None: + """`python -m opendox.cli generate --strict`, with neither sibling + importable, over the corpus that breaks one rule four times.""" + repo = _emptied_corpus(tmp_path) + out = tmp_path / "snapshot.json" + child, status = run_module( + tmp_path, "opendox.cli", "generate", "--repo-root", str(repo), + "--repository", "fixture", "--output", str(out), "--strict") + err = child.stderr_text() + assert status == 1, err + assert child.refused() == [], child.refused() + assert [(rule, count) for rule, count, _ in _grouped(err)] == [ + (REPEATED, 4)], err + assert len(re.findall(re.escape(f"[{REPEATED}]"), err)) == 1, err + assert "4 violation(s) of 1 rule(s)" in err, err + + +def test_a_report_that_names_no_rule_still_prints_its_own_last_lines( + capsys) -> None: + """A validator whose output names no rule id has nothing to group, so its + own last lines are printed, as before, and nothing is hidden.""" + lines = [f"line {n}: not conformant" for n in range(30)] + result = ps.ValidationResult(False, 1, "\n".join(lines) + "\n", "", + "a host's validator") + cli._report_non_conformance(Path("snapshot.json"), result) + err = capsys.readouterr().err + assert _grouped(err) == [], err + shown = [line.strip() for line in err.splitlines()[1:]] + assert shown == lines[-20:], err + + +def test_a_rule_broken_in_several_places_shows_each_place_beneath_it(capsys) -> None: + """Grouping keeps the order the validator found the rules in, counts every + line, names where each rule is first broken on the rule's own line, and + shows the next places beneath it without repeating the id: one rule can be + broken in different ways, and a count beside the first place alone would + read as that place repeated.""" + out = "\n".join([ + "[b-rule] /x/0: first b", + "[a-rule] /y: only a", + "[b-rule] /x/1: second b", + "[b-rule] /x/2: third b", + "4 violation(s) of the k contract, by v", + ]) + "\n" + result = ps.ValidationResult(False, 1, out, "", "v") + cli._report_non_conformance(Path("snapshot.json"), result) + err = capsys.readouterr().err + assert _grouped(err) == [("b-rule", 3, "/x/0: first b"), + ("a-rule", 1, "/y: only a")], err + body = err.splitlines() + at = body.index(" 3 × [b-rule] /x/0: first b") + assert body[at + 1:at + 4] == [" /x/1: second b", + " /x/2: third b", + " 1 × [a-rule] /y: only a"], err + assert err.count("[b-rule]") == 1 and err.count("[a-rule]") == 1, err + assert "4 violation(s) of 2 rule(s)" in err, err + assert err.rstrip().endswith("4 violation(s) of the k contract, by v"), err + + +def test_a_rule_broken_in_many_places_says_how_many_more(capsys) -> None: + """Past `_PLACES_SHOWN` places a rule says how many more it has, so the + report stays short and the count stays exact.""" + many = cli._PLACES_SHOWN + 7 + out = "".join(f"[c-rule] /z/{n}: broken\n" for n in range(many)) + result = ps.ValidationResult(False, 1, out, "", "v") + cli._report_non_conformance(Path("snapshot.json"), result) + err = capsys.readouterr().err + assert _grouped(err) == [("c-rule", many, "/z/0: broken")], err + shown = [line for line in err.splitlines() if line.startswith(" /z/")] + assert shown == [f" /z/{n}: broken" for n in range(1, cli._PLACES_SHOWN)], err + assert "… and 7 more of this rule" in err, err From 19e32f0cdb074e4ad12249f72858711403c8b0d9 Mon Sep 17 00:00:00 2001 From: Brett Heap <1513478+brettheap@users.noreply.github.com> Date: Fri, 2 Oct 2026 20:20:59 +0000 Subject: [PATCH 22/88] T072, owed at the merge round: the real entry point, and every --local child keeps its own state Phase 2 is on this stack's base, so four things T072 owed at its merge round are done. - The stand-ins go. tests_runtime/local_entrypoint_driver.py is deleted. Its stand-ins patched names that T055 has since replaced, so on the merged tree they stood in for nothing, and all three background cases failed: the real corpus-root check refused the stand-in corpus, a directory with no repository. test_bundled_postgres.py now launches `python -m opendox.cli generate-and-open --local`, with the validator on, over T050's tests/fixtures/plain-documents copied into a fresh repository, as F13.1's preamble does. - Every cheap refusal comes before the database start. main's T055 added _refuse_empty_source_options to the generate path, so the local path asks it before it builds the bundled server, beside the corpus-root and generated-at refusals. test_projection_seams.py's empty-option case now carries a tripwire bundle, so a regression neither starts a server nor passes. - No child touches the user's state directory. A `generate-and-open --local` child now starts the bundled server, and OPENDOX_STATE_DIR defaults to the user's own ~/.local/state/opendox. tests/standalone_child.py gives every child a fresh, short, private state directory under /tmp and removes it when the child is stopped. Measured before: the three --local children of T056 and T058 initialized a cluster in the (sandboxed) default state home. - T056's case 3 asserts that its bundled server's data directory is under the child's own state directory while serving, and that the directory is gone after the stop. Four mutants are killed. They drop the cheap refusal, the private state dir, its removal, and the fixture's repository. Full suite: 3195 selected, 3184 passed, 11 skipped, 0 failed. Nothing is left under ~/.local/state/opendox or /tmp/odx-child-*. Lane: openxfactory-4 (openXfactory-4-openDox_extraction) Arc: neutral-product-standalone-operability Co-Authored-By: Claude Opus 5.5 (1M context) --- src/opendox/cli.py | 5 +- tests/standalone_child.py | 12 ++++ tests/test_projection_seams.py | 20 +++++- tests/test_standalone_generate_path.py | 6 ++ tests_runtime/local_entrypoint_driver.py | 77 ------------------------ tests_runtime/test_bundled_postgres.py | 38 ++++++++---- 6 files changed, 68 insertions(+), 90 deletions(-) delete mode 100644 tests_runtime/local_entrypoint_driver.py diff --git a/src/opendox/cli.py b/src/opendox/cli.py index 67315a57..1438ac30 100644 --- a/src/opendox/cli.py +++ b/src/opendox/cli.py @@ -642,9 +642,12 @@ def cmd_generate_and_open(args: argparse.Namespace, *, opener=webbrowser.open) - args.runtime_settings = settings if settings.install_mode != runtime_config.INSTALL_MODE_LOCAL: return _generate_and_open(args, opener=opener) - # THE CHEAP REFUSALS FIRST, so a mistyped root never costs a database start. + # THE CHEAP REFUSALS FIRST, so a mistyped root never costs a database + # start: every one `_generate_and_open` asks before it mints its run + # directory, main's empty-source-option refusal (T055) included. _refuse_non_corpus_repo_root(args) _refuse_malformed_generated_at(args) + _refuse_empty_source_options(args) server = bundle_mod.BundledServer(settings) args.database_bundle = server # NO `PG*` DEFAULT REACHES THE BUNDLE'S CONNECTIONS while this process diff --git a/tests/standalone_child.py b/tests/standalone_child.py index 37d1fe5f..f8bb1c6a 100644 --- a/tests/standalone_child.py +++ b/tests/standalone_child.py @@ -20,6 +20,13 @@ * It is read on threads while it runs (`Child.wait_for_line`), so a server that never exits can still be asked where it serves, and then interrupted (`Child.interrupt`, SIGINT, as Ctrl-C sends). +* ITS STATE DIRECTORY IS ITS OWN. A `generate-and-open --local` child starts + the local install's bundled PostgreSQL server (plan 034 T072) under + `OPENDOX_STATE_DIR`, whose default is the USER's own state directory. So + every child is given a fresh, short, private one (`Child.state_dir`, under + `/tmp` because a Unix socket's whole path is bounded), and it is removed + once the child is stopped. No case ever initializes a database in the home + directory of whoever runs the suite. * CTRL-C REACHES IT AS IT WOULD AT A TERMINAL, whatever the runner's own disposition. The same `sitecustomize` sets SIGINT back to Python's KeyboardInterrupt handler. A runner started as a background job @@ -45,6 +52,7 @@ import signal import subprocess import sys +import tempfile import threading import time from pathlib import Path @@ -145,6 +153,9 @@ def __init__(self, workdir: Path, module: str, *args: str) -> None: env["PYTHONPATH"] = os.pathsep.join( [str(blocker), *filter(None, [env.get("PYTHONPATH")])]) env[REFUSED_LOG_ENV] = str(self.refused_log) + self.state_dir = Path(tempfile.mkdtemp( + prefix="odx-child-", dir="/tmp" if os.path.isdir("/tmp") else None)) + env["OPENDOX_STATE_DIR"] = str(self.state_dir) self.argv = [sys.executable, "-m", module, *args] verb = args[0] if args and not args[0].startswith("-") else "" self.label = f"python -m {module} {verb}".strip() @@ -221,6 +232,7 @@ def kill(self) -> None: if self.process.poll() is None: self.process.kill() self.process.wait(timeout=STOP_DEADLINE_SECONDS) + shutil.rmtree(self.state_dir, ignore_errors=True) def _join(self) -> None: for pump in self._pumps: diff --git a/tests/test_projection_seams.py b/tests/test_projection_seams.py index 766a848a..ac2f942d 100644 --- a/tests/test_projection_seams.py +++ b/tests/test_projection_seams.py @@ -1392,7 +1392,24 @@ def test_a_given_source_option_is_resolved_and_an_unset_one_is_not_passed( def test_generate_and_open_refuses_an_empty_source_option_before_its_run_dir( - tmp_path, capsys) -> None: + tmp_path, capsys, monkeypatch) -> None: + """And before the local install's bundled server (plan 034 T072): a + refused option costs no database start. A tripwire stands in for the + server, so a regression neither starts one nor passes.""" + started: list = [] + + class _Tripwire: + def __init__(self, settings) -> None: + started.append(settings) + + def start(self): + raise AssertionError("the bundled server was started for a " + "refused source option") + + def stop(self) -> None: + pass + + monkeypatch.setattr(cli.bundle_mod, "BundledServer", _Tripwire) calls: list = [] _declaring_generator(calls) repo = _repository(tmp_path) @@ -1406,6 +1423,7 @@ def test_generate_and_open_refuses_an_empty_source_option_before_its_run_dir( assert ("generate-and-open refused: --possibles was given an empty path" in capsys.readouterr().err) assert calls == [] and not run_dir.exists() + assert started == [], "a bundled server was built for a refused option" def test_a_root_openDoxs_predicate_refuses_is_refused_with_its_message(tmp_path, capsys) -> None: diff --git a/tests/test_standalone_generate_path.py b/tests/test_standalone_generate_path.py index b71f3650..60d50f0e 100644 --- a/tests/test_standalone_generate_path.py +++ b/tests/test_standalone_generate_path.py @@ -284,11 +284,17 @@ def test_generate_and_open_starts_a_server_that_answers_with_no_sibling(tmp_path child.wait_for_line(_SERVING) assert child.process.poll() is None, "the server exited after saying it serves" _assert_the_server_answers(base, run_dir / "snapshot.json", repo) + # THE LOCAL INSTALL'S DATABASE IS THE CHILD'S OWN (plan 034 T072): its + # bundled server was started under the private state directory the + # harness gave this child, never under the user's. + assert (child.state_dir / "postgres" / "data" / "PG_VERSION").is_file(), \ + "the bundled server was not started under the child's state dir" assert child.interrupt() == 0, child.stderr_text() finally: child.kill() _assert_the_port_is_closed(base) assert child.refused() == [], child.refused() + assert not child.state_dir.exists(), "the child's state dir outlived it" # --------------------------------------------------------------------------- diff --git a/tests_runtime/local_entrypoint_driver.py b/tests_runtime/local_entrypoint_driver.py deleted file mode 100644 index 43e6d63c..00000000 --- a/tests_runtime/local_entrypoint_driver.py +++ /dev/null @@ -1,77 +0,0 @@ -"""Run `opendox generate-and-open` for real, with ONLY its generation stood in. - -`tests_runtime/test_bundled_postgres.py` launches this file as a child process -(`python tests_runtime/local_entrypoint_driver.py generate-and-open --local …`) -so that F13.1's local probe can be run the way F13.1 runs it: in the -BACKGROUND, reached over HTTP, asked about by a SECOND process -(`runtime status`), and then stopped with a signal (plan 034 T072). - -WHAT IS STOOD IN, AND WHY IT IS ONLY THIS. At this stack's base, the generate -verbs still reach openXdox for three names (`corpus_root_refusal`, -`generate_snapshot`, `snapshot.write_snapshot`) and `serve` for two -(`_checkout_real`, `registry_mod`'s binding constants): phase 2's T055 and T056 -(openDox-code#59 and its successor) give openDox its own, and a lone checkout -cannot ask for them before then. They are replaced here exactly as -`tests/test_doxbench_entrypoint.py` replaces them, and NOTHING ELSE is: -`cli.main`, the install shape, the bundled server, `build_server` and the serve -loop all run as a user's `opendox generate-and-open --local` runs them. Once -T055 and T056 have landed on this branch's base, this driver's stand-ins go and -the probe runs the real entry point on `tests/fixtures/plain-documents`. -""" - -from __future__ import annotations - -import json -import sys -import types -from pathlib import Path - - -class _StandInSource: - """`build_server`'s snapshot source, as T011's `standalone` fixture has it.""" - - refresh_binding = None - baked_repository = None - - class registry: - active = None - - def bootstrap(self): - pass - - -def _generate_snapshot(repo_root, repository, *, source_revision=None, - **_ignored): - return {"repository": repository, - "generation": {"source_revision": source_revision}, - "documents": [{"id": "stand-in.md"}]} - - -def _write_snapshot(snapshot, output, boundary): - Path(output).write_text(json.dumps(snapshot), encoding="utf-8") - return output - - -def main(argv: list[str]) -> int: - from opendox import cli as cli_mod - from opendox import serve as serve_mod - - cli_mod.corpus_root_refusal = lambda root, shape=None: None - cli_mod.generate_snapshot = _generate_snapshot - cli_mod.snapshot_mod = types.SimpleNamespace(write_snapshot=_write_snapshot) - serve_mod._checkout_real = lambda root: False - serve_mod.registry_mod = types.SimpleNamespace( - BINDING_REGENERATE="regenerate", BINDING_REFETCH="refetch") - real_build_server = serve_mod.build_server - - def _build_server(*args, **kwargs): - kwargs.setdefault("snapshot_source", _StandInSource()) - return real_build_server(*args, **kwargs) - - serve_mod.build_server = _build_server - return cli_mod.main(argv) - - -if __name__ == "__main__": - sys.stdout.reconfigure(line_buffering=True) - sys.exit(main(sys.argv[1:])) diff --git a/tests_runtime/test_bundled_postgres.py b/tests_runtime/test_bundled_postgres.py index 62b6b3b8..bae2a7f5 100644 --- a/tests_runtime/test_bundled_postgres.py +++ b/tests_runtime/test_bundled_postgres.py @@ -3,9 +3,13 @@ T072's falsifier is F13.1's TCP-listener block, which reads the kernel's socket table at run time, and its `runtime status` block. Both are run here -against a server the REAL entry point started: `generate-and-open --local`, -launched in the background as F13.1 launches it, reached over HTTP, asked -about by a second process, and stopped with a signal. Where F13.1 reads the +against a server the REAL entry point started: `python -m opendox.cli +generate-and-open --local` over a fresh repository copied from T050's +`tests/fixtures/plain-documents`, launched in the background as F13.1 launches +it, reached over HTTP, asked about by a second process, and stopped with a +signal. Nothing is stood in: the corpus root check, the generation, the +validator and the serve loop are phase 2's own, landed on this stack's base +(T054 to T058), so the stand-in driver this module once launched is gone. Where F13.1 reads the server's pid from `caps.json`, these cases read the same pid from `runtime status`'s `database_bundle`. `/capabilities`' `install` block is T073's, and nothing here pretends it exists. @@ -54,7 +58,8 @@ ROOT = Path(__file__).resolve().parents[1] SRC = ROOT / "src" -DRIVER = Path(__file__).resolve().parent / "local_entrypoint_driver.py" +#: T050's fixture, which F13.1's preamble copies into a fresh repository. +PLAIN_DOCUMENTS = ROOT / "tests" / "fixtures" / "plain-documents" MODE = PREFIX + "INSTALL_MODE" STATE = PREFIX + "STATE_DIR" @@ -174,10 +179,9 @@ def _launch(corpus: Path, state: Path, run_dir: Path, **extra: str) -> tuple[subprocess.Popen, str]: """`generate-and-open --local` in the BACKGROUND, and the URL it serves.""" child = subprocess.Popen( - [sys.executable, str(DRIVER), "generate-and-open", config.LOCAL_FLAG, - "--repo-root", str(corpus), "--repository", "fixture", - "--run-dir", str(run_dir), "--no-open", "--no-validate", - "--port", "0"], + [sys.executable, "-m", "opendox.cli", "generate-and-open", + config.LOCAL_FLAG, "--repo-root", str(corpus), "--repository", + "fixture", "--run-dir", str(run_dir), "--no-open", "--port", "0"], env=_clean_env(**{STATE: str(state)}, **extra), cwd=ROOT, stdout=subprocess.PIPE, stderr=subprocess.PIPE) url = _first_url(child, 90) @@ -225,9 +229,21 @@ def _status(state: Path, **extra: str) -> tuple[int, dict]: @pytest.fixture() def corpus(tmp_path: Path) -> Path: - root = tmp_path / "plain-documents" - root.mkdir() - (root / "note.md").write_text("# A note\n\nPlain text.\n", encoding="utf-8") + """F13.1's preamble: T050's `plain-documents` copied into a FRESH git + repository, committed under the fixture's own identity and none of the + user's git configuration.""" + root = tmp_path / PLAIN_DOCUMENTS.name + shutil.copytree(PLAIN_DOCUMENTS, root) + env = {name: value for name, value in os.environ.items() + if not name.startswith("GIT_")} + env.update({"GIT_AUTHOR_NAME": "fixture", + "GIT_AUTHOR_EMAIL": "fixture@example.invalid", + "GIT_COMMITTER_NAME": "fixture", + "GIT_COMMITTER_EMAIL": "fixture@example.invalid", + "GIT_CONFIG_GLOBAL": os.devnull, "GIT_CONFIG_SYSTEM": os.devnull}) + for argv in (["git", "-c", "init.defaultBranch=main", "init", "-q"], + ["git", "add", "-A"], ["git", "commit", "-qm", "fixture"]): + subprocess.run(argv, cwd=root, env=env, check=True, capture_output=True) return root From e7324d459a932094dc93143ada5cb3778a689b63 Mon Sep 17 00:00:00 2001 From: Brett Heap <1513478+brettheap@users.noreply.github.com> Date: Fri, 2 Oct 2026 20:23:50 +0000 Subject: [PATCH 23/88] T084: gate and refresh are true only where a contributed route answers them Measured at openDox-code 047bb4fa, a standalone /capabilities answered actions.gate and actions.refresh true, while every POST /actions/gate/ and POST /actions/refresh answered 404 unknown_action. The gate flag followed the checkout's git identity alone. Both routes are a host's: they arrive only through the route bindings the assembly collects. build_server now passes those bindings (route_bindings) into compute_capabilities. gate is true only when a binding answers a verb under /actions/gate/ (answers_a_gate_verb), and refresh only when one answers POST /actions/refresh (answers_the_refresh). Each keeps its other conditions. Standalone both read false, which also hides the workbench's session controls, and a composed host that contributes the routes reads as before. notebook, edit and session keep their conditions, and so does intent, which governs another plane's API. The refresh block still names the plane's binding. RULED openxFactory#656 5920216845, item 1 ("Fix in T084 + #1144 note (Recommended)"); #1144 4.3 as T007 batch L's addendum reads. New tests/test_capability_honesty.py covers: - a standalone `python -m opendox.serve` child, with and without a git identity, and with no GIT_* or XF_* reaching it, so the false gate is the routes' doing; - composed hosts with both routes, gate only, refresh only, and none; - for every true actions key, a governed route that does not answer unknown_action; - the two predicates. Before (main's serve.py): 20 failed. With an identity, the standalone child answered gate true; refresh was true in both runs. After: 20 passed. Five mutants are killed: gate ignores the routes (4 failed), refresh ignores the routes (4), the gate predicate is always true (5), every flag is off (6), and gate drops the actor (1). T056's standalone assertion now reads actions.refresh false, with refresh.binding still "regenerate". tests/: 2462 passed, 11 skipped. Arc: neutral-product-standalone-operability Lane: openxfactory-4 (openXfactory-4-openDox_extraction) Co-Authored-By: Claude Opus 5.5 (1M context) --- src/opendox/serve.py | 66 ++++- tests/test_capability_honesty.py | 342 +++++++++++++++++++++++++ tests/test_standalone_generate_path.py | 6 +- 3 files changed, 409 insertions(+), 5 deletions(-) create mode 100644 tests/test_capability_honesty.py diff --git a/src/opendox/serve.py b/src/opendox/serve.py index c6420917..fbd9aeef 100644 --- a/src/opendox/serve.py +++ b/src/opendox/serve.py @@ -452,6 +452,37 @@ ACTIONS_WORKBENCH_MODEL_INTAKE_ROUTE = "/actions/workbench/model-intake" ACTIONS_WORKBENCH_MODEL_APPROVAL_ROUTE = "/actions/workbench/model-approval" LOOPBACK_HOSTS = frozenset({"127.0.0.1", "::1", "localhost"}) +# THE TWO ROUTES THE `actions` MAP NAMES THAT A HOST CONTRIBUTES (plan 034 +# T084; #1144 4.3 as T007 batch L's addendum reads, RULED openxFactory#656 +# `5920216845`, item 1). Neither is a fixed core arm: a gate verb is +# `POST /actions/gate/` and the refresh is `POST /actions/refresh`, and +# each answers only where a route binding the assembly collected carries it. +# Everything else `do_POST` reaches is core. So `gate` and `refresh` are true +# only where such a binding is assembled (`compute_capabilities`), and a +# standalone server, which carries neither, reports both false rather than +# offering two affordances that would answer `404 unknown_action`. +ACTIONS_GATE_PREFIX = "/actions/gate/" +ACTIONS_REFRESH_ROUTE = "/actions/refresh" + + +def answers_a_gate_verb(binding) -> bool: + """Whether a contributed route binding answers `POST /actions/gate/` + for some verb: a POST prefix at or under `ACTIONS_GATE_PREFIX`, or one that + covers it, or an exact POST naming one verb under it. The match rule is the + binding's own (`RouteBinding.matches`), read for a family of paths.""" + if binding.method != "POST": + return False + pattern = binding.pattern + if binding.is_prefix: + return (pattern.startswith(ACTIONS_GATE_PREFIX) + or ACTIONS_GATE_PREFIX.startswith(pattern)) + return (pattern.startswith(ACTIONS_GATE_PREFIX) + and len(pattern) > len(ACTIONS_GATE_PREFIX)) + + +def answers_the_refresh(binding) -> bool: + """Whether a contributed route binding answers `POST /actions/refresh`.""" + return binding.matches("POST", ACTIONS_REFRESH_ROUTE) _DEFAULT_CAPABILITIES = {"actions": {"notebook": False, "gate": False, "refresh": False, "session": False, "edit": False, @@ -463,7 +494,8 @@ def compute_capabilities(*, nlm_present: bool, checkout_real: bool, loopback: bool, actor: str | None = None, - refresh_binding: str | None = None) -> dict: + refresh_binding: str | None = None, + route_bindings: tuple = ()) -> dict: """The startup capability verdict. The notebook action is available only on a loopback bind with `nlm` reachable and a real checkout — the served static image satisfies none of these, so the UI hides the affordance there. GATE @@ -522,16 +554,39 @@ def compute_capabilities(*, nlm_present: bool, checkout_real: bool, loopback: bo corpus. (The committed-intent FEED does read the checkout, but a feed with nothing in it is an empty feed, not an absent capability.) - So the predicate is the plane itself, and nothing else.""" + So the predicate is the plane itself, and nothing else. + + A FLAG WHOSE AFFORDANCE IS A ROUTE THIS SERVER SERVES IS TRUE ONLY WHERE + SUCH A ROUTE ANSWERS (plan 034 T084; #1144 4.3 as T007 batch L's addendum + reads, RULED openxFactory#656 `5920216845`, item 1). `gate` and `refresh` + govern routes a HOST contributes, `POST /actions/gate/` and + `POST /actions/refresh`, so each is true only when `route_bindings`, the + bindings the assembly collected, carry a route it governs + (`answers_a_gate_verb`, `answers_the_refresh`), and otherwise its + conditions above stand as they were. Measured at openDox-code `047bb4fa`, + a standalone server answered both true while every such POST answered + `404 unknown_action`, and the gate flag followed the checkout's git + identity alone. Standalone both now read false, which also hides the + workbench's session controls (`sessionActionsLive` reads `actions.gate`), + and a composed host that contributes the routes reads as before. + `notebook`, `edit` and `session` govern core routes and keep their + conditions. `intent` governs a POST to ANOTHER plane's intent API, which + that plane answers, so its condition, the served plane, stands. The + `refresh` block below still names the plane's binding: it says which + binding a contributed refresh would use, and the flag says whether one is + offered.""" binding = refresh_binding if binding == registry_mod.BINDING_REGENERATE and not (loopback and checkout_real): binding = None local_human = bool(actor and checkout_real and loopback) + bindings = tuple(route_bindings or ()) + gate_routed = any(answers_a_gate_verb(b) for b in bindings) + refresh_routed = any(answers_the_refresh(b) for b in bindings) return { "actions": { "notebook": bool(nlm_present and checkout_real and loopback), - "gate": local_human, - "refresh": bool(binding), + "gate": local_human and gate_routed, + "refresh": bool(binding) and refresh_routed, "session": local_human, "edit": local_human, # THE HOSTED WRITE-REQUEST SEAM, and the only capability here that @@ -1960,6 +2015,9 @@ def build_server( loopback=loopback, actor=resolved_actor, refresh_binding=source.refresh_binding, + # THE ROUTES THIS ASSEMBLY COLLECTED, so a flag whose affordance is a + # contributed route is true only where one answers (T084; batch L). + route_bindings=route_bindings, ) # The human console's per-serve token (FR-019's third clause, review finding # 2). Minted only where session verbs exist at all, and published on diff --git a/tests/test_capability_honesty.py b/tests/test_capability_honesty.py new file mode 100644 index 00000000..6ee06e36 --- /dev/null +++ b/tests/test_capability_honesty.py @@ -0,0 +1,342 @@ +"""Capability honesty: a flag in `/capabilities`' `actions` map whose affordance +is a route this server serves is true only where such a route answers (plan 034 +T084; #1144 4.3 as T007 batch L's addendum reads, RULED openxFactory#656 +`5920216845`, item 1, *"Fix in T084 + #1144 note (Recommended)"*). + +MEASURED AT openDox-code `047bb4fa`: a standalone `/capabilities` answered +`actions.gate` and `actions.refresh` true, while every `POST /actions/gate/` +and `POST /actions/refresh` answered `404 unknown_action`. The gate flag +followed the checkout's git identity alone (`compute_capabilities`). Both routes +are a HOST's: a gate verb and the refresh arrive only through the route bindings +the assembly collects, so each flag is now true only when those bindings carry a +route it governs. `notebook`, `edit` and `session` govern core routes and keep +their conditions, and `intent` governs another plane's API, so its condition +stands too. + +The cases: + +1. A STANDALONE SERVER, `python -m opendox.serve` as a child with neither + sibling importable (`tests/standalone_child.py`), over a fresh repository: + `gate` and `refresh` read false, and the two routes they would govern answer + `unknown_action`, which is why. It is run WITH a git identity, so an actor + resolves and the false `gate` is the routes' doing and not a missing actor's, + and without one. The child's environment carries no `GIT_*` and no `XF_*`: + the suite itself declares a roster of several principals + (`session_fixtures.declared_gate_principals`), and a roster of several names + with no claim resolves no actor at all. +2. COMPOSED HOSTS whose other conditions hold (a loopback bind, a real checkout, + an authenticated actor), built in process with contributed bindings and the + mixins that answer them, through the handler-contribution facet: a host that + contributes both routes reads both true; a gate-only host and a refresh-only + host each read only the flag whose route it contributes. +3. ON EVERY SERVER ABOVE, for every `actions` key that reads TRUE, a route it + governs does not answer `unknown_action`. So a plane that switched every flag + off would fail the cases that require a flag true, and a plane that left one + on with no route behind it fails here. `intent` governs no route of this + server, and every key of the map must be accounted for in `GOVERNED`. +4. The two predicates, case by case. + +A CREATED FILE: no carve-manifest row (RULED OQ-C). +""" + +from __future__ import annotations + +import http.client +import json +import os +import re +import threading +from pathlib import Path + +import pytest + +from route_extension import RouteBinding +from standalone_child import Child, fresh_repository, git, run_module + +ROOT = Path(__file__).resolve().parent.parent +PLAIN = ROOT / "tests" / "fixtures" / "plain-documents" +WEB = ROOT / "src" / "opendox" / "web" + +#: For each key of the `actions` map, the route it governs on THIS server, as +#: `(method, path)`, or None where it governs no route this server serves. +GOVERNED: dict[str, tuple[str, str] | None] = { + "notebook": ("POST", "/actions/notebook"), + "gate": ("POST", "/actions/gate/demote"), + "refresh": ("POST", "/actions/refresh"), + "session": ("POST", "/actions/workbench/chat-turn"), + "edit": ("POST", "/actions/edit"), + # a POST to ANOTHER plane's intent API, which that plane answers + "intent": None, +} + +_SERVE_URL = re.compile(r"^serving ideation dashboard at " + r"(http://([0-9.]+):([0-9]+))/index\.html$") + + +# --------------------------------------------------------------------------- +# helpers +# --------------------------------------------------------------------------- + +def _request(base: tuple[str, int], method: str, path: str) -> tuple[int, dict]: + """One request; the status and the JSON body (`{}` where it is not JSON). + A dropped connection raises, which is a failure of the case.""" + connection = http.client.HTTPConnection(*base, timeout=30) + try: + body = b"{}" if method == "POST" else None + headers = {"Content-Type": "application/json"} if body else {} + connection.request(method, path, body=body, headers=headers) + response = connection.getresponse() + raw = response.read() + try: + parsed = json.loads(raw) if raw else {} + except ValueError: + parsed = {} + return response.status, parsed if isinstance(parsed, dict) else {} + finally: + connection.close() + + +def _capabilities(base: tuple[str, int]) -> dict: + status, caps = _request(base, "GET", "/capabilities") + assert status == 200, status + return caps + + +def _unknown(answer: tuple[int, dict]) -> bool: + status, body = answer + return status == 404 and body.get("error") == "unknown_action" + + +def _assert_every_true_flag_answers(base: tuple[str, int], caps: dict) -> None: + """Case 3: every `actions` key is accounted for, and each that reads true + has a route that answers something other than `unknown_action`.""" + actions = caps["actions"] + assert set(actions) == set(GOVERNED), ( + f"the actions map has keys this test does not account for: " + f"{sorted(set(actions) ^ set(GOVERNED))}") + for key, value in actions.items(): + if value is not True or GOVERNED[key] is None: + continue + method, path = GOVERNED[key] + answer = _request(base, method, path) + assert not _unknown(answer), ( + f"actions.{key} reads true, but {method} {path} answers " + f"unknown_action: {answer}") + + +def _clean_environment(monkeypatch) -> None: + """No `GIT_*` and no `XF_*` reaches the child, and no user or system git + configuration: its only identity is the one its repository carries.""" + for name in list(os.environ): + if name.startswith(("GIT_", "XF_")): + monkeypatch.delenv(name) + monkeypatch.setenv("GIT_CONFIG_GLOBAL", os.devnull) + monkeypatch.setenv("GIT_CONFIG_SYSTEM", os.devnull) + + +def _repository(tmp_path: Path, *, identity: bool) -> Path: + repo = fresh_repository(PLAIN, tmp_path) + if identity: + git(repo, "config", "user.name", "fixture") + git(repo, "config", "user.email", "fixture@example.invalid") + return repo + + +def _snapshot(tmp_path: Path, repo: Path) -> Path: + out = tmp_path / "out" / "snapshot.json" + child, status = run_module( + tmp_path, "opendox.cli", "generate", "--repo-root", str(repo), + "--repository", "fixture", "--output", str(out), "--no-validate") + assert status == 0, child.stderr_text() + return out + + +# --------------------------------------------------------------------------- +# 1 and 3 — a standalone server +# --------------------------------------------------------------------------- + +@pytest.mark.parametrize("identity", [True, False], ids=["identity", "no-identity"]) +def test_a_standalone_server_offers_neither_gate_nor_refresh( + tmp_path, monkeypatch, identity) -> None: + """`python -m opendox.serve`, nothing registered by any host: `gate` and + `refresh` read false, the routes they would govern answer + `unknown_action`, and every flag that reads true answers.""" + _clean_environment(monkeypatch) + repo = _repository(tmp_path, identity=identity) + out = _snapshot(tmp_path, repo) + child = Child(tmp_path, "opendox.serve", "--snapshot", str(out), + "--checkout-root", str(repo), "--port", "0") + try: + match = child.wait_for_line(_SERVE_URL) + base = (match.group(2), int(match.group(3))) + caps = _capabilities(base) + actions = caps["actions"] + # THE OTHER CONDITIONS, so the false `gate` is the routes' doing: with + # an identity an actor resolves and the core write flags read true. + assert caps["actor"] == ("fixture" if identity else None), caps + assert actions["session"] is identity and actions["edit"] is identity + # the plane would regenerate, and no route offers it + assert caps["refresh"]["binding"] == "regenerate", caps + assert actions["gate"] is False and actions["refresh"] is False, caps + assert _unknown(_request(base, "POST", "/actions/gate/demote")) + assert _unknown(_request(base, "POST", "/actions/refresh")) + _assert_every_true_flag_answers(base, caps) + assert child.interrupt() == 0, child.stderr_text() + finally: + child.kill() + assert child.refused() == [], child.refused() + + +# --------------------------------------------------------------------------- +# 2 and 3 — composed hosts +# --------------------------------------------------------------------------- + +class _GateColumn: + """A host's gate column: answers every verb under the gate prefix.""" + + def _handle_honesty_gate(self, remainder): + self._send_json(200, {"ok": True, "verb": remainder}) + + +class _RefreshColumn: + """A host's refresh: answers `POST /actions/refresh`.""" + + def _handle_honesty_refresh(self): + self._send_json(200, {"ok": True, "refreshed": True}) + + +class _Contribution: + """A route extension contributing `bindings` and the mixins answering them.""" + + def __init__(self, bindings, mixins) -> None: + self._bindings = tuple(bindings) + self.HANDLER_CONTRIBUTIONS = tuple(mixins) + + def routes(self): + return self._bindings + + +def _gate(): + from opendox import serve + return (RouteBinding("POST", serve.ACTIONS_GATE_PREFIX, True, + "_handle_honesty_gate"), _GateColumn) + + +def _refresh(): + from opendox import serve + return (RouteBinding("POST", serve.ACTIONS_REFRESH_ROUTE, False, + "_handle_honesty_refresh"), _RefreshColumn) + + +@pytest.fixture() +def composed(tmp_path): + """`build(*contributions)`: a composed host over a REAL checkout, on a + loopback bind, with an authenticated actor (`brett`, one of the suite's + declared principals), served on a thread. Yields `(base, capabilities)`.""" + from opendox import serve + + repo = _repository(tmp_path, identity=True) + out = _snapshot(tmp_path, repo) + servers = [] + + def build(*contributed): + bindings = [binding for binding, _mixin in contributed] + mixins = [mixin for _binding, mixin in contributed] + httpd = serve.build_server( + WEB, out, repo, port=0, actor="brett", + route_extensions=(_Contribution(bindings, mixins),) + if contributed else ()) + worker = threading.Thread(target=httpd.serve_forever, daemon=True) + worker.start() + servers.append((httpd, worker)) + base = httpd.server_address[:2] + return base, _capabilities(base) + + try: + yield build + finally: + for httpd, worker in servers: + httpd.shutdown() + httpd.server_close() + worker.join(timeout=10) + + +def test_a_composed_host_contributing_both_routes_reads_both_true(composed) -> None: + base, caps = composed(_gate(), _refresh()) + assert caps["actor"] == "brett", caps + assert caps["actions"]["gate"] is True, caps + assert caps["actions"]["refresh"] is True, caps + assert caps["actions"]["session"] is True and caps["actions"]["edit"] is True + _assert_every_true_flag_answers(base, caps) + + +def test_a_gate_only_host_reads_only_gate_true(composed) -> None: + base, caps = composed(_gate()) + assert caps["actions"]["gate"] is True, caps + assert caps["actions"]["refresh"] is False, caps + _assert_every_true_flag_answers(base, caps) + + +def test_a_refresh_only_host_reads_only_refresh_true(composed) -> None: + base, caps = composed(_refresh()) + assert caps["actions"]["refresh"] is True, caps + assert caps["actions"]["gate"] is False, caps + _assert_every_true_flag_answers(base, caps) + + +def test_the_same_host_with_nothing_contributed_reads_both_false(composed) -> None: + """The control: the same checkout, actor and bind, and no contribution.""" + base, caps = composed() + assert caps["actor"] == "brett", caps + assert caps["actions"]["gate"] is False and caps["actions"]["refresh"] is False + assert caps["actions"]["session"] is True + _assert_every_true_flag_answers(base, caps) + + +# --------------------------------------------------------------------------- +# 4 — the two predicates +# --------------------------------------------------------------------------- + +@pytest.mark.parametrize("binding,answers", [ + (RouteBinding("POST", "/actions/gate/", True, "h"), True), + (RouteBinding("POST", "/actions/", True, "h"), True), + (RouteBinding("POST", "/actions/gate/demote", False, "h"), True), + (RouteBinding("POST", "/actions/gate/lens/", True, "h"), True), + (RouteBinding("POST", "/actions/gate/", False, "h"), False), + (RouteBinding("POST", "/actions/gatehouse/", True, "h"), False), + (RouteBinding("GET", "/actions/gate/", True, "h"), False), + (RouteBinding("POST", "/actions/refresh", False, "h"), False), +]) +def test_what_answers_a_gate_verb(binding, answers) -> None: + from opendox import serve + assert serve.answers_a_gate_verb(binding) is answers + + +@pytest.mark.parametrize("binding,answers", [ + (RouteBinding("POST", "/actions/refresh", False, "h"), True), + (RouteBinding("POST", "/actions/", True, "h"), True), + (RouteBinding("GET", "/actions/refresh", False, "h"), False), + (RouteBinding("POST", "/actions/refresh/", True, "h"), False), + (RouteBinding("POST", "/actions/gate/", True, "h"), False), +]) +def test_what_answers_the_refresh(binding, answers) -> None: + from opendox import serve + assert serve.answers_the_refresh(binding) is answers + + +def test_the_verdict_follows_the_bindings_and_keeps_the_other_conditions() -> None: + """`compute_capabilities` directly: the routes are necessary, never + sufficient, so an unresolved actor still keeps `gate` off.""" + from opendox import serve + both = (_gate()[0], _refresh()[0]) + kwargs = dict(nlm_present=False, checkout_real=True, loopback=True, + refresh_binding="regenerate") + assert serve.compute_capabilities(actor="a", **kwargs)["actions"]["gate"] is False + on = serve.compute_capabilities(actor="a", route_bindings=both, **kwargs) + assert on["actions"]["gate"] is True and on["actions"]["refresh"] is True + off = serve.compute_capabilities(actor=None, route_bindings=both, **kwargs) + assert off["actions"]["gate"] is False and off["actions"]["refresh"] is True + unbound = serve.compute_capabilities( + actor="a", route_bindings=both, + **{**kwargs, "refresh_binding": None}) + assert unbound["actions"]["refresh"] is False diff --git a/tests/test_standalone_generate_path.py b/tests/test_standalone_generate_path.py index 0a95efca..9a9ad253 100644 --- a/tests/test_standalone_generate_path.py +++ b/tests/test_standalone_generate_path.py @@ -145,7 +145,11 @@ def _assert_the_server_answers(base: tuple[str, int], written: Path, assert status == 200, status capabilities = json.loads(body) assert capabilities["refresh"]["binding"] == "regenerate" - assert capabilities["actions"]["refresh"] is True + # FALSE STANDALONE (plan 034 T084; #1144 4.3 as T007 batch L's addendum + # reads, RULED openxFactory#656 5920216845 item 1): the plane would + # regenerate, but `POST /actions/refresh` is a host's contributed route, + # and a standalone server carries none, so no refresh is offered. + assert capabilities["actions"]["refresh"] is False document = "notes-toolshed-inventory.md" status, kind, body = _get(base, f"/source/{document}") assert status == 200, status From c39d960e4144a934fb2d39c2361ddd2b5ed0f8d4 Mon Sep 17 00:00:00 2001 From: Brett Heap <1513478+brettheap@users.noreply.github.com> Date: Fri, 2 Oct 2026 20:28:35 +0000 Subject: [PATCH 24/88] Fix round: a PostgreSQL scheme libpq would not read as a URI is refused (Copilot review) Copilot's review at the merge-from-main head (adeb6fed) noted that `postgresql:foo` is still accepted, although the supported URI forms need `://`. Measured, it is worse than that. urlsplit reads `postgresql:` with no `//`, and any capitalized `PostgreSQL://` or `POSTGRES://`, as the PostgreSQL scheme, so the dialect gate passed all of them. libpq reads none of them as a URI. It recognizes only the exact, lower-case `postgresql://` and `postgres://`, parses the rest as keyword/value, and refuses them with a message that repeats the whole value (psycopg 3.3.6: `missing "=" after "postgresql:svc:hunter2@db/x" in connection info string`). That is the un-named failure at the driver that 13.2 exists to stop, and it carries the password. _refuse_non_postgresql_dsn now refuses a PostgreSQL scheme in any spelling other than libpq's two. The refusal names the setting and the two spellings, and does not repeat the value. Both loaders ask it. The new case covers four spellings for each of the two settings. All 8 fail against adeb6fed's config.py and pass here. The two accepted spellings and the keyword/value form still load. Five mutants are killed: the check dropped, case-insensitive matching, a check of only the `//`, every PostgreSQL DSN refused, and the value repeated. Full suite: 3069 selected, 3058 passed, 11 skipped, 0 failed. Lane: openxfactory-4 (openXfactory-4-openDox_extraction) Arc: neutral-product-standalone-operability Co-Authored-By: Claude Opus 5.5 (1M context) --- src/opendox/runtime/config.py | 20 ++++++++++++++++++++ tests_runtime/test_runtime_cli.py | 28 ++++++++++++++++++++++++++++ 2 files changed, 48 insertions(+) diff --git a/src/opendox/runtime/config.py b/src/opendox/runtime/config.py index 5c755006..7ae28936 100644 --- a/src/opendox/runtime/config.py +++ b/src/opendox/runtime/config.py @@ -1350,6 +1350,26 @@ def _refuse_non_postgresql_dsn(name: str, dsn: str | None) -> None: "runtime keeps: a second one would double every migration and " "every schema test forever, for a database that holds no " "document (RULING Q1)") + # A POSTGRESQL SCHEME IS A URI ONLY IN LIBPQ'S OWN SPELLING (Copilot + # review of openDox-code#60, at its merge-from-main round). `urlsplit` + # reads `postgresql:` with no `//`, and any capitalized `PostgreSQL://`, + # as the PostgreSQL scheme. libpq does not: it recognizes a URI only by + # the exact, lower-case `postgresql://` or `postgres://`, and parses + # anything else as keyword/value, which it then refuses with a message + # that REPEATS THE WHOLE VALUE (measured, psycopg 3.3.6: + # `missing "=" after "postgresql:svc:hunter2@db/x" in connection info + # string`). That is the un-named failure at the driver that 13.2 exists to + # stop, and it carries the password with it. So it is refused here, named, + # and the value is not repeated. + if scheme in POSTGRESQL_SCHEMES and not dsn.startswith( + tuple(f"{known}://" for known in sorted(POSTGRESQL_SCHEMES))): + raise ConfigurationError( + f"{name} reads as the PostgreSQL scheme but is not a URI libpq " + "reads: libpq recognizes only the exact, lower-case " + "`postgresql://` or `postgres://` prefix, and would refuse any " + "other spelling with a message that repeats the whole value. " + "Write the scheme as one of those two (the value is not " + "repeated here, because it can carry a password)") def _refuse_the_same_dsn_in_both_settings( diff --git a/tests_runtime/test_runtime_cli.py b/tests_runtime/test_runtime_cli.py index feadfa1a..c3e13bb6 100644 --- a/tests_runtime/test_runtime_cli.py +++ b/tests_runtime/test_runtime_cli.py @@ -2264,6 +2264,34 @@ def test_a_non_postgresql_dsn_is_refused_naming_the_dialect_kept() -> None: "host=h dbname=db user=m"}) +@pytest.mark.parametrize("dsn", ["postgresql:svc:hunter2@db.invalid/x", + "postgresql:/svc:hunter2@db.invalid/x", + "PostgreSQL://svc:hunter2@db.invalid/x", + "POSTGRES://svc:hunter2@db.invalid/x"]) +@pytest.mark.parametrize("setting", ["DATABASE_URL", "MIGRATION_DATABASE_URL"]) +def test_a_postgresql_scheme_libpq_would_not_read_as_a_uri_is_refused( + dsn: str, setting: str) -> None: + """`urlsplit` reads each of these as the PostgreSQL scheme. libpq reads + none of them as a URI: it knows only the exact, lower-case + `postgresql://` and `postgres://`, so it parses the rest as keyword/value + and refuses them with a message that repeats the whole value, password + and all. So each is refused at configuration, named, and the value is + not repeated (Copilot review of this PR, at its merge-from-main round). + The two spellings libpq does read stay accepted (the case above).""" + from opendox.runtime.config import ConfigurationError, load_settings + + good = {PREFIX + "DATABASE_URL": "postgresql://u:p@h/db", + PREFIX + "MIGRATION_DATABASE_URL": "postgresql://m:q@h/db"} + with pytest.raises(ConfigurationError) as refused: + load_settings({PREFIX + "OIDC_ISSUER": "https://broker/realms/x", + PREFIX + "OIDC_AUDIENCE": "opendox", + **good, PREFIX + setting: dsn}) + message = str(refused.value) + assert PREFIX + setting in message, message + assert "postgresql://" in message, message + assert "hunter2" not in message and dsn not in message, message + + def test_an_unparseable_dsn_is_refused_and_never_raises_a_bare_valueerror( ) -> None: """`urlsplit` itself raises for a DSN it cannot parse, and this module's From 379ff1f4cf391eb00fef14b2d2ee12751bc508bd Mon Sep 17 00:00:00 2001 From: Brett Heap <1513478+brettheap@users.noreply.github.com> Date: Fri, 2 Oct 2026 20:33:52 +0000 Subject: [PATCH 25/88] T073: 13.4a, /capabilities gains an install block, from the serving process's own settings The entry point's served /capabilities payload now carries `install: {mode, database_bundle}`. Both fields are read from the settings THIS process loaded and the bundled server it started as its own child (R1Q16 (i)), so the block describes the server the user reached, not a second process that read the same settings. - serve.build_server(install_report=) takes a zero-argument callable that the /capabilities arm asks on each request. The pid it names is therefore the child's at that moment, and a child that has gone is reported as gone. Unset, no block is published. serve.py reads no runtime setting itself (research R9). - cli._install_report builds that callable from args.runtime_settings (T070) and args.database_bundle (T072's BundledServer.report). A hosted install reports `database_bundle: null`, as `runtime status` does. _generate_and_open hands it to build_server. Falsifier: F13.1's caps.json block, added to tests_runtime/test_bundled_postgres.py beside T072's blocks and run on a real `python -m opendox.cli generate-and-open --local`. mode is local, data_dir and socket_dir are under the fresh state dir, and the pid taken from caps.json has no TCP listener, equals the pid runtime status reports, and is a child of the serving process. Before: "AssertionError: the served process is not in local mode: {}". After: passed. tests/test_served_install_block.py adds: - a hosted generate-and-open child reports {mode: hosted, database_bundle: null}; - no install shape publishes no block; - the block is asked per request; - _install_report's three cases. Before: 6 failed, 1 passed; after: 7 passed. Three mutants are killed: the arm caches its first answer (1 failed), the mode is hard-coded local (2), and the pid is the server's own (2). Whole suite locally: 3025 passed, 177 skipped. Ruled: R1Q22 (a), 5817152735; R1Q16 (i), 5850003126. Arc: neutral-product-standalone-operability Lane: openxfactory-4 (openXfactory-4-openDox_extraction) Co-Authored-By: Claude Opus 5.5 (1M context) --- src/opendox/cli.py | 34 ++++- src/opendox/serve.py | 29 ++++ tests/test_served_install_block.py | 178 +++++++++++++++++++++++++ tests_runtime/test_bundled_postgres.py | 53 +++++++- 4 files changed, 290 insertions(+), 4 deletions(-) create mode 100644 tests/test_served_install_block.py diff --git a/src/opendox/cli.py b/src/opendox/cli.py index 1438ac30..59825e46 100644 --- a/src/opendox/cli.py +++ b/src/opendox/cli.py @@ -700,6 +700,33 @@ def _run_the_local_lifecycle(args: argparse.Namespace, server, *, opener) -> int server.stop() +def _install_report(args: argparse.Namespace): + """`/capabilities`' `install` block for THIS process, as a callable the + server asks on each request, or None where no install shape was resolved + (plan 034 T073; #1144 13.4a; RULED R1Q16 (i), `5850003126`). + + It is read from the settings `cmd_generate_and_open` resolved and loaded + (`args.runtime_settings`) and from the bundled server it started as its + own child (`args.database_bundle`), so the served process reports its own + install shape: a status probe from a second process could be right about + the settings while the server ignored them. `mode` is the install mode + those settings carry. `database_bundle` is the bundled server's report, + `data_dir`, `socket_dir` and its `pid` while it lives + (`bundle.BundledServer.report`), and it is None for a hosted install, + which bundles no server, as `runtime status` reports it.""" + settings = getattr(args, "runtime_settings", None) + if settings is None: + return None + server = getattr(args, "database_bundle", None) + + def report() -> dict: + return {"mode": settings.install_mode, + "database_bundle": (server.report() if server is not None + else None)} + + return report + + def _generate_and_open(args: argparse.Namespace, *, opener) -> int: """`generate-and-open`'s generate-then-serve half, once the install is known.""" # Ahead of minting the run dir, so a refused root leaves not even an empty @@ -774,7 +801,12 @@ def _generate_and_open(args: argparse.Namespace, *, opener) -> int: # the self-hosted half of the ratified # two-case principle knowledge_declaration=( - knowledge_mod.SELF_HOSTED_LOCAL_EMBEDDED)) + knowledge_mod.SELF_HOSTED_LOCAL_EMBEDDED), + # and THIS process's own install shape, on + # `/capabilities` (plan 034 T073; #1144 + # 13.4a): the settings it loaded, and the + # bundled server it started as its child. + install_report=_install_report(args)) url = serve_mod.server_url(httpd, "/index.html") print(f" serving {url}") print(f" snapshot {serve_mod.server_url(httpd, '/snapshot.json')}") diff --git a/src/opendox/serve.py b/src/opendox/serve.py index c6420917..cb64f426 100644 --- a/src/opendox/serve.py +++ b/src/opendox/serve.py @@ -839,6 +839,12 @@ class DashboardHandler(serve_workbench.WorkbenchRoutes, # v2 seam: the startup capability verdict, whether the bind is loopback (the # write-route gate), and an injectable adapter factory (tests supply a fake). capabilities: dict = _DEFAULT_CAPABILITIES + # THE SERVING PROCESS'S OWN INSTALL SHAPE (plan 034 T073; #1144 13.4a): + # a zero-argument callable answering `/capabilities`' `install` block, or + # None where the process that built this server resolved no install shape + # (a library caller or a test), which publishes no block. See + # `build_server(install_report=)`. + install_report = None loopback: bool = True adapter_factory = None # The session's remote-write port supplier (T082). None means "build the real @@ -1091,6 +1097,12 @@ def _route(self, head_only: bool) -> bool: # display does not make this credential-free surface a credential # holder or an auth authority (design D16 nuance). payload["hosted_actor"] = self.headers.get("X-Auth-Request-User") or None + # THE INSTALL BLOCK (plan 034 T073; #1144 13.4a), read PER REQUEST + # from the serving process's own settings and its own bundled + # server, so the pid it names is the server's at the moment it is + # asked: a child that has gone is reported as gone. + if self.install_report is not None: + payload["install"] = self.install_report() self._serve_bytes(json.dumps(payload).encode("utf-8"), JSON_CTYPE, head_only) return True @@ -1717,6 +1729,7 @@ def build_server( knowledge_declaration=None, packet_assembler=None, route_extensions: tuple = (), + install_report=None, ) -> http.server.ThreadingHTTPServer: """Build (but do not start) the loopback server. `port=0` binds an ephemeral port (read it back from `httpd.server_address`). `head` is injectable so a @@ -1754,6 +1767,18 @@ def build_server( reason: an operator must be able to read what their install talks to, and a library default that quietly built one would defeat that. + `install_report` is THE SERVING PROCESS'S OWN INSTALL SHAPE (plan 034 + T073; #1144 13.4a; RULED R1Q16 (i), `5850003126`): a zero-argument callable + answering `/capabilities`' `install` block, `{"mode": ..., + "database_bundle": ...}`, asked on each request. The ENTRY POINT supplies + it, from the settings it loaded and the bundled server it started as its + own child (`cli._install_report`), so the block describes the process a + user reached and not a second process that read the same settings. + Unset, the payload carries no `install` block: nothing in this process + resolved an install shape, and none is invented. This module reads no + runtime setting itself (research R9: the document surface never imports + the runtime). + `route_extensions` is the ROUTE EXTENSION POINT (`split-opendox-two-layer-product` § 2.4, design § D2): the tuple of `route_extension.RouteExtension`s this server is ASSEMBLED with, each @@ -2136,6 +2161,10 @@ def build_server( # The contributed routes, already in consult order (§ 2.4). One more # injected class attribute, exactly like the seams above it. "route_bindings": route_bindings, + # THE SERVING PROCESS'S OWN INSTALL SHAPE (T073; 13.4a), asked per + # request by the `/capabilities` arm. + "install_report": (staticmethod(install_report) + if install_report is not None else None), }) # A ROUTE THAT CANNOT BE SERVED MUST NOT START. Resolved against the bound # class — the object the dispatch will `getattr` on — so a binding naming a diff --git a/tests/test_served_install_block.py b/tests/test_served_install_block.py new file mode 100644 index 00000000..cfaec76d --- /dev/null +++ b/tests/test_served_install_block.py @@ -0,0 +1,178 @@ +"""`/capabilities`' `install` block: the serving process reports its own +install shape (plan 034 T073; #1144 13.4a; RULED R1Q16 (i), `5850003126`). + +13.4a: the entry point's served `/capabilities` payload gains an `install` +block, holding `mode` and `database_bundle` (`data_dir`, `socket_dir`, `pid`), +read from the settings THAT PROCESS loaded. F13.1's `caps.json` block is the +falsifier, and it needs the LOCAL install's bundled server, so it runs in +`tests_runtime/test_bundled_postgres.py` beside T072's blocks. This module +holds the rest, none of which starts a database: + +1. A HOSTED `generate-and-open`, a child with every setting a hosted install + needs: the served block says `hosted`, with no bundled server + (`database_bundle` null), as `runtime status` reports a hosted install. +2. `build_server` publishes exactly what its entry point hands it, asked on + EACH request, so a pid that changes is reported as it is now; and with + nothing handed in, the payload carries no `install` block at all. +3. `cli._install_report` reads the settings the verb loaded and the bundled + server it started, and answers None where nothing was resolved. + +A CREATED FILE: no carve-manifest row (RULED OQ-C). +""" + +from __future__ import annotations + +import argparse +import http.client +import json +import re +import threading +import types +from pathlib import Path + +import pytest + +from opendox import cli as cli_mod +from opendox import serve as serve_mod +from opendox.runtime import config as runtime_config +from standalone_child import Child, fresh_repository, run_module + +ROOT = Path(__file__).resolve().parent.parent +PLAIN = ROOT / "tests" / "fixtures" / "plain-documents" +WEB = ROOT / "src" / "opendox" / "web" +PREFIX = runtime_config.PREFIX + +#: Every setting a HOSTED install needs, none of them reached by this run: the +#: document surface reads nothing from the store in release 1 (R1Q16 (ii)). +HOSTED = { + PREFIX + "DATABASE_URL": "postgresql://serve@127.0.0.1:1/opendox", + PREFIX + "MIGRATION_DATABASE_URL": "postgresql://migrate@127.0.0.1:1/opendox", + PREFIX + "OIDC_AUDIENCE": "fixture", + PREFIX + "OIDC_ISSUER": "https://issuer.example.invalid/realms/fixture", +} + +_URL = re.compile(r"^(http://([0-9.]+):([0-9]+))/index\.html$") + + +def _get(base: tuple[str, int], path: str) -> tuple[int, dict]: + connection = http.client.HTTPConnection(*base, timeout=30) + try: + connection.request("GET", path) + response = connection.getresponse() + return response.status, json.loads(response.read() or b"{}") + finally: + connection.close() + + +# --------------------------------------------------------------------------- +# 1 — a hosted entry point reports a hosted install, and no bundled server +# --------------------------------------------------------------------------- + +def test_a_hosted_entry_point_reports_its_hosted_shape(tmp_path, monkeypatch) -> None: + for name in runtime_config.SETTING_NAMES: + monkeypatch.delenv(name, raising=False) + for name, value in HOSTED.items(): + monkeypatch.setenv(name, value) + monkeypatch.setenv(PREFIX + "INSTALL_MODE", runtime_config.INSTALL_MODE_HOSTED) + repo = fresh_repository(PLAIN, tmp_path) + child = Child(tmp_path, "opendox.cli", "generate-and-open", + "--repo-root", str(repo), "--repository", "fixture", + "--no-open", "--port", "0", "--run-dir", str(tmp_path / "run")) + try: + match = child.wait_for_line(_URL) + base = (match.group(2), int(match.group(3))) + status, caps = _get(base, "/capabilities") + assert status == 200, status + assert caps.get("install") == { + "mode": runtime_config.INSTALL_MODE_HOSTED, "database_bundle": None}, caps + assert child.interrupt() == 0, child.stderr_text() + finally: + child.kill() + + +# --------------------------------------------------------------------------- +# 2 — build_server publishes what its entry point hands it, per request +# --------------------------------------------------------------------------- + +@pytest.fixture() +def served(tmp_path): + """`serve(install_report=...)`: a server built in process over a fresh + repository's snapshot, on a thread, and the base it answers on.""" + repo = fresh_repository(PLAIN, tmp_path) + snapshot = tmp_path / "snapshot.json" + child, status = run_module(tmp_path, "opendox.cli", "generate", + "--repo-root", str(repo), "--repository", + "fixture", "--output", str(snapshot), + "--no-validate") + assert status == 0, child.stderr_text() + servers = [] + + def serve(**kwargs): + httpd = serve_mod.build_server(WEB, snapshot, repo, port=0, **kwargs) + worker = threading.Thread(target=httpd.serve_forever, daemon=True) + worker.start() + servers.append((httpd, worker)) + return httpd.server_address[:2] + + try: + yield serve + finally: + for httpd, worker in servers: + httpd.shutdown() + httpd.server_close() + worker.join(timeout=10) + + +def test_with_no_install_shape_the_payload_carries_no_install_block(served) -> None: + """A process that resolved no install shape publishes none: an absent + block, never an invented one.""" + status, caps = _get(served(), "/capabilities") + assert status == 200 and "install" not in caps, caps + + +def test_the_block_is_the_entry_points_and_is_asked_on_each_request(served) -> None: + pids = iter([4242, None]) + + def report(): + return {"mode": "local", "database_bundle": { + "data_dir": "/state/postgres/data", "socket_dir": "/state/postgres/run", + "pid": next(pids)}} + + base = served(install_report=report) + first = _get(base, "/capabilities")[1]["install"] + second = _get(base, "/capabilities")[1]["install"] + assert first["mode"] == "local" and first["database_bundle"]["pid"] == 4242 + # the server's child has gone: the next request says so + assert second["database_bundle"]["pid"] is None + + +# --------------------------------------------------------------------------- +# 3 — the entry point reads its own settings and its own bundled server +# --------------------------------------------------------------------------- + +def test_no_resolved_settings_is_no_install_report() -> None: + assert cli_mod._install_report(argparse.Namespace()) is None + + +def test_a_hosted_run_reports_no_bundled_server() -> None: + settings = types.SimpleNamespace(install_mode=runtime_config.INSTALL_MODE_HOSTED) + report = cli_mod._install_report(argparse.Namespace(runtime_settings=settings)) + assert report() == {"mode": "hosted", "database_bundle": None} + + +def test_a_local_run_reports_the_server_it_started() -> None: + class _Server: + def __init__(self): + self.pid = 7 + + def report(self): + return {"data_dir": "/s/d", "socket_dir": "/s/r", "pid": self.pid} + + server = _Server() + settings = types.SimpleNamespace(install_mode=runtime_config.INSTALL_MODE_LOCAL) + report = cli_mod._install_report(argparse.Namespace( + runtime_settings=settings, database_bundle=server)) + assert report() == {"mode": "local", "database_bundle": { + "data_dir": "/s/d", "socket_dir": "/s/r", "pid": 7}} + server.pid = None # it stopped: asked again, it says so + assert report()["database_bundle"]["pid"] is None diff --git a/tests_runtime/test_bundled_postgres.py b/tests_runtime/test_bundled_postgres.py index bae2a7f5..e291d5a6 100644 --- a/tests_runtime/test_bundled_postgres.py +++ b/tests_runtime/test_bundled_postgres.py @@ -10,9 +10,15 @@ signal. Nothing is stood in: the corpus root check, the generation, the validator and the serve loop are phase 2's own, landed on this stack's base (T054 to T058), so the stand-in driver this module once launched is gone. Where F13.1 reads the -server's pid from `caps.json`, these cases read the same pid from -`runtime status`'s `database_bundle`. `/capabilities`' `install` block is -T073's, and nothing here pretends it exists. +server's pid from `caps.json`, T072's cases read the same pid from +`runtime status`'s `database_bundle`. + +T073's falsifier, F13.1's `caps.json` block, is here too, because it needs the +same running server: the SERVING process's `/capabilities` reports its own +install shape (#1144 13.4a; RULED R1Q16 (i)), so the pid F13.1's TCP-listener +block reads is taken from that payload, as F13.1 takes it, and is held equal +to the one `runtime status` reports. `tests/test_served_install_block.py` +holds the block's other cases, none of which starts a database. R1Q16, each part asserted: (i) the server is a CHILD of the entry point's process (its `PPid`); @@ -365,6 +371,47 @@ def test_the_entry_point_owns_a_migrated_server_with_no_tcp_listener( "the data directory must survive a stop: it is the install's database" +def test_the_serving_process_reports_its_own_install_shape( + corpus: Path, state_dir: Path, tmp_path: Path) -> None: + """F13.1's `caps.json` block (T073; #1144 13.4a): the server the user + reached on its port reports its OWN mode and datastore, so the claim is + about that server and not about a second process that read the same + settings. Then F13.1's TCP-listener block, with the pid read from + `caps.json`, as F13.1 reads it.""" + server, url = _launch(corpus, state_dir, tmp_path / "run") + try: + caps_url = url.rsplit("/", 1)[0] + "/capabilities" + with urllib.request.urlopen(caps_url, timeout=10) as answer: + caps = json.loads(answer.read()) + # -- F13.1's `caps.json` block, verbatim in substance ---------------- + inst = caps.get("install") or {} + assert inst.get("mode") == "local", \ + f"the served process is not in local mode: {inst}" + state = os.path.realpath(state_dir) + for key in ("data_dir", "socket_dir"): + got = os.path.realpath((inst.get("database_bundle") or {}).get(key, "")) + assert got.startswith(state + os.sep), ( + f"the served process uses {key} {got!r}, not the bundle under " + f"{state!r}") + # -- F13.1's TCP-listener block, the pid from `caps.json` ------------ + pid = (inst.get("database_bundle") or {}).get("pid") + assert isinstance(pid, int), f"the bundle reports no server pid: {pid!r}" + assert not _tcp_listeners(pid), \ + f"the bundled server listens on TCP: {_tcp_listeners(pid)}" + # the served block and `runtime status` name ONE server, the child of + # the process that served `caps.json` (R1Q16 (i)) + _code, status = _status(state_dir) + assert (status.get("database_bundle") or {}).get("pid") == pid, status + assert _parent_of(pid) == server.pid, ( + "the reported server is not a child of the process that served " + "`/capabilities`") + finally: + server.send_signal(signal.SIGTERM) + server.communicate(timeout=60) + assert server.returncode == 0, server.returncode + assert _wait_gone(pid), "the bundled server outlived its entry point" + + #: libpq defaults that, READ, would move the bundle's connections: to an #: unroutable TCP address (TEST-NET-1, RFC 5737), through a service that does #: not exist, and into a schema that does not either. From cdf7382b7e3f30884e0c93703b8254fe782bdb2b Mon Sep 17 00:00:00 2001 From: Brett Heap <1513478+brettheap@users.noreply.github.com> Date: Fri, 2 Oct 2026 20:35:03 +0000 Subject: [PATCH 26/88] Fix round: the --local callers inherit none of the runner's runtime settings (Copilot review) Copilot's review at c8fac05e opened three threads, and all three are real. The four callers this PR gave --local took the runner's environment along: tests/standalone_child.py's Child copied os.environ, and the in-process projection-seams case read it directly. An exported OPENDOX_INSTALL_MODE=hosted, or a broker issuer, made each --local run refuse before it reached what it tests. - tests/standalone_child.py: every child's environment drops each name in opendox.runtime.config.SETTING_NAMES. This covers T056's case 3 and T058's two post-render cases, and any later child. - tests/test_projection_seams.py: the in-process empty-option case scrubs SETTING_NAMES first, as the doxBench entrypoint fixture does. - tests/test_standalone_generate_path.py: a harness case exports a hosted install's four settings and asserts that a child sees none of them. Measured with OPENDOX_INSTALL_MODE=hosted and OPENDOX_OIDC_ISSUER exported: all 5 cases fail against 9ae5e72's harness and pass here. Mutant "the child keeps the runner's settings" is killed by the harness case in a plain environment. Mutant "the in-process case keeps them" is killed under the exported hosted mode. Full suite: 3126 selected, 3115 passed, 11 skipped, 0 failed. Lane: openxfactory-4 (openXfactory-4-openDox_extraction) Arc: neutral-product-standalone-operability Co-Authored-By: Claude Opus 5.5 (1M context) --- tests/standalone_child.py | 11 ++++++++- tests/test_projection_seams.py | 11 ++++++++- tests/test_standalone_generate_path.py | 31 +++++++++++++++++++++++++- 3 files changed, 50 insertions(+), 3 deletions(-) diff --git a/tests/standalone_child.py b/tests/standalone_child.py index 37d1fe5f..34bbda01 100644 --- a/tests/standalone_child.py +++ b/tests/standalone_child.py @@ -20,6 +20,12 @@ * It is read on threads while it runs (`Child.wait_for_line`), so a server that never exits can still be asked where it serves, and then interrupted (`Child.interrupt`, SIGINT, as Ctrl-C sends). +* IT INHERITS NO RUNTIME SETTING. Every `OPENDOX_*` name the runtime reads + (`opendox.runtime.config.SETTING_NAMES`) is taken out of the child's + environment, so an `OPENDOX_INSTALL_MODE=hosted` or a broker issuer the + runner happens to export cannot make a `generate-and-open --local` child + refuse before the case it exists for (plan 034 T070; Copilot review of + openDox-code#67). * CTRL-C REACHES IT AS IT WOULD AT A TERMINAL, whatever the runner's own disposition. The same `sitecustomize` sets SIGINT back to Python's KeyboardInterrupt handler. A runner started as a background job @@ -140,7 +146,10 @@ def __init__(self, workdir: Path, module: str, *args: str) -> None: blocker.mkdir(parents=True, exist_ok=True) (blocker / "sitecustomize.py").write_text(_BLOCKER, encoding="utf-8") self.refused_log = workdir / "refused-imports.log" - env = dict(os.environ) + from opendox.runtime.config import SETTING_NAMES + + env = {name: value for name, value in os.environ.items() + if name not in SETTING_NAMES} env.pop("PYTHONUNBUFFERED", None) env["PYTHONPATH"] = os.pathsep.join( [str(blocker), *filter(None, [env.get("PYTHONPATH")])]) diff --git a/tests/test_projection_seams.py b/tests/test_projection_seams.py index 766a848a..02c5ec65 100644 --- a/tests/test_projection_seams.py +++ b/tests/test_projection_seams.py @@ -1392,7 +1392,16 @@ def test_a_given_source_option_is_resolved_and_an_unset_one_is_not_passed( def test_generate_and_open_refuses_an_empty_source_option_before_its_run_dir( - tmp_path, capsys) -> None: + tmp_path, capsys, monkeypatch) -> None: + """In process, so the runtime settings the runner exports are scrubbed + first, as the doxBench entrypoint fixture does: an exported + `OPENDOX_INSTALL_MODE=hosted` or broker issuer would otherwise make + `--local` refuse before the empty option is reached (Copilot review of + openDox-code#67).""" + from opendox.runtime.config import SETTING_NAMES + + for name in SETTING_NAMES: + monkeypatch.delenv(name, raising=False) calls: list = [] _declaring_generator(calls) repo = _repository(tmp_path) diff --git a/tests/test_standalone_generate_path.py b/tests/test_standalone_generate_path.py index b71f3650..a199de51 100644 --- a/tests/test_standalone_generate_path.py +++ b/tests/test_standalone_generate_path.py @@ -22,7 +22,8 @@ refuses `/source/.git/config`, and stops on an interrupt with status 0. 4. `python -m opendox.serve`, the server's own entry point, starts and answers the same way. -5. The harness itself: a child that ignores the interrupt is killed at the +5. The harness itself: a child inherits none of the runner's runtime + settings, and a child that ignores the interrupt is killed at the deadline, and the timeout is raised, so a server that will not stop is reported rather than waited out. And a child stops on the interrupt even when the RUNNER ignores SIGINT, as a suite started as a background job @@ -321,6 +322,34 @@ def test_serve_main_starts_a_server_that_answers_with_no_sibling(tmp_path) -> No # 5 — the harness itself: an ignored interrupt is reported, not waited out # --------------------------------------------------------------------------- +def test_a_child_inherits_none_of_the_runners_runtime_settings( + tmp_path, monkeypatch) -> None: + """The runner exports a HOSTED install's settings, and the child sees + none of them (plan 034 T070; Copilot review of openDox-code#67). The + `--local` cases above would otherwise refuse before they reach what they + test, for a reason that is the runner's configuration and not theirs.""" + from opendox.runtime.config import PREFIX, SETTING_NAMES + + exported = {PREFIX + "INSTALL_MODE": "hosted", + PREFIX + "OIDC_ISSUER": "https://issuer.example.invalid/realms/x", + PREFIX + "OIDC_AUDIENCE": "fixture", + PREFIX + "DATABASE_URL": "postgresql://s@127.0.0.1:1/x"} + for name, value in exported.items(): + monkeypatch.setenv(name, value) + blocker = tmp_path / "sibling-blocker" + blocker.mkdir() + (blocker / "t070_env_probe.py").write_text(textwrap.dedent(""" + import json, os + print(json.dumps(sorted(n for n in os.environ if n.startswith("OPENDOX_"))), + flush=True) + """), encoding="utf-8") + child, status = run_module(tmp_path, "t070_env_probe") + assert status == 0, child.stderr_text() + seen = set(json.loads(child.stdout_text().strip().splitlines()[-1])) + assert not seen & set(SETTING_NAMES), sorted(seen & set(SETTING_NAMES)) + assert set(exported) <= set(SETTING_NAMES) + + def test_a_child_that_ignores_the_interrupt_is_killed_at_the_deadline( tmp_path, monkeypatch) -> None: """The regression path cases 3 and 4 guard: a server that does not stop From 6eb0bbdb962c611c11466c9e6efd406134ebf47c Mon Sep 17 00:00:00 2001 From: Brett Heap <1513478+brettheap@users.noreply.github.com> Date: Fri, 2 Oct 2026 20:41:59 +0000 Subject: [PATCH 27/88] T072: the env probe expects the child's own state directory, never the runner's #67's harness case asserted a child sees no runtime setting at all. Under T072 every child is given one: OPENDOX_STATE_DIR, its private state directory. The probe now also exports a runner state directory and asserts that the child sees exactly OPENDOX_STATE_DIR among the runtime settings, with the value of its own Child.state_dir (not the runner's), and that the directory is gone once the child has exited. Mutants, all run under an exported hosted install's settings, are all killed: - the child keeps the runner's settings; - the scrub runs after the state directory is set; - the runner's own state directory is passed through; - the in-process empty-option case does not scrub. Lane: openxfactory-4 (openXfactory-4-openDox_extraction) Co-Authored-By: Claude Opus 5.5 (1M context) --- tests/test_standalone_generate_path.py | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/tests/test_standalone_generate_path.py b/tests/test_standalone_generate_path.py index f752f311..567484b7 100644 --- a/tests/test_standalone_generate_path.py +++ b/tests/test_standalone_generate_path.py @@ -333,27 +333,35 @@ def test_a_child_inherits_none_of_the_runners_runtime_settings( """The runner exports a HOSTED install's settings, and the child sees none of them (plan 034 T070; Copilot review of openDox-code#67). The `--local` cases above would otherwise refuse before they reach what they - test, for a reason that is the runner's configuration and not theirs.""" + test, for a reason that is the runner's configuration and not theirs. + The one runtime setting a child does see is the state directory the + harness gives it (plan 034 T072), never the runner's own.""" from opendox.runtime.config import PREFIX, SETTING_NAMES + state_setting = PREFIX + "STATE_DIR" exported = {PREFIX + "INSTALL_MODE": "hosted", PREFIX + "OIDC_ISSUER": "https://issuer.example.invalid/realms/x", PREFIX + "OIDC_AUDIENCE": "fixture", - PREFIX + "DATABASE_URL": "postgresql://s@127.0.0.1:1/x"} + PREFIX + "DATABASE_URL": "postgresql://s@127.0.0.1:1/x", + state_setting: str(tmp_path / "runners-own-state")} for name, value in exported.items(): monkeypatch.setenv(name, value) blocker = tmp_path / "sibling-blocker" blocker.mkdir() - (blocker / "t070_env_probe.py").write_text(textwrap.dedent(""" + (blocker / "t070_env_probe.py").write_text(textwrap.dedent(f""" import json, os - print(json.dumps(sorted(n for n in os.environ if n.startswith("OPENDOX_"))), - flush=True) + print(json.dumps([sorted(n for n in os.environ if n.startswith("OPENDOX_")), + os.environ.get({state_setting!r})]), flush=True) """), encoding="utf-8") - child, status = run_module(tmp_path, "t070_env_probe") + child = Child(tmp_path, "t070_env_probe") + state_dir = child.state_dir + status = child.wait() assert status == 0, child.stderr_text() - seen = set(json.loads(child.stdout_text().strip().splitlines()[-1])) - assert not seen & set(SETTING_NAMES), sorted(seen & set(SETTING_NAMES)) + names, state_value = json.loads(child.stdout_text().strip().splitlines()[-1]) + assert set(names) & set(SETTING_NAMES) == {state_setting}, names + assert state_value == str(state_dir) != exported[state_setting] assert set(exported) <= set(SETTING_NAMES) + assert not state_dir.exists(), "the child's state directory outlived it" def test_a_child_that_ignores_the_interrupt_is_killed_at_the_deadline( From d1de1fd90402ab7f3fe65de4cabc67fe6a652040 Mon Sep 17 00:00:00 2001 From: Brett Heap <1513478+brettheap@users.noreply.github.com> Date: Fri, 2 Oct 2026 20:49:17 +0000 Subject: [PATCH 28/88] Fix round: the healthy-local status case sets its schema with make_conninfo (Copilot review) OPENDOX_TEST_DATABASE_URL may be libpq's keyword/value form as well as a URI (the runtime reads both, and schema_selected_by reads the schema out of both). The case appended "?options=..." to it. On the keyword/value form that suffix becomes part of dbname, so the case failed for a reason that was the fixture's spelling, while the fixtures themselves still connected. psycopg.conninfo.make_conninfo now sets "options" on either form. Measured with a keyword/value OPENDOX_TEST_DATABASE_URL: the case failed before and passes now, and it passes on the URI form both times. A mutant that drops the search_path option is killed on both forms. Lane: openxfactory-4 (openXfactory-4-openDox_extraction) Co-Authored-By: Claude Opus 5.5 (1M context) --- tests_runtime/test_install_mode.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/tests_runtime/test_install_mode.py b/tests_runtime/test_install_mode.py index 24d44eb6..0c5477a4 100644 --- a/tests_runtime/test_install_mode.py +++ b/tests_runtime/test_install_mode.py @@ -377,10 +377,14 @@ def _no_broker(_settings): "install, which has no broker") monkeypatch.setattr(oidc, "build_verifier", _no_broker) - joiner = "&" if "?" in postgres_dsn else "?" - scrubbed.setenv(PREFIX + "DATABASE_URL", - f"{postgres_dsn}{joiner}options=-c%20search_path%3D" - f"{database.schema}%2Cpublic") + # `make_conninfo`, NOT a `?options=` suffix: `OPENDOX_TEST_DATABASE_URL` + # may be libpq's keyword/value form as well as a URI, and a suffix on + # `… dbname=opendox` names the database `opendox?options=…` instead of + # selecting the schema (Copilot review of openDox-code#67). + from psycopg.conninfo import make_conninfo + + scrubbed.setenv(PREFIX + "DATABASE_URL", make_conninfo( + postgres_dsn, options=f"-c search_path={database.schema},public")) scrubbed.setenv(MODE, "local") code, evidence = _run(["runtime", "status", "--probe-timeout", "5"]) assert evidence["database"] == "reachable", evidence From 7714751aa446d8ca1611ea763b48b996f0fd8f74 Mon Sep 17 00:00:00 2001 From: Brett Heap <1513478+brettheap@users.noreply.github.com> Date: Fri, 2 Oct 2026 20:50:33 +0000 Subject: [PATCH 29/88] T084: the consumer columns' seams, with openDox's defaults registered by the entry points opendox.column_seams declares four seams where openDox's own verbs reached openXdox's columns by module name, in projection_seams' discipline (the same _Seam class, now naming its declaring module): - gate: every gate-console name openDox reads, as one registration, because a record builder checks its own Provenance type and a caller catches its own GateRefused; - scope: the doxBench scope authority (resolve_scope, is_live_session_ref, session_created_paths_for_scope); - kickoff: the dispatched commissions and the project register's discovery; - register: the cross-reference register's adapter. opendox.default_columns is openDox's own default for each (R1Q10 (a); holder readings relayed on openxFactory#656's thread 2026-10-02, which Brett may overrule): - GATE carries the vocabulary-free primitives as real code: the HumanGate guard, the clock and stamp, the prefix, ref_target_id, Provenance with its surface and presence constants, HTTP_CONSOLE_TOKEN, GateRefused, the session constants and the first-edit gate builder. The governed record functions and GateConsole are NOT emulated. They refuse by name, as GateRecordsNotRegistered, naming opendox.column_seams.gate and its registration call (4.2). - SCOPE is a small read-only projection of a neutral tile, every path confined by the registry seam's resolve_within, with nothing editable. is_live_session_ref keeps its logic over openDox's own session layer, and there are no session-created paths. - KICKOFF answers that nothing is dispatched and there is no project register. - REGISTER is a register with no possibles. cli.build_parser(), cli.main(), serve.build_server() and serve.main() call column_seams.register_defaults() beside projection_seams'. A bare process still refuses, naming the seam. gate_records_writable() is true only with a host's gate (_Seam.holds_a_hosts). projection_seams' messages are byte-identical (tests/test_projection_seams.py 139 passed). The new tests/test_column_seams.py has 36 cases. Four mutants are killed: serve.main drops the defaults (1 failed), writable with the default (3), the governed functions answer quietly (4), and the scope grants edit (3). tests/: 2518 passed, 11 skipped. Ruled: R1Q10 (a), 5850003126; R1Q3 (a), R1Q22 (a), 5817152735. Arc: neutral-product-standalone-operability Lane: openxfactory-4 (openXfactory-4-openDox_extraction) Co-Authored-By: Claude Opus 5.5 (1M context) --- src/opendox/cli.py | 9 + src/opendox/column_seams.py | 165 +++++++++++ src/opendox/default_columns.py | 492 ++++++++++++++++++++++++++++++++ src/opendox/projection_seams.py | 34 ++- src/opendox/serve.py | 9 + tests/test_column_seams.py | 327 +++++++++++++++++++++ 6 files changed, 1027 insertions(+), 9 deletions(-) create mode 100644 src/opendox/column_seams.py create mode 100644 src/opendox/default_columns.py create mode 100644 tests/test_column_seams.py diff --git a/src/opendox/cli.py b/src/opendox/cli.py index cb464b04..619a3bbc 100644 --- a/src/opendox/cli.py +++ b/src/opendox/cli.py @@ -110,6 +110,7 @@ # register the defaults where no host has. `is_rfc3339_datetime` is not a seam: # it is the neutral contract's own date-time rule, and openDox owns it. from opendox import projection_seams # noqa: E402 +from opendox import column_seams # noqa: E402 from opendox.rfc3339 import is_rfc3339_datetime # noqa: E402 # THE COMPOSITION POINT, BOUND AT LAST (§ 4.3; RULED ASK-2 option (2), @@ -1344,6 +1345,10 @@ def build_parser(*, subcommand_extensions: tuple = ()) -> argparse.ArgumentParse # only where no host has registered its own. Registering reads nothing, so # a host that registers after this parser is built still replaces them. projection_seams.register_defaults() + # AND the consumer columns' defaults (plan 034 T084; #1144 4.3, + # R1Q10 (a)): the gate primitives, the doxBench scope, kickoff and + # the cross-reference register, the same way. + column_seams.register_defaults() parser = argparse.ArgumentParser(prog="ideation-dashboard", description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) sub = parser.add_subparsers(dest="command", required=True) @@ -1462,6 +1467,10 @@ def main(argv: list[str] | None = None, *, generator_seam.register_default(default_generator.GENERATOR) # AND openDox's own projection defaults (5.5, T055), the same way. projection_seams.register_defaults() + # AND the consumer columns' defaults (plan 034 T084; #1144 4.3, + # R1Q10 (a)): the gate primitives, the doxBench scope, kickoff and + # the cross-reference register, the same way. + column_seams.register_defaults() args = build_parser( subcommand_extensions=subcommand_extensions).parse_args(argv) try: diff --git a/src/opendox/column_seams.py b/src/opendox/column_seams.py new file mode 100644 index 00000000..94381856 --- /dev/null +++ b/src/opendox/column_seams.py @@ -0,0 +1,165 @@ +"""THE CONSUMER COLUMNS' SEAMS: the gate primitives, the doxBench scope, +kickoff and the cross-reference register. Each is a host's contribution or +openDox's own default, held for late resolution (plan 034 T084; #1144 4.3, as +T007 batch G's and batch L's addenda read). + +WHY THIS FILE EXISTS. #1144's 4.3: *"Route EVERY deferred reach through a seam +the product declares. None stays late-bound by name."* After phase 2, openDox's +own verbs still reached four mechanisms of openXdox's columns by module name: +the gate console (`branch_session` and `cli` through `consumer_reach`'s late +stand-in, and `serve_workbench`'s model approval and chat Save by deferred +imports), the doxBench scope authority (`serve_workbench`'s thread, chat-turn +and document-abstract routes), kickoff (`branch_session`'s proposal state and +`serve_project`'s register projection) and the cross-reference register +(`branch_session`'s pick fallbacks). With openXdox absent, which is the normal +state of a neutral openDox, each of those raised from inside a function, and +three of them ended a request with a dropped connection (batch L). This module +declares a seam for each, in `projection_seams`' discipline (the class is the +same one, `projection_seams._Seam`, naming this module). + +R1Q10 (a) (`openxFactory#656` comment `5850003126`, in R-G3's pattern): +openDox grows a small neutral default for each, `opendox.default_columns`, and +openXdox contributes its governed one through the same seam (T086). What each +default does, and the holder's reading behind it, is that module's docstring. + +THE ENTRY POINTS REGISTER THE DEFAULTS WHERE NO HOST HAS (R1Q3 (a), comment +`5817152735`). `cli.build_parser()`, `cli.main()`, `serve.build_server()` and +`serve.main()` call `register_defaults()`, beside `projection_seams`'. So a +process that runs none of them still meets `SeamNotRegistered`, naming the +seam and the call (4.2); a host registration made before a default has been +read replaces it; one made after is refused (`SeamAlreadyRegistered`); and the +same registration again is a no-op. These are `projection_seams`' exception +classes, so a verb that reports one projection-seam refusal in a clause reports +these in the same one. + +THE FOUR SEAMS, AND WHAT A REGISTRATION CARRIES. Each is a module, or any +object, carrying the names below. openXdox's modules carry them as they stand, +except the gate's: `first_edit_gate_factory` is `gate_routes`', so openXdox's +gate registration is `gate_console` with that one name beside it. + +* `gate`: `GATE_CALLABLES` and `GATE_VALUES`, every gate-console name openDox + reads. ONE registration for the whole family, because the names agree with + each other: a record builder checks a `Provenance` of its own type, and a + caller catches its own `GateRefused`. +* `scope`: `SCOPE_CALLABLES`, the scope authority. The scope VALUE types are + openDox's own (`opendox.doxbench_scope_types`) and are no seam. +* `kickoff`: `KICKOFF_CALLABLES`, the dispatched commissions and the project + register's discovery. +* `register`: `REGISTER_CALLABLES`, the cross-reference register's adapter. + +`gate_records_writable()` answers whether a HOST's gate is registered, so the +model-intake surface can decline to start a flow whose approval openDox's +default would refuse (plan 034 T084, a holder reading under batch G and R1Q10 +(a) that Brett may overrule). + +IMPORT WEIGHT: `opendox.projection_seams` only, which is stdlib-only, so this +module names no sibling in an import. `register_defaults()` imports +`opendox.default_columns` when it is CALLED. + +A CREATED FILE: no carve-manifest row (RULED OQ-C). +""" + +from __future__ import annotations + +from opendox import projection_seams +from opendox.projection_seams import ( + ProjectionSeamError, + SeamAlreadyRegistered, + SeamNotRegistered, +) + +__all__ = [ + "GATE_CALLABLES", + "GATE_VALUES", + "KICKOFF_CALLABLES", + "ProjectionSeamError", + "REGISTER_CALLABLES", + "SCOPE_CALLABLES", + "SeamAlreadyRegistered", + "SeamNotRegistered", + "gate", + "gate_records_writable", + "kickoff", + "register", + "register_defaults", + "scope", +] + +_MODULE = "opendox.column_seams" + +#: What a gate registration must carry as callables: the human-gate guard and +#: the types it works with, the record functions, the session-ref target id, +#: the first-edit gate builder chat's Save writes through, and the clock, the +#: stamp and the records prefix the session layer composes with them. +GATE_CALLABLES: tuple[str, ...] = ( + "GateConsole", "GateRefused", "HumanGate", "Provenance", + "build_gate_action_record", "validate_gate_action_record", + "write_gate_action_record", "validate_demotion_execution_receipt", + "require_human_gate", "ref_target_id", "first_edit_gate_factory", + "_prefix", "_stamp", "_utcnow") + +#: ...and as values: the session actions and the commit artifact, the default +#: records prefix, the HTTP console's provenance, and the CLI's surface and +#: presence proofs. +GATE_VALUES: tuple[str, ...] = ( + "ACTION_ABANDON_SESSION", "ACTION_CREATE_DOCUMENT", "ACTION_EDIT_DOCUMENT", + "ART_COMMIT", "DEFAULT_RECORDS_DIR", "HTTP_CONSOLE_TOKEN", + "PRESENCE_DECLARED", "PRESENCE_TTY", "SURFACE_CLI") + +#: What a scope registration must carry. +SCOPE_CALLABLES: tuple[str, ...] = ( + "resolve_scope", "is_live_session_ref", "session_created_paths_for_scope") + +#: What a kickoff registration must carry. +KICKOFF_CALLABLES: tuple[str, ...] = ( + "dispatched_commission_rows", "dispatched_commissions", + "dispatched_propose_topics", "discover_project_register") + +#: What a cross-reference register registration must carry. +REGISTER_CALLABLES: tuple[str, ...] = ("CrossReferenceIndexAdapter",) + +gate = projection_seams._Seam( + "gate", what="gate primitives", callables=GATE_CALLABLES, + values=GATE_VALUES, default="opendox.default_columns.GATE", + consequence="no gate action can be guarded, stamped or recorded", + module=_MODULE) + +scope = projection_seams._Seam( + "scope", what="doxBench scope authority", callables=SCOPE_CALLABLES, + default="opendox.default_columns.SCOPE", + consequence="no workbench tile can be resolved to its documents", + module=_MODULE) + +kickoff = projection_seams._Seam( + "kickoff", what="commission reader", callables=KICKOFF_CALLABLES, + default="opendox.default_columns.KICKOFF", + consequence="no dispatched commission, proposal or project register can " + "be read", + module=_MODULE) + +register = projection_seams._Seam( + "register", what="cross-reference register", callables=REGISTER_CALLABLES, + default="opendox.default_columns.REGISTER", + consequence="no cross-reference register can be read", + module=_MODULE) + + +def register_defaults() -> None: + """Register openDox's OWN default at each of the four seams, where no host + has registered one (R1Q10 (a), in R1Q3 (a)'s pattern). It registers + nothing over a host and reads nothing, so a host registration made + afterwards, and before any consumer reads a default, still replaces it. + Importing this module registers nothing.""" + from opendox import default_columns + + gate.register_default(default_columns.GATE) + scope.register_default(default_columns.SCOPE) + kickoff.register_default(default_columns.KICKOFF) + register.register_default(default_columns.REGISTER) + + +def gate_records_writable() -> bool: + """Whether a HOST's gate is registered, which is what can write a governed + gate-action record: openDox's own default writes none. Answers without + reading the seam, so it closes no default's window.""" + return gate.holds_a_hosts() diff --git a/src/opendox/default_columns.py b/src/opendox/default_columns.py new file mode 100644 index 00000000..4182f62e --- /dev/null +++ b/src/opendox/default_columns.py @@ -0,0 +1,492 @@ +"""openDox's OWN defaults for the four column seams: the gate primitives, the +doxBench scope, kickoff and the cross-reference register (plan 034 T084; +#1144 4.3 as T007 batch G's addendum reads; RULED R1Q10 (a), openxFactory#656 +comment `5850003126`). + +WHAT THIS MODULE IS. `opendox.column_seams` declares four seams where openDox's +own verbs reached the consumer's columns by module name (`openxdox.gate_console`, +`openxdox.gate_routes`, `openxdox.doxbench_scope`, `openxdox.kickoff`, +`openxdox.register`). Each seam is a host's contribution or openDox's default, +and the entry points register the defaults below where no host has (R1Q3 (a)'s +pattern, `5817152735`). openXdox contributes its governed ones through the same +seams (T086). Each default is small and neutral, and it is new code, not +openXdox's modules relocated: R1Q10's option (b), the relocation, was not the +ruling. + +WHAT EACH DEFAULT DOES, as the holder read R1Q10 (a) for T084 (relayed on +openxFactory#656's thread, 2026-10-02; Brett may overrule): + +* `GATE`, the gate primitives. The VOCABULARY-FREE primitives are real code: + the human-gate guard (`require_human_gate`, over `opendox.boundary`'s + `HumanGate`), the clock and the stamp (`_utcnow`, `_stamp`), the records + prefix (`_prefix`) and the session-ref target id (`ref_target_id`), the + gateway `Provenance` with its surface and presence constants and + `HTTP_CONSOLE_TOKEN`, `GateRefused`, the session action and artifact + constants, and the first-edit gate builder chat's Save writes through + (`first_edit_gate_factory`). The GOVERNED record functions, a gate-action + record's build, validation and write, the demotion-receipt validator and the + `GateConsole` facade, are NOT emulated: the gate-action record is a shape only + openxFactory's pinned schema declares, and openDox writes no governed shape it + does not own. With no host's gate registered they refuse, naming the seam and + the call that registers one (4.2), as `GateRecordsNotRegistered`. +* `SCOPE`, the doxBench scope authority, READ-ONLY. `resolve_scope` projects a + tile of the neutral snapshot (a group's members, a selection's files, a + candidate's claiming groups' members), each path confined to the selected + root by the registry seam's own `resolve_within`, and classifies nothing as + editable. `is_live_session_ref` keeps the governed authority's logic, which is + openDox's own session layer (`branch_session.live_session_branches`). + `session_created_paths_for_scope` answers no path: which documents a session + CREATED is known only from governed gate-action records, which this default + neither writes nor reads. +* `KICKOFF`: no dispatched commission and no dispatched proposal, and no + project register, so `/project-register.json` keeps today's structured + "no project register" answer. +* `REGISTER`: a cross-reference register with no possibles. + +NOTHING HERE NAMES THE CONSUMER OR THE PUBLISHER, at import time or in a body: +the standard library, `opendox.boundary`, `opendox.path_slug`, +`opendox.doxbench_threads` and `opendox.defaults` at import, and +`opendox.branch_session` and `opendox.projection_seams` inside the bodies that +need them, so `import opendox.default_columns` starts nothing. + +A CREATED FILE: no carve-manifest row (RULED OQ-C). +""" + +from __future__ import annotations + +import re +from dataclasses import dataclass +from datetime import datetime, timezone +from pathlib import Path, PurePosixPath +from typing import Any, Iterable, Mapping, Sequence + +from opendox import defaults +from opendox.boundary import GATE_SIDE_EFFECT, BoundaryViolation, HumanGate, Refusal +from opendox.doxbench_scope_types import ( + ScopeConfinementError, + ScopeDocument, + ScopeKey, + ScopeProjection, + ScopeSection, +) +from opendox.doxbench_threads import THREAD_PREFIX +from opendox.path_slug import slug + +__all__ = [ + "GATE", + "GATE_RECORDS_REFUSAL", + "GateRecordsNotRegistered", + "GateRefused", + "KICKOFF", + "Provenance", + "REGISTER", + "SCOPE", +] + + +class _Registration: + """One default, as a seam registration: a named object carrying the names + its seam requires. `__name__` is what a refusal names it by.""" + + def __init__(self, name: str, members: Mapping[str, Any]) -> None: + self.__name__ = name + for key, value in members.items(): + setattr(self, key, value) + + def __repr__(self) -> str: + return f"" + + +# ========================================================================== +# GATE — the vocabulary-free primitives, and refusals for the governed rest +# ========================================================================== + +class GateRefused(Exception): + """A gate action refused on a precondition. openDox's default raises it + for a refusal of its own, and `GateRecordsNotRegistered` below for the + governed functions it does not carry.""" + + +#: The fixed sentence a surface puts on the wire, and the refusal's text, where +#: a governed gate-action record is asked for and no host's gate is registered. +#: It names the seam and the call that registers one (4.2). +GATE_RECORDS_REFUSAL = ( + "this install records no governed gate actions: no host's gate is " + "registered at openDox's gate seam (opendox.column_seams.gate), and " + "openDox's own default writes no gate-action record, because that record " + "is a shape only the governing host's pinned schema declares. A host that " + "records gate actions registers its gate at process start with " + "opendox.column_seams.gate.register(). A model is " + "declared on this install with `opendox model-binding add`, which needs " + "no approval record.") + + +class GateRecordsNotRegistered(GateRefused): + """A governed gate function was asked for and no host's gate is + registered: openDox's default carries none of them (see the module + docstring).""" + + def __init__(self, what: str) -> None: + super().__init__(f"{what}: {GATE_RECORDS_REFUSAL}") + + +def _governed(what: str): + def refuse(*_args: Any, **_kwargs: Any): + raise GateRecordsNotRegistered(what) + + refuse.__name__ = what.split(" ", 1)[0] + refuse.__doc__ = (f"`{what}`, which openDox's default does not carry: " + "refused as `GateRecordsNotRegistered`.") + return refuse + + +class _NoGateConsole: + """`GateConsole`, which openDox's default does not carry: constructing it + refuses as `GateRecordsNotRegistered`.""" + + def __init__(self, *_args: Any, **_kwargs: Any) -> None: + raise GateRecordsNotRegistered("GateConsole") + + +#: The gateway SURFACES and the CONSOLE-PRESENCE proofs a provenance block may +#: name. The words are the gateway facts openDox's own entry points observe +#: (`cli.console_presence`, the console-token check), and nothing governs them +#: but this list. +SURFACE_HTTP = "http" +SURFACE_CLI = "cli" +SURFACES = (SURFACE_HTTP, SURFACE_CLI) +PRESENCE_CONSOLE_TOKEN = "console-token" +PRESENCE_TTY = "tty" +PRESENCE_DECLARED = "declared" +CONSOLE_PRESENCES = (PRESENCE_CONSOLE_TOKEN, PRESENCE_TTY, PRESENCE_DECLARED) + + +@dataclass(frozen=True) +class Provenance: + """The gateway facts about one invocation: the surface it arrived on and + how console presence was shown. A type, validated at construction, so a + mapping a request body could carry is never a provenance.""" + + surface: str + console_presence: str + + def __post_init__(self) -> None: + if self.surface not in SURFACES: + raise GateRefused( + f"unknown gateway surface {self.surface!r}: a provenance names " + f"one of {', '.join(SURFACES)}") + if self.console_presence not in CONSOLE_PRESENCES: + raise GateRefused( + f"unknown console-presence proof {self.console_presence!r}: a " + f"provenance names one of {', '.join(CONSOLE_PRESENCES)}, and " + "there is no value meaning 'presence was not shown'") + + def as_record(self) -> dict: + return {"surface": self.surface, "console_presence": self.console_presence} + + +HTTP_CONSOLE_TOKEN = Provenance(SURFACE_HTTP, PRESENCE_CONSOLE_TOKEN) + + +def require_human_gate(gate: Any) -> HumanGate: + """A `HumanGate` passes; anything else is REJECTED and REPORTED: appended to + its own refusal ledger where it has one, then raised as a + `BoundaryViolation` carrying the structured `Refusal`.""" + if isinstance(gate, HumanGate): + return gate + actor = (getattr(gate, "actor", None) + or getattr(gate, "human_actor", None) or "non-human") + refusal = Refusal( + GATE_SIDE_EFFECT, str(actor), "", + "a gate action requires a HumanGate constructed with an identified " + "human actor; an OutputBoundary or agent path cannot invoke one") + ledger = getattr(gate, "refusals", None) + if isinstance(ledger, list): + ledger.append(refusal) + raise BoundaryViolation(refusal) + + +def _utcnow() -> str: + """Wall-clock UTC, as a date-time: a gate action is a live event.""" + return datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") + + +def _stamp(at: str) -> str: + """A filesystem-safe slug of a timestamp (its separators dropped).""" + return re.sub(r"[^0-9A-Za-z]", "", at) or "unstamped" + + +def _prefix(records_dir: str) -> str: + return records_dir if records_dir.endswith("/") else records_dir + "/" + + +def ref_target_id(ref: str) -> str: + """The records-path segment for a session ref: the branch, slugged, so a + hostile value can never traverse the records tree.""" + return slug(str(ref or "").replace("/", "-")) + + +def first_edit_gate_factory(actor: str, records_dir: str): + """The worktree-rooted gate chat's Save writes through: the records tree + and the thread-sidecar tree declared, and nothing else.""" + def build(worktree): + return HumanGate(worktree, [records_dir, THREAD_PREFIX], + human_actor=actor, session_root=worktree) + return build + + +GATE = _Registration("opendox.default_columns.GATE", { + # the vocabulary-free primitives, as real code + "GateRefused": GateRefused, + "HumanGate": HumanGate, + "Provenance": Provenance, + "require_human_gate": require_human_gate, + "ref_target_id": ref_target_id, + "first_edit_gate_factory": first_edit_gate_factory, + "_prefix": _prefix, + "_stamp": _stamp, + "_utcnow": _utcnow, + "ACTION_ABANDON_SESSION": "abandon-session", + "ACTION_CREATE_DOCUMENT": "create-document", + "ACTION_EDIT_DOCUMENT": "edit-document", + "ART_COMMIT": "commit", + "DEFAULT_RECORDS_DIR": defaults.DEFAULT_RECORDS_DIR, + "HTTP_CONSOLE_TOKEN": HTTP_CONSOLE_TOKEN, + "PRESENCE_DECLARED": PRESENCE_DECLARED, + "PRESENCE_TTY": PRESENCE_TTY, + "SURFACE_CLI": SURFACE_CLI, + # the governed rest, refused by name (4.2) + "GateConsole": _NoGateConsole, + "build_gate_action_record": _governed("build_gate_action_record"), + "validate_gate_action_record": _governed("validate_gate_action_record"), + "write_gate_action_record": _governed("write_gate_action_record"), + "validate_demotion_execution_receipt": _governed( + "validate_demotion_execution_receipt"), +}) + + +# ========================================================================== +# SCOPE — a small, neutral, READ-ONLY projection of a tile +# ========================================================================== + +def _mapping(value: Any) -> Mapping[str, Any]: + return value if isinstance(value, Mapping) else {} + + +def _sequence(value: Any) -> Sequence[Any]: + return value if isinstance(value, (list, tuple)) else () + + +def _text(value: Any) -> str: + return value.strip() if isinstance(value, str) else "" + + +def _canonical(path: Any) -> str: + """`path` if it is a canonical repository-relative POSIX path, or a + `ScopeConfinementError`: a projection never carries a path it cannot + name safely.""" + if (not isinstance(path, str) or not path or "\x00" in path or "\\" in path + or path.startswith("/")): + raise ScopeConfinementError( + "scope paths must be non-empty repository-relative POSIX paths") + parts = PurePosixPath(path).parts + if PurePosixPath(path).as_posix() != path or ".." in parts or "." in parts: + raise ScopeConfinementError( + f"scope path {path!r} must use canonical repository-relative spelling") + return path + + +def _section(key: str, label: str, note: str, paths: Iterable[Any], *, + known: set[str], seen: set[str], root: Path, + inherited: bool) -> ScopeSection: + from opendox import projection_seams + + resolve_within = projection_seams.registry.current().resolve_within + rows: list[ScopeDocument] = [] + for raw in paths: + path = _canonical(raw) + if path in seen: + continue + seen.add(path) + resolved = path in known and resolve_within(root, path) is not None + rows.append(ScopeDocument(id=path, path=path, resolved=resolved)) + return ScopeSection(key=key, label=label, note=note, inherited=inherited, + owned=False, documents=tuple(rows)) + + +def resolve_scope(snapshot: Mapping[str, Any], key: ScopeKey, *, + source_root: Path, created_paths: Iterable[str] = () + ) -> ScopeProjection | None: + """One tile of the neutral snapshot, read-only, or None where the snapshot + has no such tile. + + A group (`cluster`) projects its members, a selection (`staged`) its files, + and a candidate (`possible`) the members of the groups that claim it. Each + path is confined to `source_root` by the registry seam's own + `resolve_within`, and NOTHING is editable: `editable_paths` and + `active_document_candidates` are empty, and there is no outline. A + `created_paths` entry is confined and readable, never editable.""" + if not isinstance(snapshot, Mapping): + return None + if isinstance(created_paths, (str, bytes, bytearray)): + raise ScopeConfinementError( + "created_paths must be a collection of repository-relative paths") + try: + created = tuple(created_paths) + except TypeError as error: + raise ScopeConfinementError( + "created_paths must be a collection of repository-relative paths" + ) from error + root = Path(source_root) + known = {_text(_mapping(d).get("path")) for d in _sequence(snapshot.get("documents"))} + groups = {_text(_mapping(g).get("id")): _mapping(g) + for g in _sequence(snapshot.get("clusters"))} + + def members(group: Mapping[str, Any]) -> list[Any]: + return [_mapping(edge).get("document") + for edge in _sequence(group.get("document_edges"))] + + seen: set[str] = set() + sections: list[ScopeSection] = [] + keywords: tuple[str, ...] = () + if key.tile_kind == "cluster": + group = groups.get(key.tile_id) + if group is None: + return None + title = _text(group.get("name")) or key.tile_id + keywords = tuple(_text(t) for t in _sequence(group.get("topics")) if _text(t)) + sections.append(_section( + "members", "group documents", "the group's own document edges", + members(group), known=known, seen=seen, root=root, inherited=False)) + elif key.tile_kind == "staged": + selection = next((_mapping(s) for s in _sequence(snapshot.get("staged_topics")) + if _text(_mapping(s).get("staging_id")) == key.tile_id), None) + if selection is None: + return None + title = key.tile_id + sections.append(_section( + "files", "selection files", "the documents this selection names", + _sequence(selection.get("files")), known=known, seen=seen, root=root, + inherited=False)) + elif key.tile_kind == "possible": + candidate = next((_mapping(p) for p in _sequence(snapshot.get("possibles")) + if _text(_mapping(p).get("id")) == key.tile_id), None) + if candidate is None: + return None + title = _text(candidate.get("title")) or key.tile_id + claimed = [path for group_id in _sequence(candidate.get("claiming_clusters")) + for path in members(groups.get(_text(group_id), {}))] + sections.append(_section( + "claiming", "documents of the claiming groups", + "membership inferred from the groups that claim this candidate", + claimed, known=known, seen=seen, root=root, inherited=True)) + else: + return None + context = [row.path for section in sections for row in section.documents + if row.resolved] + for raw in created: + path = _canonical(raw) + if path not in context: + context.append(path) + revision = _text(_mapping(snapshot.get("generation")).get("source_revision")) + return ScopeProjection( + key=key, title=title, keywords=keywords, source_revision=revision, + sections=tuple(sections), context_paths=tuple(context), + editable_paths=(), outline_path=None, active_document_candidates=()) + + +def _normalize_ref(ref: Any) -> str: + from opendox import projection_seams + + text = str(ref).strip() if ref is not None else "" + return text or projection_seams.registry.current().DEFAULT_REF + + +def is_live_session_ref(registry: Any, key: ScopeKey, *, repository: str, + ref: str) -> bool: + """Whether `ref` is one of THIS tile's live session branches, by openDox's + own session layer. A declared session refusal (a cross-tile collision, an + ambiguous family) is "not this tile's session"; any other failure is the + caller's to handle.""" + from opendox import branch_session + + kinds = {"cluster": branch_session.CLUSTER, + "possible": branch_session.POSSIBLE, + "staged": branch_session.STAGED_TOPIC} + scope_kind = kinds.get(key.tile_kind) + if scope_kind is None or not ref or registry is None: + return False + try: + tile = branch_session.Tile(scope_kind, key.tile_id) + live = branch_session.live_session_branches(registry, repository, tile) + except branch_session.SessionRefused: + return False + wanted = _normalize_ref(ref) + return any(wanted == _normalize_ref(branch) for branch in live) + + +def session_created_paths_for_scope(registry: Any, key: ScopeKey, *, + repository: str, ref: str, + source_root: Path | str) -> tuple[str, ...]: + """No path: which documents a session created is known only from the + governed gate-action records, which this default neither writes nor + reads.""" + return () + + +SCOPE = _Registration("opendox.default_columns.SCOPE", { + "resolve_scope": resolve_scope, + "is_live_session_ref": is_live_session_ref, + "session_created_paths_for_scope": session_created_paths_for_scope, +}) + + +# ========================================================================== +# KICKOFF and REGISTER — nothing dispatched, no register +# ========================================================================== + +def dispatched_commission_rows(records_root: Any, verb: str) -> list: + """No dispatched commission: openDox commissions no workflow.""" + return [] + + +def dispatched_commissions(records_root: Any, verb: str) -> dict: + """No dispatched commission, by target.""" + return {} + + +def dispatched_propose_topics(records_root: Any) -> set: + """No dispatched proposal.""" + return set() + + +def discover_project_register(root: Any) -> None: + """No project register: openDox's own corpus declares none.""" + return None + + +KICKOFF = _Registration("opendox.default_columns.KICKOFF", { + "dispatched_commission_rows": dispatched_commission_rows, + "dispatched_commissions": dispatched_commissions, + "dispatched_propose_topics": dispatched_propose_topics, + "discover_project_register": discover_project_register, +}) + + +class _NoPossibles: + def possibles(self) -> tuple: + return () + + +class CrossReferenceIndexAdapter: + """A cross-reference register with no possibles: openDox's own corpus + declares none.""" + + @classmethod + def discover(cls, root: Any) -> _NoPossibles: + return _NoPossibles() + + +REGISTER = _Registration("opendox.default_columns.REGISTER", { + "CrossReferenceIndexAdapter": CrossReferenceIndexAdapter, +}) diff --git a/src/opendox/projection_seams.py b/src/opendox/projection_seams.py index c426196a..76638bde 100644 --- a/src/opendox/projection_seams.py +++ b/src/opendox/projection_seams.py @@ -234,12 +234,21 @@ class _Seam: `callables` and `values` are the names a registration must carry. `default` names openDox's own default, for the refusal. The records a registration keeps are whether it is the entry point's default, and whether - a consumer has read that default since it was registered.""" + a consumer has read that default since it was registered. + + `module` names the module that declares the seam, in every refusal and in + the call a host makes. It is this module's own by default, and + `opendox.column_seams` declares its four seams through the same class + (plan 034 T084), so every seam of openDox's keeps one discipline.""" def __init__(self, name: str, *, what: str, callables: tuple[str, ...], values: tuple[str, ...] = (), default: str, - consequence: str) -> None: + consequence: str, + module: str = "opendox.projection_seams") -> None: self.name = name + self.module = module + #: The module's own name without the package, as a refusal names a call. + self._short = module.rsplit(".", 1)[-1] self.what = what self.callables = callables self.values = values @@ -247,7 +256,7 @@ def __init__(self, name: str, *, what: str, callables: tuple[str, ...], self.consequence = consequence #: The ONE call a host makes, quoted verbatim in every refusal. self.registration_call = ( - f"opendox.projection_seams.{name}.register()") + f"{module}.{name}.register()") self._registered: Any = None self._is_default = False self._default_read = False @@ -272,7 +281,7 @@ def register(self, registration: Any) -> Any: if registration is not None and registration is self._registered: return registration _probe(registration, self.callables, self.values, - f"projection_seams.{self.name}.register()", f"the host's {self.what}") + f"{self._short}.{self.name}.register()", f"the host's {self.what}") with self._lock: held = self._registered if held is registration: @@ -289,7 +298,7 @@ def register(self, registration: Any) -> Any: f"{name_of(registration)} would replace it. Registration " "happens ONCE, at process start: one process holding two " f"would split its consumers between them. Call " - f"opendox.projection_seams.{self.name}.unregister() first if " + f"{self.module}.{self.name}.unregister() first if " "the swap is deliberate.") raise SeamAlreadyRegistered( f"openDox's own default {self.what} ({name_of(held)}) is " @@ -300,7 +309,7 @@ def register(self, registration: Any) -> Any: "would (R1Q3 (ii), openxFactory#656 comment 5817152735; RN-1 (a), " "comment 5850003126). Register the host's own at process start, " f"ahead of {_ENTRY_POINTS}. Call " - f"opendox.projection_seams.{self.name}.unregister() first if the " + f"{self.module}.{self.name}.unregister() first if the " "swap is deliberate.") def register_default(self, registration: Any) -> Any: @@ -315,7 +324,7 @@ def register_default(self, registration: Any) -> Any: if registration is not None and registration is self._registered: return registration _probe(registration, self.callables, self.values, - f"projection_seams.{self.name}.register_default()", + f"{self._short}.{self.name}.register_default()", f"openDox's own default {self.what}") with self._lock: if self._registered is None: @@ -332,6 +341,13 @@ def is_registered(self) -> bool: """Is anything registered? Answers without reading or refusing.""" return self._registered is not None + def holds_a_hosts(self) -> bool: + """Is a HOST's registration held here, not the entry point's default + and not nothing? Answers without reading, so it closes no default's + window, and without refusing.""" + with self._lock: + return self._registered is not None and not self._is_default + def current(self) -> Any: """The registration, or a refusal naming this seam and its call. @@ -344,13 +360,13 @@ def current(self) -> Any: if registered is None: raise SeamNotRegistered( f"no {self.what} is registered at openDox's {self.name} seam " - f"(opendox.projection_seams.{self.name}), so {self.consequence}. " + f"({self.module}.{self.name}), so {self.consequence}. " f"openDox ships its own, {self.default}, but it is a " "registration an ENTRY POINT makes and never a fallback here " f"({_DEFAULTS_RULING}). {_ENTRY_POINTS} register it where no " "host has. Nothing is registered now, so either nothing in " "this process has run one of them, or " - f"opendox.projection_seams.{self.name}.unregister() has " + f"{self.module}.{self.name}.unregister() has " "dropped the registration since. A host that contributes its " "own registers it at process start with\n\n " + self.registration_call + "\n\nbefore anything reads it.") diff --git a/src/opendox/serve.py b/src/opendox/serve.py index a47d4b8b..ff9ef498 100644 --- a/src/opendox/serve.py +++ b/src/opendox/serve.py @@ -151,6 +151,7 @@ from opendox import consumer_reach # noqa: E402 from opendox import defaults # noqa: E402 from opendox import projection_seams # noqa: E402 +from opendox import column_seams # noqa: E402 # § 3.4 slice S5: `build_server()` publishes the VIEW MANIFEST on # `/capabilities`, the one line slice S3 built both ends of and left for the # slice at which a contribution first exists to deliver. @@ -1881,6 +1882,10 @@ def build_server( # from the registered registry, which is what lets a server be BUILT with # nothing else installed (plan 034, research R7). projection_seams.register_defaults() + # AND the consumer columns' defaults (plan 034 T084; #1144 4.3, + # R1Q10 (a)): the gate primitives, the doxBench scope, kickoff and + # the cross-reference register, the same way. + column_seams.register_defaults() from opendox import doxbench_turns # Imported HERE rather than at module scope, for the reason that is @@ -2400,6 +2405,10 @@ def main(argv: list[str] | None = None) -> int: # BEFORE the parser: its option defaults below read the registered # registry (`registry_mod.DEFAULT_REF` and the data source's defaults). projection_seams.register_defaults() + # AND the consumer columns' defaults (plan 034 T084; #1144 4.3, + # R1Q10 (a)): the gate primitives, the doxBench scope, kickoff and + # the cross-reference register, the same way. + column_seams.register_defaults() parser = argparse.ArgumentParser(prog="ideation-dashboard-serve", description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) parser.add_argument("--web-dir", default=str(Path(__file__).resolve().parent / "web"), diff --git a/tests/test_column_seams.py b/tests/test_column_seams.py new file mode 100644 index 00000000..f47c3ca0 --- /dev/null +++ b/tests/test_column_seams.py @@ -0,0 +1,327 @@ +"""The consumer columns' seams and openDox's defaults for them (plan 034 T084; +#1144 4.3 as T007 batch G's addendum reads; RULED R1Q10 (a), `5850003126`). + +`opendox.column_seams` declares four seams, the gate primitives, the doxBench +scope, kickoff and the cross-reference register, in `projection_seams`' +discipline, and `opendox.default_columns` is openDox's own default for each. +These cases hold: + +1. A BARE PROCESS, in which no entry point ran, meets `SeamNotRegistered` at + each seam, naming the seam and the call that registers one (4.2). A default + is a registration an entry point makes, never a fallback inside the seam. +2. EACH ENTRY POINT registers the four defaults where no host has + (`cli.build_parser()`, `cli.main()`, `serve.build_server()`, `serve.main()` + read with `ast`, and `cli.build_parser()` run in a child). A host's + registration made before a default is read replaces it, one made after is + refused, and a registration that lacks a name is refused naming it. +3. THE GATE DEFAULT carries the vocabulary-free primitives as real code, and + refuses the governed record functions and `GateConsole` by name, as + `GateRecordsNotRegistered` (a `GateRefused`). `gate_records_writable()` is + true only with a host's gate. +4. THE SCOPE DEFAULT projects each kind of tile read-only, confines every + path, and answers no session-created path; the kickoff and register defaults + answer nothing dispatched and no possibles. + +A CREATED FILE: no carve-manifest row (RULED OQ-C). +""" + +from __future__ import annotations + +import ast +import subprocess +import sys +import textwrap +from pathlib import Path + +import pytest + +from opendox import column_seams as cs +from opendox import default_columns as dc +from opendox import projection_seams as ps +from opendox.boundary import BoundaryViolation, HumanGate, OutputBoundary +from opendox.doxbench_scope_types import ScopeConfinementError, ScopeKey + +ROOT = Path(__file__).resolve().parent.parent +SRC = ROOT / "src" / "opendox" +SEAMS = (cs.gate, cs.scope, cs.kickoff, cs.register) + + +@pytest.fixture() +def isolated(): + """Each column seam empty, and put back whole afterwards.""" + held = [(seam._registered, seam._is_default, seam._default_read) for seam in SEAMS] + for seam in SEAMS: + seam.unregister() + try: + yield + finally: + for seam, (registered, is_default, read) in zip(SEAMS, held): + seam._registered, seam._is_default, seam._default_read = ( + registered, is_default, read) + + +def _child(program: str) -> subprocess.CompletedProcess: + return subprocess.run([sys.executable, "-c", textwrap.dedent(program)], + capture_output=True, text=True, cwd=str(ROOT)) + + +# --------------------------------------------------------------------------- +# 1 — a bare process refuses at each seam, naming it +# --------------------------------------------------------------------------- + +@pytest.mark.parametrize("name", ["gate", "scope", "kickoff", "register"]) +def test_a_bare_process_refuses_naming_the_seam_and_the_call(name) -> None: + done = _child(f""" + from opendox import column_seams as cs + try: + cs.{name}.current() + except cs.SeamNotRegistered as e: + print("REFUSED", e) + else: + print("ANSWERED") + """) + assert done.returncode == 0, done.stderr + out = done.stdout + assert out.startswith("REFUSED"), out + assert f"opendox.column_seams.{name}" in out, out + assert f"opendox.column_seams.{name}.register(" in out, out + assert "opendox.default_columns" in out, out + + +# --------------------------------------------------------------------------- +# 2 — the entry points register the defaults; hosts replace or are refused +# --------------------------------------------------------------------------- + +def _calls_register_defaults(function: ast.FunctionDef) -> bool: + return any(isinstance(node, ast.Call) + and isinstance(node.func, ast.Attribute) + and node.func.attr == "register_defaults" + and isinstance(node.func.value, ast.Name) + and node.func.value.id == "column_seams" + for node in ast.walk(function)) + + +@pytest.mark.parametrize("module,function", [ + ("cli.py", "build_parser"), ("cli.py", "main"), + ("serve.py", "build_server"), ("serve.py", "main")]) +def test_each_entry_point_registers_the_column_defaults(module, function) -> None: + tree = ast.parse((SRC / module).read_text(encoding="utf-8")) + [found] = [node for node in tree.body + if isinstance(node, ast.FunctionDef) and node.name == function] + assert _calls_register_defaults(found), ( + f"{module}:{function}() does not call column_seams.register_defaults()") + + +def test_building_the_parser_registers_openDoxs_own_defaults() -> None: + done = _child(""" + from opendox import cli, column_seams as cs, default_columns as dc + cli.build_parser() + print(cs.gate.current() is dc.GATE, cs.scope.current() is dc.SCOPE, + cs.kickoff.current() is dc.KICKOFF, + cs.register.current() is dc.REGISTER, + cs.gate_records_writable()) + """) + assert done.returncode == 0, done.stderr + assert done.stdout.split() == ["True", "True", "True", "True", "False"] + + +class _HostGate: + """A host's gate: openDox's default's names, with a record writer.""" + + def __init__(self) -> None: + for name in (*cs.GATE_CALLABLES, *cs.GATE_VALUES): + setattr(self, name, getattr(dc.GATE, name)) + self.written = [] + self.write_gate_action_record = lambda gate, records_dir, record: ( + self.written.append(record) or Path(records_dir) / "record.yaml") + + +def test_a_host_gate_before_a_read_replaces_the_default(isolated) -> None: + cs.register_defaults() + host = _HostGate() + assert cs.gate.register(host) is host + assert cs.gate.current() is host + assert cs.gate_records_writable() is True + + +def test_a_host_gate_after_the_default_was_read_is_refused(isolated) -> None: + cs.register_defaults() + assert cs.gate.current() is dc.GATE + with pytest.raises(cs.SeamAlreadyRegistered, + match="opendox.column_seams.gate.unregister"): + cs.gate.register(_HostGate()) + assert cs.gate_records_writable() is False + + +def test_a_registration_lacking_a_name_is_refused_naming_it(isolated) -> None: + class _Partial: + resolve_scope = staticmethod(dc.resolve_scope) + + with pytest.raises(TypeError, match="is_live_session_ref") as caught: + cs.scope.register(_Partial()) + assert "column_seams.scope.register()" in str(caught.value) + + +def test_gate_records_are_writable_only_with_a_hosts_gate(isolated) -> None: + assert cs.gate_records_writable() is False # nothing at all + cs.register_defaults() + assert cs.gate_records_writable() is False # openDox's default + cs.gate.unregister() + cs.gate.register(_HostGate()) + assert cs.gate_records_writable() is True + + +def test_the_defaults_carry_every_name_their_seams_require() -> None: + for default, names in ((dc.GATE, (*cs.GATE_CALLABLES, *cs.GATE_VALUES)), + (dc.SCOPE, cs.SCOPE_CALLABLES), + (dc.KICKOFF, cs.KICKOFF_CALLABLES), + (dc.REGISTER, cs.REGISTER_CALLABLES)): + missing = [name for name in names if not hasattr(default, name)] + assert missing == [], (default, missing) + + +# --------------------------------------------------------------------------- +# 3 — the gate default: real primitives, governed functions refused by name +# --------------------------------------------------------------------------- + +def test_the_human_gate_guard_passes_a_human_and_reports_anything_else(tmp_path) -> None: + human = HumanGate(tmp_path, ["records/"], human_actor="fixture") + assert dc.GATE.require_human_gate(human) is human + machinery = OutputBoundary(tmp_path, ["records/"], actor="agent") + with pytest.raises(BoundaryViolation): + dc.GATE.require_human_gate(machinery) + assert machinery.refusals, "the refusal was not reported on its own ledger" + + +def test_the_stamp_the_clock_the_prefix_and_the_ref_target() -> None: + assert dc.GATE._stamp("2026-10-02T20:00:00Z") == "20261002T200000Z" + assert dc.GATE._prefix("records") == "records/" + assert dc.GATE._prefix("records/") == "records/" + assert dc.GATE.ref_target_id("cluster/cl-a") == "cluster-cl-a" + assert len(dc.GATE._utcnow()) == len("2026-10-02T20:00:00Z") + + +def test_a_provenance_is_a_validated_type() -> None: + assert dc.GATE.HTTP_CONSOLE_TOKEN.as_record() == { + "surface": "http", "console_presence": "console-token"} + cli_tty = dc.GATE.Provenance(dc.GATE.SURFACE_CLI, dc.GATE.PRESENCE_TTY) + assert cli_tty.as_record() == {"surface": "cli", "console_presence": "tty"} + with pytest.raises(dc.GATE.GateRefused): + dc.GATE.Provenance("smoke-signal", dc.GATE.PRESENCE_TTY) + + +def test_the_first_edit_gate_declares_the_records_and_thread_trees(tmp_path) -> None: + from opendox.doxbench_threads import THREAD_PREFIX + + gate = dc.GATE.first_edit_gate_factory("fixture", "records/")(tmp_path) + assert isinstance(gate, HumanGate) + assert set(gate.output.allowlist) >= {"records/", THREAD_PREFIX} + assert gate.output.session_root == tmp_path.resolve() + + +@pytest.mark.parametrize("name", [ + "build_gate_action_record", "validate_gate_action_record", + "write_gate_action_record", "validate_demotion_execution_receipt", + "GateConsole"]) +def test_the_governed_functions_refuse_naming_the_seam(name) -> None: + with pytest.raises(dc.GateRecordsNotRegistered) as caught: + getattr(dc.GATE, name)(object(), "records/", {}) + assert isinstance(caught.value, dc.GATE.GateRefused) + text = str(caught.value) + assert "opendox.column_seams.gate" in text, text + assert "opendox.column_seams.gate.register(" in text, text + assert "model-binding add" in text, text + + +# --------------------------------------------------------------------------- +# 4 — the scope, kickoff and register defaults +# --------------------------------------------------------------------------- + +def _snapshot() -> dict: + return { + "generation": {"source_revision": "abc123"}, + "documents": [{"id": p, "path": p} for p in + ("a.md", "b.md", "c.md", "sel.md", "gone.md")], + "clusters": [ + {"id": "g1", "name": "Group one", "topics": ["barrel"], + "document_edges": [{"document": "a.md"}, {"document": "b.md"}]}, + {"id": "g2", "name": "Group two", "topics": ["shed"], + "document_edges": [{"document": "c.md"}, {"document": "gone.md"}]}], + "possibles": [{"id": "p1", "title": "A candidate", + "claiming_clusters": ["g1", "g2"]}], + "staged_topics": [{"staging_id": "s1", "files": ["sel.md"]}], + } + + +@pytest.fixture() +def corpus(tmp_path): + for name in ("a.md", "b.md", "c.md", "sel.md"): + (tmp_path / name).write_text("# x\n", encoding="utf-8") + ps.register_defaults() # the registry seam's containment rule + return tmp_path + + +def _key(kind: str, tile: str) -> ScopeKey: + return ScopeKey(repository="fixture", ref="main", tile_kind=kind, tile_id=tile) + + +@pytest.mark.parametrize("kind,tile,context,title", [ + ("cluster", "g1", ("a.md", "b.md"), "Group one"), + ("staged", "s1", ("sel.md",), "s1"), + ("possible", "p1", ("a.md", "b.md", "c.md"), "A candidate"), +]) +def test_each_tile_projects_read_only(corpus, kind, tile, context, title) -> None: + projection = dc.resolve_scope(_snapshot(), _key(kind, tile), source_root=corpus) + assert projection.title == title + assert projection.context_paths == context + assert projection.editable_paths == () + assert projection.active_document_candidates == () + assert projection.outline_path is None + assert projection.source_revision == "abc123" + assert all(not section.owned for section in projection.sections) + + +def test_a_listed_document_missing_from_the_tree_is_not_resolved(corpus) -> None: + projection = dc.resolve_scope(_snapshot(), _key("cluster", "g2"), source_root=corpus) + rows = {row.path: row.resolved for row in projection.sections[0].documents} + assert rows == {"c.md": True, "gone.md": False} + assert projection.context_paths == ("c.md",) + + +def test_an_unknown_tile_is_none(corpus) -> None: + assert dc.resolve_scope(_snapshot(), _key("cluster", "nope"), source_root=corpus) is None + + +@pytest.mark.parametrize("path", ["../escape.md", "/etc/passwd", "a/../b.md", "a\\b.md", ""]) +def test_a_path_the_scope_cannot_name_safely_is_refused(corpus, path) -> None: + snapshot = _snapshot() + snapshot["clusters"][0]["document_edges"].append({"document": path}) + with pytest.raises(ScopeConfinementError): + dc.resolve_scope(snapshot, _key("cluster", "g1"), source_root=corpus) + + +def test_a_symlink_out_of_the_root_is_not_resolved(corpus, tmp_path_factory) -> None: + outside = tmp_path_factory.mktemp("outside") / "secret.md" + outside.write_text("secret\n", encoding="utf-8") + (corpus / "link.md").symlink_to(outside) + snapshot = _snapshot() + snapshot["documents"].append({"id": "link.md", "path": "link.md"}) + snapshot["clusters"][0]["document_edges"].append({"document": "link.md"}) + projection = dc.resolve_scope(snapshot, _key("cluster", "g1"), source_root=corpus) + assert "link.md" not in projection.context_paths + + +def test_no_session_created_path_and_no_live_session_without_a_registry() -> None: + key = _key("cluster", "g1") + assert dc.session_created_paths_for_scope( + None, key, repository="fixture", ref="cluster/g1", source_root=".") == () + assert dc.is_live_session_ref(None, key, repository="fixture", + ref="cluster/g1") is False + + +def test_kickoff_and_register_answer_nothing(tmp_path) -> None: + assert dc.KICKOFF.dispatched_commissions(tmp_path, "create-project") == {} + assert dc.KICKOFF.dispatched_commission_rows(tmp_path, "edit-project") == [] + assert dc.KICKOFF.dispatched_propose_topics(tmp_path) == set() + assert dc.KICKOFF.discover_project_register(tmp_path) is None + assert tuple(dc.REGISTER.CrossReferenceIndexAdapter.discover(tmp_path).possibles()) == () From 0488f5bd3d7fb10b1096ef91d6cee2e7bb6b2950 Mon Sep 17 00:00:00 2001 From: Brett Heap <1513478+brettheap@users.noreply.github.com> Date: Fri, 2 Oct 2026 20:58:33 +0000 Subject: [PATCH 30/88] Fix round: nothing is made through a path the tree check would refuse (Copilot review) Copilot's review at 19e32f0c: _prepare_directories made the missing postgres/run before the tree check judged the path. So a component the check refuses had already been written through: another user's link (the directories were made in its target), or a directory every user can write. In a sticky directory such as /tmp, another user could also put the state directory's name in place between the check and the mkdir. - What exists is judged first. _refuse_an_unsafe_tree(existing_only=True) asks the same rules of only what exists, the link-ownership loop first so a foreign link is named even when broken. The whole tree is still judged again after the directories are made, before the socket directory's chmod. - _make_private_directories opens the deepest existing directory and makes each missing component relative to its parent's descriptor. It opens each one O_NOFOLLOW and asks fstat that it is this user's alone before anything is made beneath it. A planted link, non-directory or foreign directory is the named refusal, never followed and never re-moded. - Each component is born 0700 under a umask of 077, which is put back after, so there is no chmod window at all. The six new cases fail against 19e32f0c's bundle.py and pass here. Seven mutants are killed, one of them only after the umask test was taught to assert the restore. Lane: openxfactory-4 (openXfactory-4-openDox_extraction) Co-Authored-By: Claude Opus 5.5 (1M context) --- src/opendox/runtime/bundle.py | 104 +++++++++++++++++------ tests_runtime/test_local_lifecycle.py | 114 +++++++++++++++++++++++--- 2 files changed, 185 insertions(+), 33 deletions(-) diff --git a/src/opendox/runtime/bundle.py b/src/opendox/runtime/bundle.py index f5ff03c0..bd7834d4 100644 --- a/src/opendox/runtime/bundle.py +++ b/src/opendox/runtime/bundle.py @@ -346,7 +346,7 @@ def _preexec() -> None: # pragma: no cover - runs in the child def _make_private_directories(leaf: Path) -> None: - """`leaf` and every missing directory above it, each created 0700. + """`leaf` and every missing directory above it, each born exactly 0700. `Path.mkdir(parents=True)` gives the directories it creates on the way the default mode less the umask, whatever mode the leaf is given. So @@ -354,20 +354,59 @@ def _make_private_directories(leaf: Path) -> None: create `.local` and `state` group-writable, and the tree check would then refuse the directories this install had just made (Copilot review of openDox-code#69). Each missing component is created here, one at a - time, and set to exactly 0700, whatever the umask is. A directory that - already exists is left as it is, and the tree check judges it. + time, under a umask of 077, so it is born 0700, whatever the user's + umask is, with no `chmod` after it. A directory that already exists is + left as it is, and the tree check judges it. + + NOTHING IS MADE THROUGH A PATH THAT WAS NOT JUDGED FIRST (Copilot review + of openDox-code#69). The caller has refused an unsafe EXISTING prefix + before this runs (`BundledServer._prepare_directories`), so the deepest + directory that exists is safe to open. Each missing component is then + made RELATIVE TO ITS PARENT'S DESCRIPTOR and opened with `O_NOFOLLOW` + before anything is made beneath it. A name that another user put there + first, in a sticky directory such as `/tmp`, is refused, never followed + or written through: a symbolic link, something that is not a directory, + or a directory that is not this user's alone. + + THE UMASK IS PROCESS-WIDE, and it is narrowed only for these few + `mkdir`s and then put back. A file another thread creates meanwhile can + only come out more private than asked, never less. """ - missing = [] - for directory in (leaf, *leaf.parents): - if os.path.lexists(directory): - break - missing.append(directory) - for directory in reversed(missing): - try: - directory.mkdir(mode=0o700) - except FileExistsError: - continue # made by a concurrent start; judged below - os.chmod(directory, 0o700) + uid = os.getuid() + missing: list[str] = [] + base = leaf + while not os.path.lexists(base): + missing.append(base.name) + base = base.parent + if not missing: + return + descriptor = os.open(base, os.O_RDONLY | os.O_DIRECTORY) + path = base + previous = os.umask(0o077) + try: + for name in reversed(missing): + path = path / name + try: + os.mkdir(name, 0o700, dir_fd=descriptor) + except FileExistsError: + pass # made first, by a concurrent start or by someone else: judged next + try: + child = os.open(name, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW, + dir_fd=descriptor) + except OSError: + info = os.stat(name, dir_fd=descriptor, follow_symlinks=False) + reason = _unsafe_because(info, uid=uid, own=True) + if reason is None: + raise + raise BundledServer._unsafe(path, reason) from None + os.close(descriptor) + descriptor = child + reason = _unsafe_because(os.fstat(descriptor), uid=uid, own=True) + if reason is not None: + raise BundledServer._unsafe(path, reason) + finally: + os.umask(previous) + os.close(descriptor) def os_user() -> str: @@ -555,13 +594,19 @@ def _prepare_directories(self) -> None: under its own state directory, and it is narrowed to 0700 whatever it was. """ + # JUDGED BEFORE ANY WRITE, AND AGAIN AFTER (Copilot review of + # openDox-code#69). What exists already is checked first, so no + # directory is made through a link, or beneath a directory, that the + # tree check would refuse, and `_make_private_directories` refuses a + # name someone else put in its way. The whole tree is then checked + # BEFORE THE CHMOD, which follows a symbolic link: a `run` placed + # there as a link would otherwise have its TARGET re-moded. + self._refuse_an_unsafe_tree(existing_only=True) _make_private_directories(self.bundle.socket_dir) - # CHECKED BEFORE THE CHMOD, which follows a symbolic link: a `run` - # placed there as a link would otherwise have its TARGET re-moded. self._refuse_an_unsafe_tree() os.chmod(self.bundle.socket_dir, 0o700) - def _refuse_an_unsafe_tree(self) -> None: + def _refuse_an_unsafe_tree(self, *, existing_only: bool = False) -> None: """The socket's whole path is this user's to change, or it is refused. The socket's directory is how this install's clients find ITS @@ -587,9 +632,26 @@ def _refuse_an_unsafe_tree(self) -> None: `OPENDOX_STATE_DIR` never holds `..` (`config.state_dir` refuses it), so the configured path's components are the ones the kernel walks. + + With `existing_only`, the same rules are asked of only what exists + yet. `_prepare_directories` asks that BEFORE it creates anything, + so the links are checked first, a broken one included (Copilot + review of openDox-code#69). """ uid = os.getuid() configured = self.bundle.state_dir + + def present(path: Path) -> bool: + return not existing_only or os.path.lexists(path) + + for component in (configured, *configured.parents): + if not present(component): + continue + info = os.lstat(component) + if stat.S_ISLNK(info.st_mode) and info.st_uid not in (uid, 0): + raise self._unsafe( + component, f"is a symbolic link owned by uid {info.st_uid}, " + "neither this user nor root, who could point it elsewhere") state = configured.resolve() tree = [state, state / BUNDLE_TREE[0], state / BUNDLE_TREE[0] / BUNDLE_TREE[1]] # AND THE DATA DIRECTORY, where one exists already, a broken link @@ -604,16 +666,12 @@ def _refuse_an_unsafe_tree(self) -> None: (path, False) for path in dict.fromkeys( [*state.parents, *configured.parents])] for directory, mine in checks: + if not present(directory): + continue info = os.lstat(directory) if mine else os.stat(directory) reason = _unsafe_because(info, uid=uid, own=mine) if reason is not None: raise self._unsafe(directory, reason) - for component in (configured, *configured.parents): - info = os.lstat(component) - if stat.S_ISLNK(info.st_mode) and info.st_uid not in (uid, 0): - raise self._unsafe( - component, f"is a symbolic link owned by uid {info.st_uid}, " - "neither this user nor root, who could point it elsewhere") @staticmethod def _unsafe(directory: Path, reason: str) -> BundleRefused: diff --git a/tests_runtime/test_local_lifecycle.py b/tests_runtime/test_local_lifecycle.py index 58696b0b..e8688393 100644 --- a/tests_runtime/test_local_lifecycle.py +++ b/tests_runtime/test_local_lifecycle.py @@ -498,26 +498,33 @@ def test_this_users_own_link_to_a_private_directory_is_accepted( assert "initdb" in str(caught.value), caught.value -def test_a_link_another_user_owns_is_refused( - monkeypatch, tmp_path: Path, short_state: Path) -> None: - """Another user's link could be pointed elsewhere after the check. A - non-root suite cannot create one, so its `lstat` is stood in, for that - one path only.""" - private = short_state / "private" - private.mkdir(mode=0o700) - link = short_state / "link" - link.symlink_to(private) +def _foreign_lstat(monkeypatch, *links: Path) -> None: + """`os.lstat` answers that each of `links` belongs to another user. A + non-root suite cannot create another user's link, so it is stood in for + those paths only.""" real_lstat = os.lstat def _lstat(path, *args, **kwargs): info = real_lstat(path, *args, **kwargs) - if Path(path) == link: + if Path(path) in links: fields = list(info) fields[4] = os.getuid() + 4242 # st_uid return os.stat_result(fields) return info monkeypatch.setattr(bundle_mod.os, "lstat", _lstat) + + +def test_a_link_another_user_owns_is_refused( + monkeypatch, tmp_path: Path, short_state: Path) -> None: + """Another user's link could be pointed elsewhere after the check. A + non-root suite cannot create one, so its `lstat` is stood in, for that + one path only.""" + private = short_state / "private" + private.mkdir(mode=0o700) + link = short_state / "link" + link.symlink_to(private) + _foreign_lstat(monkeypatch, link) server = _prepared(monkeypatch, tmp_path, link / "state") with pytest.raises(bundle_mod.BundleRefused) as caught: server.start() @@ -539,6 +546,8 @@ def test_missing_directories_are_created_0700_whatever_the_umask( try: with pytest.raises(bundle_mod.BundleRefused) as caught: server.start() + # The narrowed umask is the start's alone: the user's is put back. + assert os.umask(umask) == umask, "the start left its own umask in place" finally: os.umask(previous) assert "initdb" in str(caught.value), caught.value @@ -547,6 +556,91 @@ def test_missing_directories_are_created_0700_whatever_the_umask( assert stat.S_IMODE(directory.stat().st_mode) == 0o700, directory +@pytest.mark.parametrize("shape", ["foreign-link", "foreign-broken-link", + "open-ancestor", "link-in-open-dir"]) +def test_nothing_is_made_through_a_path_the_tree_check_refuses( + monkeypatch, tmp_path: Path, short_state: Path, shape: str) -> None: + """What exists is judged BEFORE anything is created (Copilot review of + #69). Before, the missing `postgres/run` was made through the path first + and refused only after, so it was created in a foreign link's target, or + beneath a directory every user can write.""" + private = short_state / "private" + private.mkdir(mode=0o700) + open_dir = short_state / "open" + open_dir.mkdir() + open_dir.chmod(0o777) + if shape in {"foreign-link", "foreign-broken-link"}: + target = private if shape == "foreign-link" else private / "gone" + state = short_state / "link" + state.symlink_to(target) + _foreign_lstat(monkeypatch, state) + expected, made = "symbolic link owned by", target / "postgres" + elif shape == "open-ancestor": + state = open_dir / "state" + expected, made = "not sticky", state + else: + (open_dir / "link").symlink_to(private) + state = open_dir / "link" / "state" + expected, made = "not sticky", private / "state" + server = _prepared(monkeypatch, tmp_path, state) + with pytest.raises(bundle_mod.BundleRefused) as caught: + server.start() + assert expected in str(caught.value), caught.value + assert not os.path.lexists(made), f"{made} was made before the refusal" + + +@pytest.mark.parametrize("shape", ["link", "foreign-directory"]) +def test_a_name_put_in_the_way_first_is_refused_never_followed( + monkeypatch, tmp_path: Path, short_state: Path, shape: str) -> None: + """The race in a sticky directory, `/tmp`'s shape (Copilot review of + #69): every user can create a name there, so another user can put the + state directory's name in place between the check and the `mkdir`. The + stand-in `mkdir` plays that user, once. The name is refused, and nothing + is made through it or beneath it, nor is it re-moded.""" + sticky = short_state / "sticky" + sticky.mkdir() + sticky.chmod(0o1777) + state = sticky / "state" + target = tmp_path / "somewhere-else" + target.mkdir(mode=0o755) + target.chmod(0o755) + real_mkdir, real_fstat = os.mkdir, os.fstat + planted: dict = {} + + def _mkdir(path, mode=0o777, *, dir_fd=None): + if dir_fd is not None and path == state.name and not planted: + if shape == "link": + os.symlink(target, path, dir_fd=dir_fd) + else: + real_mkdir(path, 0o755, dir_fd=dir_fd) + os.chmod(state, 0o755) + planted["inode"] = os.lstat(state).st_ino + raise FileExistsError(path) + return real_mkdir(path, mode, dir_fd=dir_fd) + + def _fstat(descriptor): + info = real_fstat(descriptor) + if info.st_ino == planted.get("inode"): + fields = list(info) + fields[4] = os.getuid() + 4242 # st_uid + return os.stat_result(fields) + return info + + monkeypatch.setattr(bundle_mod.os, "mkdir", _mkdir) + monkeypatch.setattr(bundle_mod.os, "fstat", _fstat) + server = _prepared(monkeypatch, tmp_path, state) + with pytest.raises(bundle_mod.BundleRefused) as caught: + server.start() + assert planted, "the stand-in never ran: the state directory was not made by descriptor" + message = str(caught.value) + assert str(state) in message, message + assert ("is a symbolic link" if shape == "link" + else "is owned by uid") in message, message + beneath = target if shape == "link" else state + assert not (beneath / "postgres").exists(), "made beneath the planted name" + assert stat.S_IMODE(os.stat(beneath).st_mode) == 0o755, "the planted name was re-moded" + + @pytest.mark.parametrize("shape", ["link", "broken-link", "open"]) def test_a_data_directory_that_is_not_this_installs_own_is_refused( monkeypatch, tmp_path: Path, short_state: Path, shape: str) -> None: From 20032d023f2bad7f9a50d8bdc48e1a30ddbe0dd5 Mon Sep 17 00:00:00 2001 From: Brett Heap <1513478+brettheap@users.noreply.github.com> Date: Fri, 2 Oct 2026 21:01:58 +0000 Subject: [PATCH 31/88] T073: the hosted case hands its child a hosted install's settings on purpose #67's fix round (cdf7382b, merged in 47a6574d) strips every runtime setting the runner exports from a standalone child, so the hosted generate-and-open case's monkeypatched settings no longer reached it: the child refused for want of an issuer. Child now takes `extra_env`, the settings a case gives its child on purpose, applied after the strip, so nothing is inherited by accident. The hosted case passes a hosted install's settings through it. Whole suite locally: 3034 passed, 177 skipped. Arc: neutral-product-standalone-operability Lane: openxfactory-4 (openXfactory-4-openDox_extraction) Co-Authored-By: Claude Opus 5.5 (1M context) --- tests/standalone_child.py | 11 +++++++++-- tests/test_served_install_block.py | 13 ++++++------- 2 files changed, 15 insertions(+), 9 deletions(-) diff --git a/tests/standalone_child.py b/tests/standalone_child.py index e9e23534..2956ae25 100644 --- a/tests/standalone_child.py +++ b/tests/standalone_child.py @@ -147,9 +147,15 @@ def fresh_repository(fixture: Path, parent: Path, *, class Child: """One `python -m ...` child, with the siblings refused and its - standard output a buffered pipe. `workdir` holds the blocker and the log.""" + standard output a buffered pipe. `workdir` holds the blocker and the log. - def __init__(self, workdir: Path, module: str, *args: str) -> None: + `extra_env` names the settings a case gives its child ON PURPOSE (a hosted + install's issuer and DSNs, say). They are applied after the runner's own + runtime settings are taken out, so a case still inherits none by + accident (plan 034 T073).""" + + def __init__(self, workdir: Path, module: str, *args: str, + extra_env: dict[str, str] | None = None) -> None: blocker = workdir / "sibling-blocker" blocker.mkdir(parents=True, exist_ok=True) (blocker / "sitecustomize.py").write_text(_BLOCKER, encoding="utf-8") @@ -165,6 +171,7 @@ def __init__(self, workdir: Path, module: str, *args: str) -> None: self.state_dir = Path(tempfile.mkdtemp( prefix="odx-child-", dir="/tmp" if os.path.isdir("/tmp") else None)) env["OPENDOX_STATE_DIR"] = str(self.state_dir) + env.update(extra_env or {}) self.argv = [sys.executable, "-m", module, *args] verb = args[0] if args and not args[0].startswith("-") else "" self.label = f"python -m {module} {verb}".strip() diff --git a/tests/test_served_install_block.py b/tests/test_served_install_block.py index cfaec76d..ae56c757 100644 --- a/tests/test_served_install_block.py +++ b/tests/test_served_install_block.py @@ -68,16 +68,15 @@ def _get(base: tuple[str, int], path: str) -> tuple[int, dict]: # 1 — a hosted entry point reports a hosted install, and no bundled server # --------------------------------------------------------------------------- -def test_a_hosted_entry_point_reports_its_hosted_shape(tmp_path, monkeypatch) -> None: - for name in runtime_config.SETTING_NAMES: - monkeypatch.delenv(name, raising=False) - for name, value in HOSTED.items(): - monkeypatch.setenv(name, value) - monkeypatch.setenv(PREFIX + "INSTALL_MODE", runtime_config.INSTALL_MODE_HOSTED) +def test_a_hosted_entry_point_reports_its_hosted_shape(tmp_path) -> None: + """The child inherits no runtime setting from the runner + (`standalone_child.Child`), and is given a hosted install's, on purpose.""" repo = fresh_repository(PLAIN, tmp_path) child = Child(tmp_path, "opendox.cli", "generate-and-open", "--repo-root", str(repo), "--repository", "fixture", - "--no-open", "--port", "0", "--run-dir", str(tmp_path / "run")) + "--no-open", "--port", "0", "--run-dir", str(tmp_path / "run"), + extra_env={**HOSTED, PREFIX + "INSTALL_MODE": + runtime_config.INSTALL_MODE_HOSTED}) try: match = child.wait_for_line(_URL) base = (match.group(2), int(match.group(3))) From 105f2f1222fbdd19d245110d443c3fc6475ebeb9 Mon Sep 17 00:00:00 2001 From: Brett Heap <1513478+brettheap@users.noreply.github.com> Date: Fri, 2 Oct 2026 22:22:13 +0000 Subject: [PATCH 32/88] Fix round: no runtime setting the runner exports reaches a tests_runtime case (Copilot review) Since T070 the runtime reads a selector, OPENDOX_INSTALL_MODE. A runner that exported OPENDOX_INSTALL_MODE=local, as a local install's own shell would, turned every hosted case that leaves it unset into a configuration refusal. Measured: 20 cases failed, 18 in test_runtime_cli.py and 2 in test_migrations_apply.py (status then has no "settings" key, so the case raises KeyError). The scrubbed fixture covered only test_install_mode.py. tests_runtime/conftest.py gains an autouse fixture that clears every name in opendox.runtime.config.SETTING_NAMES before each case, and a case that wants one sets it. OPENDOX_TEST_DATABASE_URL is not a runtime setting, so the harness's own DSN is kept. The production refusal is unchanged. Measured full-suite results: - with OPENDOX_INSTALL_MODE=local and a broker issuer and audience exported: 3126 selected, 3115 passed, 11 skipped, 0 failed; - in a clean environment: the same. A mutant that clears nothing is killed: the 20 cases fail. Lane: openxfactory-4 (openXfactory-4-openDox_extraction) Co-Authored-By: Claude Opus 5.5 (1M context) --- tests_runtime/conftest.py | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/tests_runtime/conftest.py b/tests_runtime/conftest.py index 2c6e0b7b..55553539 100644 --- a/tests_runtime/conftest.py +++ b/tests_runtime/conftest.py @@ -57,6 +57,31 @@ ) +@pytest.fixture(autouse=True) +def _no_inherited_runtime_setting(monkeypatch: pytest.MonkeyPatch) -> None: + """No `OPENDOX_*` runtime setting the runner exports reaches a case. + + Since plan 034 T070 the runtime reads a selector, `OPENDOX_INSTALL_MODE`, + and a runner that exports `local` (as a local install's own shell would) + turned every hosted case that leaves it unset into a configuration + refusal: 20 cases in `test_runtime_cli.py` and `test_migrations_apply.py` + (measured; Copilot review of openDox-code#67). So every name the runtime + reads (`opendox.runtime.config.SETTING_NAMES`) is cleared before each + case, and a case that wants one sets it. `OPENDOX_TEST_DATABASE_URL` is + not one of them (see `TEST_DSN_ENV`), so the harness's own DSN is kept. + The production refusal is unchanged. + + An `opendox` that cannot be imported leaves nothing to clear: the case + then fails on its own import, which is the failure worth seeing. + """ + try: + from opendox.runtime.config import SETTING_NAMES + except ImportError: + return + for name in SETTING_NAMES: + monkeypatch.delenv(name, raising=False) + + def in_ci() -> bool: """Whether this run is CI's, by CI's own variable. From 3426c753eff5701987b808ef2296ba74161fa7c2 Mon Sep 17 00:00:00 2001 From: Brett Heap <1513478+brettheap@users.noreply.github.com> Date: Fri, 2 Oct 2026 22:29:58 +0000 Subject: [PATCH 33/88] Fix round 12: the auth files are exactly 0600, and the cluster runs on its own files (Copilot review) Two threads from Copilot's reviews at 6eb0bbdb and fedfa75d. write_authentication's 0600 was only a creation request. The umask filters it, and it changes nothing about a file that already exists. So a 0277 umask left a 0400 file, a temporary file an interrupted start left at the same name kept its 0644, and a link left there was written through. Now: - the stale temporary is unlinked first (a link itself, never its target); - the new one is opened O_CREAT|O_EXCL|O_NOFOLLOW; - its descriptor is fchmod-ed to 0600 before anything is written. A link raced in after the unlink is a FileExistsError, never followed. An existing cluster's postgresql.conf could point data_directory, hba_file and ident_file elsewhere: at an outside trust file the rewritten files would never replace, or at a cluster outside the state directory. The launch now pins all three on the command line, which outranks every configuration file. New cases: - tests_runtime/test_local_lifecycle.py: the umask, a stale 0644 temporary, a stale link, and a raced link; - tests_runtime/test_bundled_postgres.py: a real cluster whose postgresql.conf points all three elsewhere still shows its own three and serves peer. The first four fail against fedfa75d's bundle.py, and so does the real-cluster case. Six mutants are all killed (one only after the raced-link case was added). Lane: openxfactory-4 (openXfactory-4-openDox_extraction) Co-Authored-By: Claude Opus 5.5 (1M context) --- src/opendox/runtime/bundle.py | 42 ++++++++++++++--- tests_runtime/test_bundled_postgres.py | 33 +++++++++++++ tests_runtime/test_local_lifecycle.py | 65 ++++++++++++++++++++++++++ 3 files changed, 134 insertions(+), 6 deletions(-) diff --git a/src/opendox/runtime/bundle.py b/src/opendox/runtime/bundle.py index bd7834d4..2da965d3 100644 --- a/src/opendox/runtime/bundle.py +++ b/src/opendox/runtime/bundle.py @@ -470,15 +470,34 @@ def write_authentication(data_dir: Path, user: str) -> None: Before EVERY launch, not only after `initdb`: a data directory an earlier build initialized, or a file edited by hand, is brought back to the one configuration this install runs with. + + THE 0600 IS SET, NOT ASKED FOR (Copilot review of openDox-code#69). An + `open` mode is only a creation request: the umask filters it, and it + changes nothing about a file that exists already. So a temporary file an + interrupted start left at the same name (a 0644 one, or a symbolic link) + is removed first. The new one is created EXCLUSIVELY and without following + a link, and its descriptor is set to exactly 0600 before anything is + written into it. """ for name, content in authentication_files(user).items(): target = data_dir / name temporary = data_dir / f".{name}.opendox-{os.getpid()}" - descriptor = os.open(temporary, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, - 0o600) - with os.fdopen(descriptor, "w", encoding="utf-8") as handle: - handle.write(content) - os.replace(temporary, target) + with contextlib.suppress(FileNotFoundError): + os.unlink(temporary) # an interrupted start's; a link itself, never its target + descriptor = os.open( + temporary, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, 0o600) + try: + os.fchmod(descriptor, 0o600) + with os.fdopen(descriptor, "w", encoding="utf-8") as handle: + descriptor = -1 # the handle closes it now + handle.write(content) + os.replace(temporary, target) + except BaseException: + if descriptor >= 0: + os.close(descriptor) + with contextlib.suppress(FileNotFoundError): + os.unlink(temporary) + raise def _unsafe_because(info: os.stat_result, *, uid: int, own: bool) -> str | None: @@ -765,7 +784,18 @@ def _launch(self, binaries: Path) -> None: # port can stand in for this server, and nothing off this # machine can reach it. "-c", "listen_addresses=", - "-c", "unix_socket_permissions=0700"], + "-c", "unix_socket_permissions=0700", + # THE CLUSTER'S OWN FILES, PINNED (Copilot review of + # openDox-code#69). An existing cluster's + # `postgresql.conf` can point `data_directory`, `hba_file` + # and `ident_file` elsewhere: at an outside `trust` file + # that the files just rewritten would never replace, or at + # a cluster outside the state directory. The command line + # outranks every configuration file, so these three are + # the data directory and the two files written above. + "-c", f"data_directory={self.bundle.data_dir}", + "-c", f"hba_file={self.bundle.data_dir / 'pg_hba.conf'}", + "-c", f"ident_file={self.bundle.data_dir / 'pg_ident.conf'}"], stdin=subprocess.DEVNULL, stdout=log, stderr=subprocess.STDOUT, env=_child_environment(), # ITS OWN SESSION, so a terminal's Ctrl-C reaches this process diff --git a/tests_runtime/test_bundled_postgres.py b/tests_runtime/test_bundled_postgres.py index bae2a7f5..e650275f 100644 --- a/tests_runtime/test_bundled_postgres.py +++ b/tests_runtime/test_bundled_postgres.py @@ -561,6 +561,39 @@ def test_an_older_trust_cluster_is_brought_back_to_peer_on_start( assert method == f"peer:{bundle_mod.os_user()}", method +def test_a_cluster_whose_configuration_points_elsewhere_runs_on_its_own_files( + state_dir: Path, tmp_path: Path) -> None: + """An existing cluster's `postgresql.conf` can point `hba_file` and + `ident_file` at outside files, a `trust` one say, and `data_directory` + at a cluster that is not this install's (Copilot review of #69). The + launch pins all three on the command line, which outranks the file. So + the server reads the two files this install wrote, from its own data + directory, and every connection is still peer.""" + settings = config.load_settings({MODE: "local", STATE: str(state_dir)}) + bundle_mod.BundledServer(settings).start().stop() + data = config.DatabaseBundle(state_dir).data_dir + outside = tmp_path / "outside" + outside.mkdir() + (outside / "trust_hba.conf").write_text("local all all trust\n", encoding="utf-8") + (outside / "ident.conf").write_text("", encoding="utf-8") + with (data / "postgresql.conf").open("a", encoding="utf-8") as conf: + conf.write(f"\nhba_file = '{outside / 'trust_hba.conf'}'\n" + f"ident_file = '{outside / 'ident.conf'}'\n" + f"data_directory = '{outside / 'no-such-cluster'}'\n") + with bundle_mod.BundledServer(settings) as server: + import psycopg + + with _owner(server) as conn: + shown = {name: conn.execute(f"show {name}").fetchone()[0] + for name in ("data_directory", "hba_file", "ident_file")} + with psycopg.connect(server.bundle.served_dsn) as conn: + method = conn.execute("select system_user").fetchone()[0] + assert shown == {"data_directory": str(data), + "hba_file": str(data / "pg_hba.conf"), + "ident_file": str(data / "pg_ident.conf")}, shown + assert method == f"peer:{bundle_mod.os_user()}", method + + def test_migrate_under_the_local_mode_uses_the_bundle_and_refuses_a_dsn( state_dir: Path) -> None: """`runtime migrate` is part of the same install: it reads the bundle's diff --git a/tests_runtime/test_local_lifecycle.py b/tests_runtime/test_local_lifecycle.py index e8688393..7bc74535 100644 --- a/tests_runtime/test_local_lifecycle.py +++ b/tests_runtime/test_local_lifecycle.py @@ -759,6 +759,71 @@ def test_the_files_are_written_0600_and_replace_what_was_there( assert sorted(p.name for p in tmp_path.iterdir()) == ["pg_hba.conf", "pg_ident.conf"] +@pytest.mark.parametrize("shape", ["umask", "stale-0644", "stale-link"]) +def test_the_files_are_exactly_0600_whatever_the_umask_or_a_stale_temporary( + tmp_path: Path, shape: str) -> None: + """The 0600 is SET, not asked for (Copilot review of #69). A umask that + takes the owner's write bit would leave a 0400 file. A temporary file an + interrupted start left at the same name would keep its own mode, 0644, + and a link left there would be written through to its target.""" + data = tmp_path / "data" + data.mkdir(mode=0o700) + outside = tmp_path / "outside.conf" + outside.write_text("untouched\n", encoding="utf-8") + outside.chmod(0o644) + for name in bundle_mod.authentication_files("alice"): + stale = data / f".{name}.opendox-{os.getpid()}" + if shape == "stale-0644": + stale.write_text("local all all trust\n", encoding="utf-8") + stale.chmod(0o644) + elif shape == "stale-link": + stale.symlink_to(outside) + previous = os.umask(0o277 if shape == "umask" else 0o022) + try: + bundle_mod.write_authentication(data, "alice") + finally: + os.umask(previous) + for name, content in bundle_mod.authentication_files("alice").items(): + written = data / name + assert not written.is_symlink(), name + assert written.read_text(encoding="utf-8") == content, name + assert stat.S_IMODE(written.stat().st_mode) == 0o600, name + assert sorted(p.name for p in data.iterdir()) == ["pg_hba.conf", "pg_ident.conf"] + assert outside.read_text(encoding="utf-8") == "untouched\n" + assert stat.S_IMODE(outside.stat().st_mode) == 0o644, "a stale link's target was re-moded" + + +def test_a_link_put_at_the_temporary_name_after_its_removal_is_never_followed( + monkeypatch, tmp_path: Path) -> None: + """The temporary file is created EXCLUSIVELY and without following a + link, so a link that appears at its name between the removal and the + open is a refusal, never a file written through (Copilot review of + #69). The stand-in `unlink` plays that race once.""" + data = tmp_path / "data" + data.mkdir(mode=0o700) + outside = tmp_path / "outside.conf" + outside.write_text("untouched\n", encoding="utf-8") + outside.chmod(0o644) + real_unlink = os.unlink + raced: list = [] + + def _unlink(path, *args, **kwargs): + try: + real_unlink(path, *args, **kwargs) + finally: + if not raced and Path(path).name.startswith(".pg_hba.conf.opendox-"): + raced.append(path) + os.symlink(outside, path) + + monkeypatch.setattr(bundle_mod.os, "unlink", _unlink) + with pytest.raises(FileExistsError): + bundle_mod.write_authentication(data, "alice") + assert raced, "the stand-in never ran" + assert outside.read_text(encoding="utf-8") == "untouched\n" + assert stat.S_IMODE(outside.stat().st_mode) == 0o644, "the link's target was re-moded" + assert not (data / "pg_hba.conf").exists() + + @pytest.mark.parametrize("name", ["/regex", 'quo"te', "has space", "hash#tag", ""]) def test_an_os_user_name_the_map_cannot_hold_plainly_is_refused( monkeypatch: pytest.MonkeyPatch, name: str) -> None: From 5d4e433b42ebbde88a2b72215c84579bf5ab4e99 Mon Sep 17 00:00:00 2001 From: Brett Heap <1513478+brettheap@users.noreply.github.com> Date: Fri, 2 Oct 2026 22:37:25 +0000 Subject: [PATCH 34/88] T084: route the last deferred reaches through the column seams (4.3, batch L) Every deferred reach left in openDox's own verbs now reads the registration current at an `opendox.column_seams` seam, a host's or openDox's own default (R1Q10 (a)). F4.1's scan prints "no deferred reach names the consumer or the publisher". - serve_workbench: the chat turn's and the document abstract's scope authority (`column_seams.scope`). The abstract reads it below its step-1 check (RULED 5920216845 item 1), so a plane that refuses never reaches it. The scope value types are openDox's own (`doxbench_scope_types`). - serve_workbench, model intake and approval (RULED by Brett Heap, 2026-10-02, "Refuse by name, hide intake (Recommended)"). With no host's gate registered, `GET /workbench/model-intake` answers `offered: false` with `column_seams.GATE_RECORDS_REFUSAL`, even beside a hand-written broker block. The intake act and the approval refuse with that sentence before a broker is spawned, a declaration is written or a record is built, and each drains the body it was sent. The order is no gate-record writer first, then no broker. - serve_project: the project register's records prefix and kickoff readers (`column_seams.gate`, `column_seams.kickoff`). openDox's kickoff default discovers no register, so a lone openDox answers today's 404 "no project register". - branch_session and cli: `gate_console` and `gate_mod` are proxies over `column_seams.gate`, replacing `consumer_reach`'s late stand-in. The pick fallbacks read `column_seams.register`, and the proposal state reads `column_seams.kickoff`. - default_columns: the editable set is ONE named function, `editable_paths`. `GATE_RECORDS_REFUSAL` moves to `column_seams`, beside the predicate. Tests: - tests/test_capability_honesty.py, sections 5 and 6: - the three sites that dropped a connection, on a standalone child; - a chat turn with a binding configured (`model-binding add`); - composed hosts whose turn and abstract reach the scope step; - intake not offered standalone, with a hand-written broker block; - a host that registers its gate is offered intake and approves. Before (a39e0201): five cases fail. Four are RemoteDisconnected, and the standalone intake reads offered true. - tests/test_projection_seams.py's import-time rule now holds the column seams' proxies. tests/test_consumer_reach.py's CONVERTED_SITES guard is retired into it, since no name binds a stand-in. The census gains column_seams and default_columns. Lane: openxfactory-4 (openXfactory-4-openDox_extraction) Co-Authored-By: Claude Opus 5.5 (1M context) --- src/opendox/branch_session.py | 29 +- src/opendox/cli.py | 11 +- src/opendox/column_seams.py | 14 + src/opendox/default_columns.py | 37 +-- src/opendox/serve_project.py | 29 +- src/opendox/serve_workbench.py | 141 +++++++--- tests/test_capability_honesty.py | 435 +++++++++++++++++++++++++++++ tests/test_consumer_reach.py | 220 ++------------- tests/test_profile_registration.py | 4 +- tests/test_projection_seams.py | 16 +- 10 files changed, 654 insertions(+), 282 deletions(-) diff --git a/src/opendox/branch_session.py b/src/opendox/branch_session.py index 5604cc7a..f59efd3a 100644 --- a/src/opendox/branch_session.py +++ b/src/opendox/branch_session.py @@ -74,13 +74,17 @@ import yaml from . import doxbench_hash -# THE GATE COLUMN, NAMED LATE (BUILD slice 2b, `split-opendox-two-layer-product` -# § 3.5/3.6). `gate_console` is openXdox's — the layer that PINS this one — so an -# import statement here made `import opendox.branch_session` require openXdox to -# be installed, which `design.md`:243 refuses: *"what must not survive is the -# direction, not the calls."* The stand-in resolves on first attribute access and -# refuses naming the layering; every `gate_console.X` below is unchanged. -from .consumer_reach import gate_console +# THE GATE COLUMN, THROUGH ITS SEAM (plan 034 T084; #1144 4.3, R1Q10 (a)). +# `gate_console` is openXdox's, the layer that PINS this one, so an import +# statement here would make `import opendox.branch_session` require openXdox, +# which `design.md`:243 refuses. BUILD slice 2b named it late through +# `consumer_reach`'s stand-in, which still raised where openXdox was absent. +# It is now a proxy over `column_seams.gate`: each `gate_console.X` below reads +# the registration current when it runs, a host's or openDox's own default +# (`default_columns.GATE`, whose governed record functions refuse by name). +# Every `gate_console.X` below is unchanged. Stdlib-only, so no edge. +from . import column_seams +gate_console = column_seams.gate.proxy # ...EXCEPT the nine `records_dir` DEFAULTS at :1740, :1859, :1878, :4179, :4217, # :4254, :4287, :4504 and :4991, which no stand-in can defer: a default argument @@ -1588,8 +1592,13 @@ def _active_pick_fallbacks( A change picked from two staging ids is ambiguous and is refused instead of selecting whichever register row happened to be encountered first. + + The register is the one registered at `column_seams.register` (plan 034 + T084): a host's, or openDox's own default, whose register holds no row, + so a lone openDox proves no fallback. """ - from openxdox.register import CrossReferenceIndexAdapter + CrossReferenceIndexAdapter = ( + column_seams.register.current().CrossReferenceIndexAdapter) rows = rows if rows is not None else _change_rows(checkout_root) active_without_origin = { @@ -2007,7 +2016,9 @@ def proposal_state_for(tile: "Tile", *, records_root: Path | str | None = None, Only a STAGED-TOPIC tile can carry a proposal: both signals are keyed on a staging id, and a cluster or possible tile has none.""" - from openxdox import kickoff as kickoff_mod # lazy: mirrors gate_console's cycle note + # Kickoff through its seam (plan 034 T084): a host's commission reader, or + # openDox's own default, which reads no dispatched commission. + kickoff_mod = column_seams.kickoff.current() staged = tile.scope_kind == STAGED_TOPIC landed = None diff --git a/src/opendox/cli.py b/src/opendox/cli.py index 619a3bbc..610720c8 100644 --- a/src/opendox/cli.py +++ b/src/opendox/cli.py @@ -72,8 +72,14 @@ from opendox import branch_session as branch_session_mod # noqa: E402 from opendox import doxbench_install as install_mod # noqa: E402 from opendox import doxbench_knowledge as knowledge_mod # noqa: E402 -from opendox import consumer_reach # noqa: E402 -gate_mod = consumer_reach.gate_console # noqa: E402 +# THE GATE PRIMITIVES, THROUGH THEIR SEAM (plan 034 T084; #1144 4.3, R1Q10 +# (a)). This was `consumer_reach.gate_console`, a late stand-in over openXdox's +# `gate_console` that still raised where openXdox was absent. `gate_mod.X` now +# reads the registration current at `column_seams.gate` when it runs, a host's +# or openDox's own default, which `build_parser()` and `main()` register. +# Stdlib-only, so this adds no reach. +from opendox import column_seams # noqa: E402 +gate_mod = column_seams.gate.proxy # noqa: E402 from opendox import serve as serve_mod # noqa: E402 from opendox import workbench as workbench_mod # noqa: E402 # THE HOME-CORPUS SEAM'S DEFAULT (4.1a; plan 034 T022) -- see @@ -110,7 +116,6 @@ # register the defaults where no host has. `is_rfc3339_datetime` is not a seam: # it is the neutral contract's own date-time rule, and openDox owns it. from opendox import projection_seams # noqa: E402 -from opendox import column_seams # noqa: E402 from opendox.rfc3339 import is_rfc3339_datetime # noqa: E402 # THE COMPOSITION POINT, BOUND AT LAST (§ 4.3; RULED ASK-2 option (2), diff --git a/src/opendox/column_seams.py b/src/opendox/column_seams.py index 94381856..ad553c1a 100644 --- a/src/opendox/column_seams.py +++ b/src/opendox/column_seams.py @@ -70,6 +70,7 @@ __all__ = [ "GATE_CALLABLES", + "GATE_RECORDS_REFUSAL", "GATE_VALUES", "KICKOFF_CALLABLES", "ProjectionSeamError", @@ -87,6 +88,19 @@ _MODULE = "opendox.column_seams" +#: The fixed sentence a surface puts on the wire, and the refusal's text, where +#: a governed gate-action record is asked for and no host's gate is registered. +#: It names the seam and the call that registers one (4.2). +GATE_RECORDS_REFUSAL = ( + "this install records no governed gate actions: no host's gate is " + "registered at openDox's gate seam (opendox.column_seams.gate), and " + "openDox's own default writes no gate-action record, because that record " + "is a shape only the governing host's pinned schema declares. A host that " + "records gate actions registers its gate at process start with " + "opendox.column_seams.gate.register(). A model is " + "declared on this install with `opendox model-binding add`, which needs " + "no approval record.") + #: What a gate registration must carry as callables: the human-gate guard and #: the types it works with, the record functions, the session-ref target id, #: the first-edit gate builder chat's Save writes through, and the clock, the diff --git a/src/opendox/default_columns.py b/src/opendox/default_columns.py index 4182f62e..9e9d6c84 100644 --- a/src/opendox/default_columns.py +++ b/src/opendox/default_columns.py @@ -61,6 +61,7 @@ from typing import Any, Iterable, Mapping, Sequence from opendox import defaults +from opendox.column_seams import GATE_RECORDS_REFUSAL from opendox.boundary import GATE_SIDE_EFFECT, BoundaryViolation, HumanGate, Refusal from opendox.doxbench_scope_types import ( ScopeConfinementError, @@ -107,20 +108,6 @@ class GateRefused(Exception): governed functions it does not carry.""" -#: The fixed sentence a surface puts on the wire, and the refusal's text, where -#: a governed gate-action record is asked for and no host's gate is registered. -#: It names the seam and the call that registers one (4.2). -GATE_RECORDS_REFUSAL = ( - "this install records no governed gate actions: no host's gate is " - "registered at openDox's gate seam (opendox.column_seams.gate), and " - "openDox's own default writes no gate-action record, because that record " - "is a shape only the governing host's pinned schema declares. A host that " - "records gate actions registers its gate at process start with " - "opendox.column_seams.gate.register(). A model is " - "declared on this install with `opendox model-binding add`, which needs " - "no approval record.") - - class GateRecordsNotRegistered(GateRefused): """A governed gate function was asked for and no host's gate is registered: openDox's default carries none of them (see the module @@ -314,6 +301,19 @@ def _section(key: str, label: str, note: str, paths: Iterable[Any], *, owned=False, documents=tuple(rows)) +def editable_paths(sections: Sequence[ScopeSection], + context_paths: Sequence[str]) -> tuple[str, ...]: + """The paths a projected tile lets a turn edit: NONE. openDox's default + scope is READ-ONLY (the holder's reading of R1Q10 (a) for T084, put to + Brett on openxFactory#656's thread, 2026-10-02), so a tile grants no edit + authority, and openDox's own turn guard, which requires a turn's paths to + be in scope AND editable (`doxbench_turns._require_in_scope_and_editable`), + discloses none of them. It is ONE named function so that a ruling either way + is one change here: the readable paths a tile's own sections carry are + `context_paths`.""" + return () + + def resolve_scope(snapshot: Mapping[str, Any], key: ScopeKey, *, source_root: Path, created_paths: Iterable[str] = () ) -> ScopeProjection | None: @@ -323,8 +323,8 @@ def resolve_scope(snapshot: Mapping[str, Any], key: ScopeKey, *, A group (`cluster`) projects its members, a selection (`staged`) its files, and a candidate (`possible`) the members of the groups that claim it. Each path is confined to `source_root` by the registry seam's own - `resolve_within`, and NOTHING is editable: `editable_paths` and - `active_document_candidates` are empty, and there is no outline. A + `resolve_within`, and NOTHING is editable (`editable_paths`), so + `active_document_candidates` is empty too, and there is no outline. A `created_paths` entry is confined and readable, never editable.""" if not isinstance(snapshot, Mapping): return None @@ -389,10 +389,13 @@ def members(group: Mapping[str, Any]) -> list[Any]: if path not in context: context.append(path) revision = _text(_mapping(snapshot.get("generation")).get("source_revision")) + editable = editable_paths(sections, context) return ScopeProjection( key=key, title=title, keywords=keywords, source_revision=revision, sections=tuple(sections), context_paths=tuple(context), - editable_paths=(), outline_path=None, active_document_candidates=()) + editable_paths=editable, outline_path=None, + # what the turn guard would accept: in scope AND editable + active_document_candidates=tuple(p for p in context if p in editable)) def _normalize_ref(ref: Any) -> str: diff --git a/src/opendox/serve_project.py b/src/opendox/serve_project.py index 82ccf400..ebb6f999 100644 --- a/src/opendox/serve_project.py +++ b/src/opendox/serve_project.py @@ -46,6 +46,10 @@ # proxy is bound at module level here: `tests/test_projection_seams.py` holds # the set of modules that bind one, and this module reads the seam in a body. from opendox import projection_seams +# THE GATE'S RECORDS PREFIX AND KICKOFF'S READERS, THROUGH THEIR SEAMS (plan +# 034 T084; #1144 4.3, R1Q10 (a)): `_serve_project_register` reads both per +# request, a host's registration or openDox's own default. Stdlib-only. +from opendox import column_seams from opendox.serve_wire import ( AGENT_INVOCATION_REFUSAL, JSON_CTYPE, @@ -266,13 +270,19 @@ def _serve_project_register(self, head_only: bool) -> None: duplicate guard uses, so a fresh commission is visible as pending instead of looking like it did nothing. A pending id the register already carries is dropped: the register wins the moment the - fulfilment lands, even before the descriptor's status flips.""" + fulfilment lands, even before the descriptor's status flips. + + THROUGH THE COLUMN SEAMS (plan 034 T084; #1144 batch L, RULED + `5920216845`). These were deferred `openxdox.gate_console` and + `openxdox.kickoff` imports, so where openXdox is not installed every + request ended in a dropped connection. openDox's own kickoff default + discovers no register, so a lone openDox answers today's structured + 404 "no project register" and the picker hides, as it does in any + checkout without one.""" import yaml - from openxdox.gate_console import DEFAULT_RECORDS_DIR - from openxdox.kickoff import ( - dispatched_commission_rows, dispatched_commissions, - discover_project_register) - source = discover_project_register(Path(self.checkout_root)) + gate = column_seams.gate.current() + kickoff = column_seams.kickoff.current() + source = kickoff.discover_project_register(Path(self.checkout_root)) register = None if source is not None: try: @@ -289,7 +299,7 @@ def _serve_project_register(self, head_only: bool) -> None: if isinstance(p, dict) and p.get("id") ] real_ids = {p["id"] for p in projects} - records_root = Path(self.checkout_root) / DEFAULT_RECORDS_DIR + records_root = Path(self.checkout_root) / gate.DEFAULT_RECORDS_DIR def _job(descriptor): try: @@ -300,7 +310,8 @@ def _job(descriptor): pending = [] for pid, descriptor in sorted( - dispatched_commissions(records_root, "create-project").items()): + kickoff.dispatched_commissions(records_root, + "create-project").items()): if pid in real_ids: continue job = _job(descriptor) @@ -319,7 +330,7 @@ def _job(descriptor): # neither plane is dropped (nothing to badge). pending_ids = {p["id"] for p in pending} pending_edits = [] - for pid, _descriptor, job in dispatched_commission_rows( + for pid, _descriptor, job in kickoff.dispatched_commission_rows( records_root, "edit-project"): if pid not in real_ids and pid not in pending_ids: continue diff --git a/src/opendox/serve_workbench.py b/src/opendox/serve_workbench.py index 037ed2fb..e3b7c8ab 100644 --- a/src/opendox/serve_workbench.py +++ b/src/opendox/serve_workbench.py @@ -51,6 +51,14 @@ # moment, openDox's own where no host has contributed one, so the workbench # routes confine by the same rule `/source` does, in a lone openDox too. from opendox import projection_seams +# THE CONSUMER COLUMNS' SEAMS (plan 034 T084; #1144 4.3, R1Q10 (a)): the gate +# primitives, the doxBench scope authority, kickoff and the register. Each +# reach below that named `openxdox.gate_console`, `gate_routes` or +# `doxbench_scope` inside a function now reads the registration current at the +# moment it runs, a host's or openDox's own default. Stdlib-only, so the +# import adds no edge. The scope VALUE types are openDox's own. +from opendox import column_seams +from opendox.doxbench_scope_types import ScopeConfinementError, ScopeKey registry_mod = projection_seams.registry.proxy from opendox.serve_wire import ( DOXBENCH_ABSTRACT_REFUSED_PROSE_BYTES, @@ -343,13 +351,15 @@ def _session_worktree_for(self, key): def _is_live_session_ref(self, key, entry) -> bool: """Whether `key.ref` is one of this tile's LIVE session branches. - ONE spelling, in `doxbench_scope` beside the other consumer of the same - question (re-verify N-6). This method had grown as a second copy and had + ONE spelling, in the registered scope authority (`column_seams.scope`: + openXdox's `doxbench_scope`, or openDox's own default) beside the other + consumer of the same question (re-verify N-6). This method had grown as a second copy and had already diverged from it — different ref comparison, different exception breadth — which is precisely how the two would have drifted apart on the next change to what counts as a live session.""" - from openxdox import doxbench_scope - return doxbench_scope.is_live_session_ref( + # THROUGH THE SCOPE SEAM (T084): the registered authority's one + # spelling of the question, a host's or openDox's default. + return column_seams.scope.current().is_live_session_ref( self.source.registry, key, repository=entry.repository or key.repository, ref=key.ref) @@ -408,10 +418,9 @@ def _thread_gate(self, worktree): it — the one gate on this surface whose allowlist carries the thread prefix (`gate_routes.first_edit_gate_factory`, task 9.5). Built through that factory rather than beside it, so the widening has one spelling.""" - from openxdox import gate_console - from openxdox import gate_routes - return gate_routes.first_edit_gate_factory( - self.actor, gate_console.DEFAULT_RECORDS_DIR)(worktree) + gate = column_seams.gate.current() # THROUGH THE GATE SEAM (T084) + return gate.first_edit_gate_factory( + self.actor, gate.DEFAULT_RECORDS_DIR)(worktree) def _mirror_turn_into_sidecar(self, key, *, document: str, turn_id: str, model_id: str, bound_buffer_key: str, @@ -545,8 +554,8 @@ def _one(name): doxbench_error_status(DOXBENCH_ERR_INVALID_TURN_REQUEST), doxbench_error_body(DOXBENCH_ERR_INVALID_TURN_REQUEST)) return - from openxdox import doxbench_scope - key = doxbench_scope.ScopeKey( + # openDox's OWN scope type (`doxbench_scope_types`), no seam (T084) + key = ScopeKey( repository=fields["repository"], ref=fields["ref"], tile_kind=fields["tile_kind"], tile_id=fields["tile_id"]) worktree = self._session_worktree_for(key) @@ -950,7 +959,14 @@ def _handle_workbench_model_intake_surface(self, head_only: bool) -> None: # on top of one would be offering to write into a file it could not # read first. disclosure = None - offered = bool(disclosure and disclosure.get("broker")) + # NOT OFFERED WHERE IT COULD NOT FINISH (plan 034 T084; RULED by + # Brett Heap, 2026-10-02, "Refuse by name, hide intake + # (Recommended)"): the enrolment ends in a recorded approval, a + # governed gate-action record only a host's gate writes, so with none + # registered the flow is not offered, even beside a hand-written + # broker block, and the reason names the seam. + records = column_seams.gate_records_writable() + offered = bool(disclosure and disclosure.get("broker")) and records from opendox import doxbench_binding envelope: dict = { "kind": "workbench-model-intake", @@ -968,7 +984,8 @@ def _handle_workbench_model_intake_surface(self, head_only: bool) -> None: "declarations": (disclosure or {}).get("declarations", []), } if not offered: - envelope["reason"] = doxbench_intake.NO_BROKER_NOTICE + envelope["reason"] = (doxbench_intake.NO_BROKER_NOTICE if records + else column_seams.GATE_RECORDS_REFUSAL) self._serve_bytes(json.dumps(envelope).encode("utf-8"), JSON_CTYPE, head_only) @@ -1074,6 +1091,22 @@ def _handle_workbench_model_intake(self) -> None: # broker's own declared flow rather than by this check. self._send_error_or_intake(DOXBENCH_ERR_INVALID_INTAKE_REQUEST) return + if not column_seams.gate_records_writable(): + # AN ENROLMENT THIS INSTALL COULD NEVER APPROVE IS NOT STARTED + # (plan 034 T084; RULED by Brett Heap, 2026-10-02, "Refuse by name, + # hide intake (Recommended)"). Enrolling writes a PENDING + # declaration, which suppresses its binding until a recorded + # approval, and the approval is a governed gate-action record that + # openDox's default does not write. So with no host's gate + # registered the act refuses here, naming the seam, before a + # broker is spawned or a declaration is written, and the body it + # sent is drained unread. The surface already answered + # `offered: false` with the same sentence. + if length > 0: + _drain_refused_body(self.rfile, length) + self._intake_refusal(DOXBENCH_ERR_INTAKE_REFUSED, + column_seams.GATE_RECORDS_REFUSAL) + return try: broker = store.broker() except doxbench_intake.IntakeRefused as error: @@ -1216,7 +1249,27 @@ def _handle_workbench_model_approval(self) -> None: doxbench_error_body(refusal)) return from opendox import doxbench_intake - from openxdox import gate_console + # THROUGH THE GATE SEAM (plan 034 T084; #1144 4.3 as T007 batch L's + # addendum reads). This was `from openxdox import gate_console`, which + # dropped the connection of every standalone approval. The approval + # IS a governed gate-action record, which openDox's own default does + # not write (`default_columns.GATE`), so with no host's gate registered + # the act refuses here, NAMING THE SEAM (4.2), before it parses a body + # or reads a store, and no record is written. RULED by Brett Heap, + # 2026-10-02: "Refuse by name, hide intake (Recommended)". The body is + # drained unread first, as the intake act drains a refused one, so the + # refusal is not lost to a reset of a socket closed with bytes unread. + if not column_seams.gate_records_writable(): + try: + length = int(self.headers.get("Content-Length", "0")) + except (TypeError, ValueError): + length = 0 + if length > 0: + _drain_refused_body(self.rfile, length) + self._intake_refusal(DOXBENCH_ERR_APPROVAL_REFUSED, + column_seams.GATE_RECORDS_REFUSAL) + return + gate_console = column_seams.gate.current() store = self._workbench_declaration_store() if store is None: self._send_json( @@ -1260,18 +1313,21 @@ def _handle_workbench_model_approval(self) -> None: approved_by=str(self.actor), expires_at=doxbench_intake.approval_expiry(), audit_ref=binding.credential_ref) - record = gate_console.build_gate_action_record( - actor=str(self.actor), - action=doxbench_intake.GATE_ACTION_APPROVE_MODEL, - at=at, - model_declaration=binding_id, - model_approval=approved.approval_block(), - provenance=gate_console.HTTP_CONSOLE_TOKEN, - artifacts=[{"kind": "other", - "reference": doxbench_intake.declarations_path( - Path(self.checkout_root)).relative_to( - Path(self.checkout_root)).as_posix()}]) try: + # INSIDE the refusal net (T084): a gate that refuses to build the + # record is answered as a stated refusal, never a dropped + # connection. + record = gate_console.build_gate_action_record( + actor=str(self.actor), + action=doxbench_intake.GATE_ACTION_APPROVE_MODEL, + at=at, + model_declaration=binding_id, + model_approval=approved.approval_block(), + provenance=gate_console.HTTP_CONSOLE_TOKEN, + artifacts=[{"kind": "other", + "reference": doxbench_intake.declarations_path( + Path(self.checkout_root)).relative_to( + Path(self.checkout_root)).as_posix()}]) gate_console.validate_gate_action_record(record) human = gate_console.HumanGate( Path(self.checkout_root), @@ -1666,13 +1722,17 @@ def _handle_workbench_chat_turn(self) -> None: from opendox import doxbench_hash from opendox import doxbench_model - from openxdox import doxbench_scope + # The scope authority through its seam (plan 034 T084; #1144 4.3, + # batch L): a host's registration or openDox's own default, never a + # deferred `openxdox` import that drops the connection where openXdox + # is not installed. + scope_authority = column_seams.scope.current() from opendox import doxbench_turns - key = doxbench_scope.ScopeKey(repository=scope_fields["repository"], - ref=scope_fields["ref"], - tile_kind=scope_fields["tile_kind"], - tile_id=scope_fields["tile_id"]) + key = ScopeKey(repository=scope_fields["repository"], + ref=scope_fields["ref"], + tile_kind=scope_fields["tile_kind"], + tile_id=scope_fields["tile_id"]) # ---- step 5: scope, all from SERVER truth ---- projection = None session_base = None @@ -1723,11 +1783,11 @@ def _session_text(rel, _root=source_root): # cannot add a path to this set. With no live session on the # scope's own branch family the answer is empty and this # projection is what it was before T107. - created_paths = doxbench_scope.session_created_paths_for_scope( + created_paths = scope_authority.session_created_paths_for_scope( self.source.registry, key, repository=entry.repository, ref=entry.ref, source_root=Path(entry.source_root)) - projection = doxbench_scope.resolve_scope( + projection = scope_authority.resolve_scope( snapshot, key, source_root=Path(entry.source_root), created_paths=created_paths) if projection is None: @@ -1750,7 +1810,7 @@ def _session_text(rel, _root=source_root): doxbench_turns.buffer_key_for(b) for b in turn_buffers), paths=tuple(b.path for b in turn_buffers), ) - except (doxbench_turns.TurnScopeError, doxbench_scope.ScopeConfinementError, + except (doxbench_turns.TurnScopeError, ScopeConfinementError, ValueError, OSError): scope_refused = True @@ -2608,7 +2668,6 @@ def _handle_workbench_document_abstract(self) -> None: only then a provider.""" from opendox import doxbench_hash from opendox import doxbench_model - from openxdox import doxbench_scope from opendox import doxbench_turns # ---- step 1: the plane. The SAME three-part verdict the catalog and @@ -2663,9 +2722,17 @@ def _handle_workbench_document_abstract(self) -> None: subject_path = fields["subject_path"] model_id = fields["model_id"] refresh = fields["refresh"] - key = doxbench_scope.ScopeKey(**fields["scope"]) + key = ScopeKey(**fields["scope"]) # ---- step 4: scope, all from SERVER truth ---- + # THE SCOPE AUTHORITY IS READ HERE, BELOW STEP 1 (plan 034 T084; #1144 + # batch L, RULED `5920216845`). It was a deferred import at the top of + # this handler, above the plane check, so in an openDox with no + # openXdox installed every request, even one step 1 would have + # refused, ended in a dropped connection. It is now the registration + # current at `column_seams.scope`, a host's or openDox's own default, + # and a plane that refuses never reads it. + scope_authority = column_seams.scope.current() projection = None source_root = None snapshot = None @@ -2677,15 +2744,15 @@ def _handle_workbench_document_abstract(self) -> None: else: source_root = Path(entry.source_root) snapshot = json.loads(entry.read_bytes()) - created_paths = doxbench_scope.session_created_paths_for_scope( + created_paths = scope_authority.session_created_paths_for_scope( self.source.registry, key, repository=entry.repository, ref=entry.ref, source_root=source_root) - projection = doxbench_scope.resolve_scope( + projection = scope_authority.resolve_scope( snapshot, key, source_root=source_root, created_paths=created_paths) if projection is None: scope_refused = True - except (doxbench_scope.ScopeConfinementError, ValueError, OSError): + except (ScopeConfinementError, ValueError, OSError): scope_refused = True if scope_refused: # The same fail-closed refusal the chat route gives, so no response diff --git a/tests/test_capability_honesty.py b/tests/test_capability_honesty.py index 6ee06e36..ae5c1f97 100644 --- a/tests/test_capability_honesty.py +++ b/tests/test_capability_honesty.py @@ -340,3 +340,438 @@ def test_the_verdict_follows_the_bindings_and_keeps_the_other_conditions() -> No actor="a", route_bindings=both, **{**kwargs, "refresh_binding": None}) assert unbound["actions"]["refresh"] is False + + +# --------------------------------------------------------------------------- +# 5 — the three sites that dropped a connection, and the chat turn +# --------------------------------------------------------------------------- + +def _call(base: tuple[str, int], method: str, path: str, *, + body: bytes | None = None, token: str | None = None + ) -> tuple[int, dict, str]: + """One request carrying `body` and, where given, the console token, from + a same-origin JSON client. The status, the JSON body (`{}` where it is not + JSON) and the raw text. A dropped connection RAISES (`RemoteDisconnected`, + a reset), which fails the case: that is the failure this section exists + to rule out.""" + from opendox import serve + connection = http.client.HTTPConnection(*base, timeout=30) + try: + headers = {"Content-Type": "application/json"} + if token: + headers[serve.CONSOLE_TOKEN_HEADER] = token + connection.request(method, path, body=body, headers=headers) + response = connection.getresponse() + raw = response.read().decode("utf-8", errors="replace") + try: + parsed = json.loads(raw) if raw else {} + except ValueError: + parsed = {} + return (response.status, parsed if isinstance(parsed, dict) else {}, + raw) + finally: + connection.close() + + +def _json(payload: dict) -> bytes: + return json.dumps(payload).encode("utf-8") + + +#: A scope no snapshot of the fixture's declares a document under: each +#: request below is refused, and none is served a document. +_SCOPE = {"repository": "fixture", "ref": "main", "tile_kind": "staged", + "tile_id": "honesty-topic"} + +#: A well-formed abstract request: the shape step 3 accepts. +_ABSTRACT = {"scope": _SCOPE, "subject_path": "notes/one.md", + "model_id": "honesty-model"} + + +def _chat_turn() -> dict: + """A well-formed v2 chat turn: the shape the body parser accepts, an + outline and one document, bound to the document.""" + from opendox.serve_wire import DOXBENCH_CHAT_TURN_V2_KIND + + def buffer(kind, path, content): + return {"kind": kind, "repository": "fixture", "path": path, + "base_ref": "main", "base_revision": "0" * 40, + "base_hash": "0" * 64, "content_hash": "0" * 64, + "content": content, "dirty": False} + + return {"schema_version": 1, "kind": DOXBENCH_CHAT_TURN_V2_KIND, + "client_turn_id": "honesty-turn-1", "scope": _SCOPE, + "working_subject": "", "message": "What does this note claim?", + "model_id": "honesty-model", "transcript": [], + "bound_buffer": "notes/one.md", + "buffers": [buffer("outline", None, "# outline"), + buffer("document", "notes/one.md", "# one")]} + + +def _structured(answer: tuple[int, dict, str]) -> bool: + """An answer a client can read: a status, and a JSON body naming its + error, or a stated 404. Anything else (an empty 500, a reset) is not.""" + status, body, raw = answer + if body.get("ok") is False and isinstance(body.get("error"), str): + return True + return status == 404 and bool(raw.strip()) + + +def _standalone(tmp_path, repo): + """`python -m opendox.serve` over `repo`, as section 1 runs it, with its + base address and its capabilities.""" + out = _snapshot(tmp_path, repo) + child = Child(tmp_path, "opendox.serve", "--snapshot", str(out), + "--checkout-root", str(repo), "--port", "0") + match = child.wait_for_line(_SERVE_URL) + base = (match.group(2), int(match.group(3))) + return child, base, _capabilities(base) + + +def test_the_three_crash_sites_answer_a_standalone_server( + tmp_path, monkeypatch) -> None: + """#1144 batch L (RULED `5920216845`, item 1). At `047bb4fa` each of these + three ended a standalone request with a dropped connection, on a deferred + `openxdox` import: the document abstract's above its step-1 check, model + approval's past its only check, and the project register's with no check + at all. Each now gets a structured answer, and none reaches for openXdox. + Run with an identity, so `session` reads true and the abstract and the + approval pass the checks a plane with no actor would refuse at.""" + from opendox import column_seams + from opendox.serve_wire import (DOXBENCH_ERR_APPROVAL_REFUSED, + DOXBENCH_ERR_MODEL_CAPABILITY_UNAVAILABLE) + _clean_environment(monkeypatch) + repo = _repository(tmp_path, identity=True) + child, base, caps = _standalone(tmp_path, repo) + try: + token = caps.get("console_token") + assert caps["actions"]["session"] is True and token, caps + abstract = _call(base, "POST", "/actions/workbench/document-abstract", + body=_json(_ABSTRACT), token=token) + approval = _call(base, "POST", "/actions/workbench/model-approval", + body=_json({"binding": "honesty-binding"}), + token=token) + register = _call(base, "GET", "/project-register.json") + for name, answer in (("document abstract", abstract), + ("model approval", approval), + ("project register", register)): + assert _structured(answer), f"{name}: {answer}" + # step 1 refuses once `gate` reads false (batch L) + assert abstract[1]["error"] == DOXBENCH_ERR_MODEL_CAPABILITY_UNAVAILABLE + # the seam refuses by name: no host's gate, so no record is written + assert approval[1]["error"] == DOXBENCH_ERR_APPROVAL_REFUSED, approval + assert approval[1]["reason"] == column_seams.GATE_RECORDS_REFUSAL + # openDox's own kickoff discovers no register: today's 404 + assert register[0] == 404 and "no project register" in register[2] + assert not (repo / "ideation" / "dashboard" / "gate-records").exists() + assert child.interrupt() == 0, child.stderr_text() + finally: + child.kill() + assert child.refused() == [], child.refused() + + +#: The binding `opendox model-binding add` declares for the cases below, as +#: `tests/test_model_provider_broker.py` declares its own. Nothing is spawned +#: and nothing is contacted: no case dispatches a turn. +_BINDING = ["--id", "honesty-binding", "--label", "Honesty binding", + "--provider", "honesty-provider", + "--credential-ref", "opref-4f2a91c07be3d5a8140b6e77", + "--auth-kind", "api_key", + "--credential-approver", "fixture@example.invalid", + "--endpoint", "https://provider.invalid/turn", + "--dialect", "xfactory-prompt-v1", + "--", "honesty-broker", "--home", "/srv/{binding_id}"] + + +def test_a_chat_turn_with_a_binding_configured_is_answered_standalone( + tmp_path, monkeypatch) -> None: + """The chat-turn route over a standalone server whose checkout DECLARES a + model binding (`opendox model-binding add`), so the turn does not stop at + "no model configured" (T081) and runs on toward its scope step, which + reached for openXdox by a deferred import until this task. The answer is + structured, whichever step refuses it.""" + _clean_environment(monkeypatch) + repo = _repository(tmp_path, identity=True) + added, status = run_module(tmp_path, "opendox.cli", "model-binding", "add", + "--repo-root", str(repo), *_BINDING) + assert status == 0, added.stderr_text() + child, base, caps = _standalone(tmp_path, repo) + try: + token = caps.get("console_token") + assert caps["actions"]["session"] is True and token, caps + answer = _call(base, "POST", "/actions/workbench/chat-turn", + body=_json(_chat_turn()), token=token) + assert _structured(answer), answer + assert child.interrupt() == 0, child.stderr_text() + finally: + child.kill() + assert child.refused() == [], child.refused() + + +class _Conforms: + """A validator every instance conforms to.""" + + @staticmethod + def iter_errors(_instance): + return iter(()) + + +class _EveryKind(dict): + """The released validators, as a plane that can read its contract has + them: one for every kind, each of which every instance conforms to. So a + turn's own shape carries it to the scope step.""" + + def get(self, _kind, _default=None): + return _Conforms() + + +class _Port: + """A model port. Never dispatched: every case is refused before.""" + + +def test_a_chat_turn_reaching_its_scope_step_is_refused_not_dropped( + composed_turns) -> None: + """A composed host with the released validators and a model port, so a + well-formed turn reaches step 5. At `047bb4fa` step 5 opened with a + deferred `from openxdox import doxbench_scope`, and the connection dropped. + The scope authority is now the one registered at `column_seams.scope`, + openDox's own default here, and the scope is refused in the released + failure envelope.""" + from opendox.serve_wire import DOXBENCH_ERR_TURN_SCOPE_REFUSED + base, caps = composed_turns() + status, body, raw = _call(base, "POST", "/actions/workbench/chat-turn", + body=_json(_chat_turn()), + token=caps["console_token"]) + assert body.get("error") == DOXBENCH_ERR_TURN_SCOPE_REFUSED, (status, raw) + assert body.get("client_turn_id") == "honesty-turn-1", body + + +def test_a_document_abstract_past_step_one_is_refused_not_dropped( + composed_turns) -> None: + """A composed host that contributes the gate routes, so `gate` reads true + and an abstract request passes step 1 and reaches its scope step, which + reads the scope authority through its seam (batch L): refused, stated.""" + from opendox.serve_wire import DOXBENCH_ERR_TURN_SCOPE_REFUSED + base, caps = composed_turns(_gate()) + assert caps["actions"]["gate"] is True, caps + status, body, raw = _call(base, "POST", + "/actions/workbench/document-abstract", + body=_json(_ABSTRACT), + token=caps["console_token"]) + assert body.get("error") == DOXBENCH_ERR_TURN_SCOPE_REFUSED, (status, raw) + + +@pytest.fixture() +def composed_turns(tmp_path): + """`build(*contributions)`: `composed`'s host, with the released + validators and a model port declared, as a plane that can run a turn has + them. Yields `(base, capabilities)`.""" + from opendox import serve + + repo = _repository(tmp_path, identity=True) + out = _snapshot(tmp_path, repo) + servers = [] + + def build(*contributed): + bindings = [binding for binding, _mixin in contributed] + mixins = [mixin for _binding, mixin in contributed] + httpd = serve.build_server( + WEB, out, repo, port=0, actor="brett", + route_extensions=(_Contribution(bindings, mixins),) + if contributed else (), + schema_validator_factory=_EveryKind, + model_port_factory=_Port) + worker = threading.Thread(target=httpd.serve_forever, daemon=True) + worker.start() + servers.append((httpd, worker)) + base = httpd.server_address[:2] + return base, _capabilities(base) + + try: + yield build + finally: + for httpd, worker in servers: + httpd.shutdown() + httpd.server_close() + worker.join(timeout=10) + + +# --------------------------------------------------------------------------- +# 6 — model intake and approval: not offered where no record can be written +# --------------------------------------------------------------------------- +# +# RULED by Brett Heap, 2026-10-02, "Refuse by name, hide intake +# (Recommended)". The enrolment ends in a recorded approval, a governed +# gate-action record that only a HOST's gate writes; openDox's own default +# writes none. So with no host's gate registered the surface answers +# `offered: false` with a stated reason, even beside a hand-written broker +# block, and the intake act and the approval refuse naming the seam, writing +# nothing. A host that registers its gate is offered the flow, and its +# approval is recorded. + +#: The pending declaration a hand-written document carries, for the binding +#: `_BINDING` declares. +_PENDING = {"binding_id": "honesty-binding", "status": "pending", + "install_posture": "single-operator", "proposed_by": "brett", + "proposed_at": "2026-10-02T00:00:00Z"} + +#: The intake act's declared facts, on its query string. +_INTAKE = ("/actions/workbench/model-intake?binding=honesty-intake" + "&label=Honesty&provider=honesty-provider" + "&endpoint=https%3A%2F%2Fprovider.invalid%2Fturn" + "&dialect=xfactory-prompt-v1&kind=api_key") + + +def _declare(tmp_path: Path, repo: Path) -> Path: + """A binding declared with `opendox model-binding add`, and a declarations + document written BY HAND beside it, naming a broker and the binding's + pending declaration. Returns the document's path.""" + import yaml + from opendox import doxbench_intake + + added, status = run_module(tmp_path, "opendox.cli", "model-binding", "add", + "--repo-root", str(repo), *_BINDING) + assert status == 0, added.stderr_text() + path = doxbench_intake.declarations_path(repo) + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(yaml.safe_dump({ + "schema_version": doxbench_intake.SCHEMA_VERSION, + "kind": doxbench_intake.DECLARATIONS_KIND, + "broker": {"kind": doxbench_intake.BROKER_KIND, + "argv": ["honesty-broker", "intake"]}, + "declarations": [dict(_PENDING)], + }, sort_keys=False), encoding="utf-8") + return path + + +def test_a_standalone_server_does_not_offer_an_intake_it_could_not_approve( + tmp_path, monkeypatch) -> None: + from opendox import column_seams + from opendox.serve_wire import (DOXBENCH_ERR_APPROVAL_REFUSED, + DOXBENCH_ERR_INTAKE_REFUSED) + _clean_environment(monkeypatch) + repo = _repository(tmp_path, identity=True) + document = _declare(tmp_path, repo) + before = document.read_bytes() + child, base, caps = _standalone(tmp_path, repo) + try: + token = caps.get("console_token") + assert caps["actions"]["session"] is True and token, caps + status, surface, raw = _call(base, "GET", "/workbench/model-intake", + token=token) + assert status == 200, raw + # a broker block is declared, and still the flow is not offered + assert surface["offered"] is False, surface + assert surface["reason"] == column_seams.GATE_RECORDS_REFUSAL, surface + assert surface["auth_kinds"] == [] and surface["dialects"] == [] + intake = _call(base, "POST", _INTAKE, body=b"not-a-real-credential", + token=token) + assert intake[1].get("error") == DOXBENCH_ERR_INTAKE_REFUSED, intake + assert intake[1]["reason"] == column_seams.GATE_RECORDS_REFUSAL + approval = _call(base, "POST", "/actions/workbench/model-approval", + body=_json({"binding": "honesty-binding"}), + token=token) + assert approval[1].get("error") == DOXBENCH_ERR_APPROVAL_REFUSED + assert approval[1]["reason"] == column_seams.GATE_RECORDS_REFUSAL + assert child.interrupt() == 0, child.stderr_text() + finally: + child.kill() + assert child.refused() == [], child.refused() + # nothing was written: the declaration is still pending, no record exists + assert document.read_bytes() == before + assert not (repo / "ideation" / "dashboard" / "gate-records").exists() + + +class _HostGate: + """A HOST's gate: openDox's own default for every name, except that it + WRITES the gate-action record the default refuses to, into a list.""" + + def __init__(self) -> None: + from opendox import column_seams, default_columns + for name in (*column_seams.GATE_CALLABLES, *column_seams.GATE_VALUES): + if name not in type(self).__dict__: + setattr(self, name, getattr(default_columns.GATE, name)) + self.__name__ = "tests.test_capability_honesty._HostGate" + self.written: list[tuple[str, dict]] = [] + + def build_gate_action_record(self, **fields): + return dict(fields) + + def validate_gate_action_record(self, record): + assert record["action"] and record["actor"], record + + def HumanGate(self, root, prefixes, *, human_actor): # noqa: N802 + return (root, tuple(prefixes), human_actor) + + def write_gate_action_record(self, human, records_dir, record): + self.written.append((records_dir, record)) + return Path(human[0]) / records_dir / "honesty.gate-action.yaml" + + +@pytest.fixture() +def host_gate(): + """A host's gate registered at `column_seams.gate` for the case, as a host + registers it at process start, and dropped afterwards. Whatever this + process registered before (a default an earlier case read) is dropped + first: the swap is deliberate.""" + from opendox import column_seams + gate = _HostGate() + column_seams.gate.unregister() + column_seams.gate.register(gate) + try: + yield gate + finally: + column_seams.gate.unregister() + + +def test_a_host_that_registers_its_gate_is_offered_intake_and_approves( + tmp_path, host_gate) -> None: + import yaml + from opendox import doxbench_intake, serve + + repo = _repository(tmp_path, identity=True) + document = _declare(tmp_path, repo) + out = _snapshot(tmp_path, repo) + httpd = serve.build_server(WEB, out, repo, port=0, actor="brett") + worker = threading.Thread(target=httpd.serve_forever, daemon=True) + worker.start() + try: + base = httpd.server_address[:2] + caps = _capabilities(base) + token = caps["console_token"] + status, surface, raw = _call(base, "GET", "/workbench/model-intake", + token=token) + assert status == 200, raw + assert surface["offered"] is True and "reason" not in surface, surface + assert surface["auth_kinds"], surface + status, approved, raw = _call( + base, "POST", "/actions/workbench/model-approval", + body=_json({"binding": "honesty-binding"}), token=token) + assert status == 200 and approved.get("ok") is True, raw + finally: + httpd.shutdown() + httpd.server_close() + worker.join(timeout=10) + # the host's gate wrote the record, BEFORE the document moved + assert len(host_gate.written) == 1, host_gate.written + records_dir, record = host_gate.written[0] + assert record["action"] == doxbench_intake.GATE_ACTION_APPROVE_MODEL + assert record["model_declaration"] == "honesty-binding" + stored = yaml.safe_load(document.read_text(encoding="utf-8")) + assert stored["declarations"][0]["status"] == doxbench_intake.STATUS_APPROVED + assert stored["declarations"][0]["approved_by"] == "brett" + + +def test_whether_a_gate_record_can_be_written_follows_the_registration( + host_gate) -> None: + """The predicate itself: a host's registration answers true; openDox's + own default, registered by an entry point, answers false; and asking reads + nothing, so it closes no default's window.""" + from opendox import column_seams + assert column_seams.gate_records_writable() is True + column_seams.gate.unregister() + assert column_seams.gate_records_writable() is False + column_seams.register_defaults() + assert column_seams.gate_records_writable() is False + # still replaceable: asking did not read the default + column_seams.gate.register(host_gate) + assert column_seams.gate_records_writable() is True diff --git a/tests/test_consumer_reach.py b/tests/test_consumer_reach.py index 0248a22f..874e98c3 100644 --- a/tests/test_consumer_reach.py +++ b/tests/test_consumer_reach.py @@ -153,6 +153,14 @@ def _import_in_subprocess(module: str, *, consumer_blocked: bool, "opendox.default_registry", "opendox.default_projection", "opendox.rfc3339", + # Plan 034 T084 (#1144 4.3; R1Q10 (a)). The consumer columns' seams and + # openDox's own defaults behind them, which retired the last stand-in, the + # gate column's. As with T055's four: a seam whose whole job is to let a + # HOST hand openDox its gate, its scope authority, its commission reader + # and its register is where a reach into `openxdox` would look reasonable, + # and neither module makes one. + "opendox.column_seams", + "opendox.default_columns", ) #: Modules that STILL require the consumer at import time, with the reason. They @@ -815,208 +823,16 @@ def test_the_prefix_is_refused_rather_than_doubled() -> None: # 3 — the converted sites, and the blind spot that made this file necessary # -------------------------------------------------------------------------- -#: `module path -> the names this slice rebound to the late seam`. Each must be -#: reachable ONLY from a function body: a default argument, an annotation, a -#: decorator or a module-level expression would resolve the consumer at import -#: time and make the conversion a census trick. -CONVERTED_SITES = { - # RE-DERIVED BY PLAN 034 T034 from the tree, where phase 1's lanes joined: - # every module-level name bound to a `consumer_reach` stand-in, which - # `test_every_name_bound_to_the_seam_is_guarded` below now derives on every - # run. The table had fallen behind by five names in two files, and the - # guard never read them: - # * `branch_session.py`'s `gate_console`. This is one of the two reverts - # the guard is named for (the NINE default-argument sites this file's - # docstring gives), and it was the one module the table left out; - # * `cli.py`'s other four aliases, the two late callables BUILD slice 2b - # bound for the generate verbs and the `--generated-at` check, and the - # one late constant. - # None of the five is read at import time, so the tree was already right. - # - # NARROWED BY PLAN 034 T055 to the gate column alone. `cli.py`'s - # `snapshot_mod`, `corpus_root_refusal`, `generate_snapshot`, - # `is_rfc3339_datetime` and `SCANNED_ROOTS`, `serve.py`'s `registry_mod` - # and `hosted_ref_refused`, `serve_workbench.py`'s `registry_mod` and - # `workbench.py`'s `find_validator` bind no stand-in any more: the - # projection mechanism is read from declared seams - # (`opendox.projection_seams`, `opendox.generator_seam`), and the two - # `registry_mod`s are the registry seam's proxy, which - # `tests/test_projection_seams.py` holds to the same import-time rule. - # `serve.py` still names the two late COLUMNS as mixin bases, which a class - # statement needs before its first instance exists, so it binds no name - # here either. - "branch_session.py": ("gate_console",), - "cli.py": ("gate_mod",), -} - - -def _import_time_uses(path: Path, names: frozenset[str]) -> list[tuple[int, str]]: - """Every use of `names` that runs when the module is imported. - - The module body, module-level `if`/`try`/`with`, CLASS bodies, and — the - case the openXdox-side census cannot see — a function's DEFAULTS, - ANNOTATIONS and DECORATORS, which are evaluated where the `def` sits and - not where it is called. Only a function BODY defers. - """ - hits: list[tuple[int, str]] = [] - - def used(node: ast.AST, why: str) -> None: - for inner in ast.walk(node): - # READS only. The one module-level STORE of each of these names is - # the seam binding itself (`gate_mod = consumer_reach.gate_console`) - # — the line this slice wrote, which resolves nothing. - if isinstance(inner, ast.Name) and inner.id in names \ - and isinstance(inner.ctx, ast.Load): - hits.append((inner.lineno, f"{inner.id} ({why})")) - - def walk(body: list[ast.stmt], at_import_time: bool) -> None: - for node in body: - if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)): - if at_import_time: - args = node.args - for default in [*args.defaults, *(d for d in args.kw_defaults if d)]: - used(default, "default argument") - for arg in [*args.posonlyargs, *args.args, *args.kwonlyargs, - args.vararg, args.kwarg]: - if arg is not None and arg.annotation is not None: - used(arg.annotation, "annotation") - for decorator in node.decorator_list: - used(decorator, "decorator") - if node.returns is not None: - used(node.returns, "return annotation") - continue - if not at_import_time: - continue - nested: list[ast.stmt] = [] - for _field, value in ast.iter_fields(node): - items = value if isinstance(value, list) else [value] - for item in items: - if isinstance(item, ast.stmt): - nested.append(item) - elif isinstance(item, ast.AST): - used(item, "module level") - walk(nested, True) - - walk(ast.parse(path.read_text(encoding="utf-8")).body, True) - return sorted(set(hits)) - - -def _module_level_statements(body: list[ast.stmt]): - """The statements a module runs when it is imported, in source order: its - body, and the bodies of a module-level `if`, `try`, `with`, `for`, - `while` or `match`, with their handlers and `else` blocks. A function's - body and a class's are not the module's names.""" - for node in body: - yield node - if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)): - continue - for _field, value in ast.iter_fields(node): - for item in value if isinstance(value, list) else []: - if isinstance(item, ast.stmt): - yield from _module_level_statements([item]) - elif isinstance(item, (ast.ExceptHandler, ast.match_case)): - yield from _module_level_statements(item.body) - - -def _names_bound_to_the_seam(path: Path) -> set[str]: - """Every name a module binds, at its top level, to a `consumer_reach` - stand-in. - - Every spelling of the seam counts. `from .consumer_reach import X` (`..` - in a subpackage) and `from opendox.consumer_reach import X` bind one - directly. Once the seam itself is reachable, so do `Y = .X`, its - annotated form `Y: T = .X`, and `Y = .f(...)`, which mints one - (`module`, `function`, `constant`). `` is a name bound to the - module (`from . import consumer_reach`, `import opendox.consumer_reach as - cr`, or `cr = consumer_reach` after either), or the package's attribute - `opendox.consumer_reach`, once `import opendox` or `import - opendox.consumer_reach` has bound `opendox`. Each is read wherever the - module runs it at import time, a module-level `if` or `try` included.""" - tree = ast.parse(path.read_text(encoding="utf-8")) - # The leading dots from this file to the `opendox` package: one for a - # module at the top of it, two in a subpackage, and so on. - level = len(path.relative_to(PACKAGE).parts) - seam_aliases: set[str] = set() - package_aliases: set[str] = set() - bound: set[str] = set() - statements = list(_module_level_statements(tree.body)) - for node in statements: - if isinstance(node, ast.ImportFrom): - package = (node.level == level and node.module is None) or \ - (node.level == 0 and node.module == "opendox") - seam = (node.level == level and node.module == "consumer_reach") or \ - (node.level == 0 and node.module == "opendox.consumer_reach") - if package: - seam_aliases |= {a.asname or a.name for a in node.names - if a.name == "consumer_reach"} - if seam: - bound |= {a.asname or a.name for a in node.names} - elif isinstance(node, ast.Import): - for alias in node.names: - if alias.name == "opendox.consumer_reach" and alias.asname: - seam_aliases.add(alias.asname) - elif alias.name == "opendox" or ( - alias.name.startswith("opendox.") and not alias.asname): - package_aliases.add(alias.asname or "opendox") - - def is_the_seam(expr: ast.expr | None) -> bool: - return (isinstance(expr, ast.Name) and expr.id in seam_aliases) or ( - isinstance(expr, ast.Attribute) and expr.attr == "consumer_reach" - and isinstance(expr.value, ast.Name) - and expr.value.id in package_aliases) - - for node in statements: - if not isinstance(node, (ast.Assign, ast.AnnAssign)): - continue - targets = node.targets if isinstance(node, ast.Assign) else [node.target] - names = {t.id for t in targets if isinstance(t, ast.Name)} - if is_the_seam(node.value): - seam_aliases |= names - continue - value = node.value.func if isinstance(node.value, ast.Call) else node.value - if isinstance(value, ast.Attribute) and is_the_seam(value.value): - bound |= names - return bound - - -def test_every_name_bound_to_the_seam_is_guarded() -> None: - """The guard's table is the tree's, name for name (plan 034 T034). - - A name bound to the seam and missing from `CONVERTED_SITES` is a name the - import-time guard never reads. A name the table keeps and no module binds - any more is a guard over nothing. The seam's own module is left out: it - DEFINES the stand-ins.""" - derived = {} - for path in sorted(PACKAGE.rglob("*.py")): - if path.name == "consumer_reach.py": - continue - bound = _names_bound_to_the_seam(path) - if bound: - derived[path.relative_to(PACKAGE).as_posix()] = bound - starred = sorted(module for module, names in derived.items() if "*" in names) - assert not starred, ( - f"{starred} import the seam's stand-ins with a wildcard. The guard " - "reads each converted name by name, and a wildcard gives it none to " - "read: import each stand-in by its name") - declared = {module: set(names) for module, names in CONVERTED_SITES.items()} - assert derived == declared, ( - f"the names each module binds to `consumer_reach` are {derived}, and " - f"CONVERTED_SITES guards {declared}. Add a new binding to the table, so " - "its import-time uses are refused, and take a retired one out") - - -@pytest.mark.parametrize("module_file", sorted(CONVERTED_SITES)) -def test_a_converted_name_is_never_used_at_import_time(module_file: str) -> None: - """The guard that would have caught the two reverts before they were made.""" - found = _import_time_uses(PACKAGE / module_file, - frozenset(CONVERTED_SITES[module_file])) - assert found == [], ( - f"src/opendox/{module_file} uses a late-bound consumer name where it " - f"runs AT IMPORT TIME: {found}. The stand-in would resolve `openxdox` " - "there, so removing the import statement would lower openXdox-code's " - "ratchet without removing the dependency — a census that reads better " - "than the tree. Defer the use, or leave the import alone and ask for " - "the declared-edit ruling") +# RETIRED BY PLAN 034 T084 (#1144 4.3). This section held `CONVERTED_SITES`, +# every module-level name bound to a `consumer_reach` stand-in, and refused any +# read of one at import time: a default argument, an annotation, a decorator or +# a module-level expression, the blind spot this file was written for. The last +# two, `branch_session.py`'s `gate_console` and `cli.py`'s `gate_mod`, are now +# proxies over `opendox.column_seams.gate`, so no name binds a stand-in. The +# same rule holds them where every seam proxy is held: +# `tests/test_projection_seams.py`'s +# `test_no_proxy_over_a_seam_is_read_at_import_time`, which names both, with no +# import-time read. def test_no_module_under_src_names_the_pre_carve_package_at_import_time() -> None: diff --git a/tests/test_profile_registration.py b/tests/test_profile_registration.py index 18662836..29780fd1 100644 --- a/tests/test_profile_registration.py +++ b/tests/test_profile_registration.py @@ -544,8 +544,8 @@ def _import_time_nodes(body: list[ast.stmt]): """Every node evaluated when the module is IMPORTED. A function's BODY defers; its decorators and default arguments do not, and a - class body runs outright. `tests/test_consumer_reach.py::_import_time_uses` - draws the line in the same place for the consumer seam, and for the same + class body runs outright. `tests/test_projection_seams.py::_import_time_reads` + draws the line in the same place for every seam proxy, and for the same reason: `ast.walk` over a module descends into function bodies and would call every deferred binding an import-time one. """ diff --git a/tests/test_projection_seams.py b/tests/test_projection_seams.py index cc49d645..371d6c7e 100644 --- a/tests/test_projection_seams.py +++ b/tests/test_projection_seams.py @@ -1717,15 +1717,23 @@ def test_an_explicit_manifest_validator_script_still_runs(tmp_path) -> None: # 9 — no proxy over a seam is read at import time # --------------------------------------------------------------------------- +#: The modules whose seams are `projection_seams._Seam`s, so whose proxies +#: this rule holds: the projection mechanism's, and the consumer columns' +#: (`opendox.column_seams`, plan 034 T084), which took over the import-time +#: guard `tests/test_consumer_reach.py` kept over the `consumer_reach` +#: stand-ins those proxies replace. +SEAM_MODULES = ("projection_seams", "column_seams") + + def _proxy_bindings(tree: ast.Module) -> set[str]: - """Module-level names bound to `projection_seams..proxy`.""" + """Module-level names bound to `..proxy`.""" bound = set() for node in tree.body: if isinstance(node, ast.Assign) and isinstance(node.value, ast.Attribute) \ and node.value.attr == "proxy" \ and isinstance(node.value.value, ast.Attribute) \ and isinstance(node.value.value.value, ast.Name) \ - and node.value.value.value.id == "projection_seams": + and node.value.value.value.id in SEAM_MODULES: bound |= {t.id for t in node.targets if isinstance(t, ast.Name)} return bound @@ -1783,7 +1791,9 @@ def test_no_proxy_over_a_seam_is_read_at_import_time() -> None: if names: found[path.relative_to(PACKAGE).as_posix()] = ( sorted(names), _import_time_reads(tree, names)) - assert found == {"serve.py": (["registry_mod"], []), + assert found == {"branch_session.py": (["gate_console"], []), + "cli.py": (["gate_mod"], []), + "serve.py": (["registry_mod"], []), "serve_workbench.py": (["registry_mod"], [])}, found From 65551e6fded3eb9acc1b00ee57032edd3666354d Mon Sep 17 00:00:00 2001 From: Brett Heap <1513478+brettheap@users.noreply.github.com> Date: Fri, 2 Oct 2026 22:42:04 +0000 Subject: [PATCH 35/88] T084: a tile's own documents are editable in openDox's default scope RULED by Brett Heap, openxFactory#656 comment 5961651355, "Tile's own documents editable (Recommended)". This supersedes the holder's read-only reading of R1Q10 (a). Under the read-only default, openDox's turn guard (`doxbench_turns._require_in_scope_and_editable`, FR-015) refused every turn a lone openDox ran as "readable but not editable". The default now marks each section a tile projects as owned: a group's members, a selection's files and a candidate's claiming groups' members. This mirrors how openXdox's authority marks its owned sections. `default_columns.editable_paths(sections)`, the ONE named function, takes the owned sections' resolved rows once each, in order. Nothing outside the tile is editable, an unresolved row is not, and neither is a created path (the default records none). Tests: - tests/test_neutral_turn_scope.py (new), over a composed host with the released validators and the port declared over a `model-binding add` binding: - a turn over the tile's own document passes the guard and reaches the model step (`model_unavailable`), with nothing spawned or contacted; - a document outside the tile is still refused (`turn_scope_refused`); - Save is still the gate's: `POST /actions/gate/first-edit` answers `unknown_action`, and the governed record a Save writes is refused naming `opendox.column_seams.gate`; - the editable set is exactly the group's two members. - tests/test_column_seams.py: each tile's own documents are editable, nothing outside is, an unresolved row and a created path are not, and the owned sections' rows are the set. Before (read-only default): 9 failed. The own-document turn answered 403 `turn_scope_refused`. Mutants killed: - widen the editable set to every corpus document: 7 failed; - empty the set: 9 failed; - widen the tile itself to the corpus: 2 failed, including the outside-tile turn. Lane: openxfactory-4 (openXfactory-4-openDox_extraction) Co-Authored-By: Claude Opus 5.5 (1M context) --- src/opendox/default_columns.py | 81 ++++++----- tests/test_column_seams.py | 48 ++++++- tests/test_neutral_turn_scope.py | 222 +++++++++++++++++++++++++++++++ 3 files changed, 317 insertions(+), 34 deletions(-) create mode 100644 tests/test_neutral_turn_scope.py diff --git a/src/opendox/default_columns.py b/src/opendox/default_columns.py index 9e9d6c84..2654c31c 100644 --- a/src/opendox/default_columns.py +++ b/src/opendox/default_columns.py @@ -29,11 +29,15 @@ openxFactory's pinned schema declares, and openDox writes no governed shape it does not own. With no host's gate registered they refuse, naming the seam and the call that registers one (4.2), as `GateRecordsNotRegistered`. -* `SCOPE`, the doxBench scope authority, READ-ONLY. `resolve_scope` projects a - tile of the neutral snapshot (a group's members, a selection's files, a - candidate's claiming groups' members), each path confined to the selected - root by the registry seam's own `resolve_within`, and classifies nothing as - editable. `is_live_session_ref` keeps the governed authority's logic, which is +* `SCOPE`, the doxBench scope authority. `resolve_scope` projects a tile of the + neutral snapshot (a group's members, a selection's files, a candidate's + claiming groups' members), each path confined to the selected root by the + registry seam's own `resolve_within`. A TILE'S OWN DOCUMENTS ARE EDITABLE + (RULED by Brett Heap, openxFactory#656 comment `5961651355`, "Tile's own + documents editable (Recommended)", superseding the holder's read-only + reading): those are exactly the three sections above, as openXdox's own + authority marks its owned sections, and the set is one named function, + `editable_paths`. `is_live_session_ref` keeps the governed authority's logic, which is openDox's own session layer (`branch_session.live_session_branches`). `session_created_paths_for_scope` answers no path: which documents a session CREATED is known only from governed gate-action records, which this default @@ -253,7 +257,7 @@ def build(worktree): # ========================================================================== -# SCOPE — a small, neutral, READ-ONLY projection of a tile +# SCOPE — a small, neutral projection of a tile, its own documents editable # ========================================================================== def _mapping(value: Any) -> Mapping[str, Any]: @@ -285,7 +289,7 @@ def _canonical(path: Any) -> str: def _section(key: str, label: str, note: str, paths: Iterable[Any], *, known: set[str], seen: set[str], root: Path, - inherited: bool) -> ScopeSection: + inherited: bool, owned: bool) -> ScopeSection: from opendox import projection_seams resolve_within = projection_seams.registry.current().resolve_within @@ -298,34 +302,49 @@ def _section(key: str, label: str, note: str, paths: Iterable[Any], *, resolved = path in known and resolve_within(root, path) is not None rows.append(ScopeDocument(id=path, path=path, resolved=resolved)) return ScopeSection(key=key, label=label, note=note, inherited=inherited, - owned=False, documents=tuple(rows)) - - -def editable_paths(sections: Sequence[ScopeSection], - context_paths: Sequence[str]) -> tuple[str, ...]: - """The paths a projected tile lets a turn edit: NONE. openDox's default - scope is READ-ONLY (the holder's reading of R1Q10 (a) for T084, put to - Brett on openxFactory#656's thread, 2026-10-02), so a tile grants no edit - authority, and openDox's own turn guard, which requires a turn's paths to - be in scope AND editable (`doxbench_turns._require_in_scope_and_editable`), - discloses none of them. It is ONE named function so that a ruling either way - is one change here: the readable paths a tile's own sections carry are - `context_paths`.""" - return () + owned=owned, documents=tuple(rows)) + + +def editable_paths(sections: Sequence[ScopeSection]) -> tuple[str, ...]: + """The paths a projected tile lets a turn edit: THE TILE'S OWN DOCUMENTS. + + RULED by Brett Heap, openxFactory#656 comment `5961651355`, "Tile's own + documents editable (Recommended)", which supersedes the holder's read-only + reading of R1Q10 (a). A tile's own documents are the resolved rows of its + OWNED sections, in order and once each, as openXdox's own authority derives + its editable set from its owned sections. In openDox's default every + section a tile projects is its own: a group's members, a selection's files + and a candidate's claiming groups' members. Nothing outside them is + editable, a row that does not resolve is not, and neither is a created path + (this default records none). openDox's turn guard requires a turn's paths + to be in scope AND editable (`doxbench_turns._require_in_scope_and_editable`), + so a turn over a tile's own document passes it, and one over any other + document is still refused. ONE named function, so the set is decided in + one place.""" + editable: list[str] = [] + for section in sections: + if not section.owned: + continue + for row in section.documents: + if row.resolved and row.path not in editable: + editable.append(row.path) + return tuple(editable) def resolve_scope(snapshot: Mapping[str, Any], key: ScopeKey, *, source_root: Path, created_paths: Iterable[str] = () ) -> ScopeProjection | None: - """One tile of the neutral snapshot, read-only, or None where the snapshot - has no such tile. + """One tile of the neutral snapshot, or None where the snapshot has no + such tile. A group (`cluster`) projects its members, a selection (`staged`) its files, and a candidate (`possible`) the members of the groups that claim it. Each path is confined to `source_root` by the registry seam's own - `resolve_within`, and NOTHING is editable (`editable_paths`), so - `active_document_candidates` is empty too, and there is no outline. A - `created_paths` entry is confined and readable, never editable.""" + `resolve_within`. Each of those sections is the tile's OWN, so its + resolved documents are editable (`editable_paths`, RULED `5961651355`), + `active_document_candidates` are the documents the turn guard would + accept, and there is no outline. A `created_paths` entry is confined and + readable, never editable.""" if not isinstance(snapshot, Mapping): return None if isinstance(created_paths, (str, bytes, bytearray)): @@ -357,7 +376,8 @@ def members(group: Mapping[str, Any]) -> list[Any]: keywords = tuple(_text(t) for t in _sequence(group.get("topics")) if _text(t)) sections.append(_section( "members", "group documents", "the group's own document edges", - members(group), known=known, seen=seen, root=root, inherited=False)) + members(group), known=known, seen=seen, root=root, inherited=False, + owned=True)) elif key.tile_kind == "staged": selection = next((_mapping(s) for s in _sequence(snapshot.get("staged_topics")) if _text(_mapping(s).get("staging_id")) == key.tile_id), None) @@ -367,7 +387,7 @@ def members(group: Mapping[str, Any]) -> list[Any]: sections.append(_section( "files", "selection files", "the documents this selection names", _sequence(selection.get("files")), known=known, seen=seen, root=root, - inherited=False)) + inherited=False, owned=True)) elif key.tile_kind == "possible": candidate = next((_mapping(p) for p in _sequence(snapshot.get("possibles")) if _text(_mapping(p).get("id")) == key.tile_id), None) @@ -379,7 +399,8 @@ def members(group: Mapping[str, Any]) -> list[Any]: sections.append(_section( "claiming", "documents of the claiming groups", "membership inferred from the groups that claim this candidate", - claimed, known=known, seen=seen, root=root, inherited=True)) + claimed, known=known, seen=seen, root=root, inherited=True, + owned=True)) else: return None context = [row.path for section in sections for row in section.documents @@ -389,7 +410,7 @@ def members(group: Mapping[str, Any]) -> list[Any]: if path not in context: context.append(path) revision = _text(_mapping(snapshot.get("generation")).get("source_revision")) - editable = editable_paths(sections, context) + editable = editable_paths(sections) return ScopeProjection( key=key, title=title, keywords=keywords, source_revision=revision, sections=tuple(sections), context_paths=tuple(context), diff --git a/tests/test_column_seams.py b/tests/test_column_seams.py index f47c3ca0..6b47313d 100644 --- a/tests/test_column_seams.py +++ b/tests/test_column_seams.py @@ -270,15 +270,28 @@ def _key(kind: str, tile: str) -> ScopeKey: ("staged", "s1", ("sel.md",), "s1"), ("possible", "p1", ("a.md", "b.md", "c.md"), "A candidate"), ]) -def test_each_tile_projects_read_only(corpus, kind, tile, context, title) -> None: +def test_each_tile_projects_its_own_documents_editable( + corpus, kind, tile, context, title) -> None: + """RULED `5961651355` ("Tile's own documents editable"): every section a + tile projects is its own, so its resolved documents are both readable and + editable, and the candidates are what the turn guard would accept.""" projection = dc.resolve_scope(_snapshot(), _key(kind, tile), source_root=corpus) assert projection.title == title assert projection.context_paths == context - assert projection.editable_paths == () - assert projection.active_document_candidates == () + assert projection.editable_paths == context + assert projection.active_document_candidates == context assert projection.outline_path is None assert projection.source_revision == "abc123" - assert all(not section.owned for section in projection.sections) + assert all(section.owned for section in projection.sections) + + +def test_nothing_outside_the_tile_is_editable(corpus) -> None: + """The group's own two, never the corpus's other documents.""" + projection = dc.resolve_scope(_snapshot(), _key("cluster", "g1"), source_root=corpus) + assert set(projection.editable_paths) == {"a.md", "b.md"} + for other in ("c.md", "sel.md"): + assert other not in projection.editable_paths + assert other not in projection.context_paths def test_a_listed_document_missing_from_the_tree_is_not_resolved(corpus) -> None: @@ -286,6 +299,33 @@ def test_a_listed_document_missing_from_the_tree_is_not_resolved(corpus) -> None rows = {row.path: row.resolved for row in projection.sections[0].documents} assert rows == {"c.md": True, "gone.md": False} assert projection.context_paths == ("c.md",) + assert projection.editable_paths == ("c.md",), "an unresolved row is not editable" + + +def test_a_created_path_is_readable_and_never_editable(corpus) -> None: + projection = dc.resolve_scope(_snapshot(), _key("cluster", "g1"), + source_root=corpus, created_paths=["new.md"]) + assert projection.context_paths == ("a.md", "b.md", "new.md") + assert projection.editable_paths == ("a.md", "b.md") + + +def test_the_editable_set_is_the_owned_sections_resolved_rows() -> None: + """`editable_paths` itself: owned sections only, resolved rows only, once + each and in order.""" + from opendox.doxbench_scope_types import ScopeDocument, ScopeSection + + def section(owned, *rows): + return ScopeSection( + key="k", label="l", note="n", inherited=False, owned=owned, + documents=tuple(ScopeDocument(id=p, path=p, resolved=r) + for p, r in rows)) + + assert dc.editable_paths([section(False, ("a.md", True))]) == () + assert dc.editable_paths([ + section(True, ("a.md", True), ("gone.md", False)), + section(False, ("b.md", True)), + section(True, ("c.md", True), ("a.md", True)), + ]) == ("a.md", "c.md") def test_an_unknown_tile_is_none(corpus) -> None: diff --git a/tests/test_neutral_turn_scope.py b/tests/test_neutral_turn_scope.py new file mode 100644 index 00000000..7d823e6c --- /dev/null +++ b/tests/test_neutral_turn_scope.py @@ -0,0 +1,222 @@ +"""A tile's OWN documents are editable in openDox's default scope, and nothing +else is (plan 034 T084; RULED by Brett Heap, openxFactory#656 comment +`5961651355`, "Tile's own documents editable (Recommended)", which supersedes +the holder's read-only reading of R1Q10 (a)). + +WHY IT MATTERS. openDox's turn guard (`doxbench_turns._require_in_scope_and_ +editable`, FR-015) refuses a turn whose buffer names a path that is in scope +but not editable. Under a read-only default, every turn a lone openDox ran was +refused "readable but not editable", so its chat could never answer. The +default now marks the tile's own sections editable, which are a group's +members, a selection's files and a candidate's claiming groups' members, as +openXdox's authority marks its owned sections. The set is ONE named function, +`default_columns.editable_paths`. + +THE CASES run over a composed host in process: the plain fixture in a fresh +repository, a loopback bind, an authenticated actor, the binding +`opendox model-binding add` declares, and the port the entry points declare +over it (`doxbench_install.declared_model_port_factory`). The host also has +the released validators, as a plane with a readable contract has them. A +standalone `python -m opendox.serve` has none until T085's defaults land +(openDox-code#71), so its turn stops at the validators step, before scope. +`tests/test_capability_honesty.py` holds that standalone turn to a structured +answer with a binding configured. + +1. A turn whose buffer names a document of the tile passes the guard and + reaches the model step. It asks for a model the catalog does not carry, so + step 7 answers `model_unavailable`, and nothing is spawned or contacted. +2. A turn naming a corpus document OUTSIDE the tile is still refused at the + guard (`turn_scope_refused`). +3. Save is still refused. "Editable" is the scope's word, and it grants no + write. Writing is the Save gate's, a host's gate route: this host + contributes none, so `POST /actions/gate/first-edit` answers + `unknown_action`. And the record a Save writes is a governed gate-action + record, which the gate seam's default refuses by name. + +A CREATED FILE: no carve-manifest row (RULED OQ-C). +""" + +from __future__ import annotations + +import http.client +import json +import threading +from pathlib import Path + +import pytest + +from standalone_child import fresh_repository, git, run_module + +ROOT = Path(__file__).resolve().parent.parent +PLAIN = ROOT / "tests" / "fixtures" / "plain-documents" +WEB = ROOT / "src" / "opendox" / "web" + +#: The plain fixture's group of two, and a corpus document outside it. +TILE = {"repository": "fixture", "ref": "main", "tile_kind": "cluster", + "tile_id": "barrel-rain"} +OWN = "notes-rain-barrel-leak.md" +OUTSIDE = "notes-toolshed-inventory.md" + +#: The binding `opendox model-binding add` declares. Its broker is never run. +BINDING = ["--id", "scope-binding", "--label", "Scope binding", + "--provider", "scope-provider", + "--credential-ref", "opref-4f2a91c07be3d5a8140b6e77", + "--auth-kind", "api_key", + "--credential-approver", "fixture@example.invalid", + "--endpoint", "https://provider.invalid/turn", + "--dialect", "xfactory-prompt-v1", + "--", "scope-broker", "--home", "/srv/{binding_id}"] + + +class _Conforms: + @staticmethod + def iter_errors(_instance): + return iter(()) + + +class _EveryKind(dict): + """The released validators, as a plane that can read its contract has + them: one for every kind, and every instance conforms.""" + + def get(self, _kind, _default=None): + return _Conforms() + + +def _call(base, method, path, *, body=None, token=None): + from opendox import serve + connection = http.client.HTTPConnection(*base, timeout=30) + try: + headers = {"Content-Type": "application/json"} + if token: + headers[serve.CONSOLE_TOKEN_HEADER] = token + connection.request(method, path, body=body, headers=headers) + response = connection.getresponse() + raw = response.read().decode("utf-8", errors="replace") + try: + parsed = json.loads(raw) if raw else {} + except ValueError: + parsed = {} + return response.status, parsed if isinstance(parsed, dict) else {}, raw + finally: + connection.close() + + +def _turn(repo: Path, document: str) -> bytes: + """A well-formed v2 turn over `TILE`, bound to `document`, whose buffer + carries the document's committed text with its true content identity.""" + from opendox import doxbench_hash + from opendox.serve_wire import DOXBENCH_CHAT_TURN_V2_KIND + + def buffer(kind, path, content): + identity = doxbench_hash.content_identity(content, max_bytes=None).hex + return {"kind": kind, "repository": "fixture", "path": path, + "base_ref": "main", "base_revision": "0" * 40, + "base_hash": identity, "content_hash": identity, + "content": content, "dirty": False} + + text = (repo / document).read_text(encoding="utf-8") + return json.dumps({ + "schema_version": 1, "kind": DOXBENCH_CHAT_TURN_V2_KIND, + "client_turn_id": f"scope-{document}", "scope": TILE, + "working_subject": "", "message": "What does this note claim?", + "model_id": "a-model-the-catalog-does-not-carry", "transcript": [], + "bound_buffer": document, + "buffers": [buffer("outline", None, "# outline\n"), + buffer("document", document, text)], + }).encode("utf-8") + + +@pytest.fixture() +def host(tmp_path): + """The composed host the module docstring describes, its actor `brett` + one of the suite's declared principals + (`session_fixtures.declared_gate_principals`). Yields + `(base, capabilities, repo)`.""" + from opendox import doxbench_install, serve + + repo = fresh_repository(PLAIN, tmp_path) + git(repo, "config", "user.name", "fixture") + git(repo, "config", "user.email", "fixture@example.invalid") + added, status = run_module(tmp_path, "opendox.cli", "model-binding", "add", + "--repo-root", str(repo), *BINDING) + assert status == 0, added.stderr_text() + out = tmp_path / "out" / "snapshot.json" + generated, status = run_module( + tmp_path, "opendox.cli", "generate", "--repo-root", str(repo), + "--repository", "fixture", "--output", str(out), "--no-validate") + assert status == 0, generated.stderr_text() + httpd = serve.build_server( + WEB, out, repo, port=0, actor="brett", + schema_validator_factory=_EveryKind, + model_port_factory=doxbench_install.declared_model_port_factory( + doxbench_install.session_root_beside(out), checkout_root=repo)) + worker = threading.Thread(target=httpd.serve_forever, daemon=True) + worker.start() + try: + base = httpd.server_address[:2] + status, caps, raw = _call(base, "GET", "/capabilities") + assert status == 200, raw + assert caps["actions"]["session"] is True, caps + yield base, caps, repo + finally: + httpd.shutdown() + httpd.server_close() + worker.join(timeout=10) + + +def test_a_turn_over_the_tiles_own_document_reaches_the_model_step(host) -> None: + from opendox.serve_wire import DOXBENCH_ERR_MODEL_UNAVAILABLE + base, caps, repo = host + status, body, raw = _call(base, "POST", "/actions/workbench/chat-turn", + body=_turn(repo, OWN), + token=caps["console_token"]) + # past the guard (no "readable but not editable"), past identity, and + # answered by the model step: the catalog carries no such model + assert body.get("error") == DOXBENCH_ERR_MODEL_UNAVAILABLE, (status, raw) + assert body.get("client_turn_id") == f"scope-{OWN}", body + + +def test_a_turn_over_a_document_outside_the_tile_is_refused(host) -> None: + from opendox.serve_wire import DOXBENCH_ERR_TURN_SCOPE_REFUSED + base, caps, repo = host + assert (repo / OUTSIDE).is_file() + status, body, raw = _call(base, "POST", "/actions/workbench/chat-turn", + body=_turn(repo, OUTSIDE), + token=caps["console_token"]) + assert body.get("error") == DOXBENCH_ERR_TURN_SCOPE_REFUSED, (status, raw) + + +def test_save_is_still_the_gates_and_refused_by_name(host) -> None: + """Editable is not saveable: no Save route answers without a host's gate, + and the governed record a Save writes is refused naming the seam.""" + from opendox import column_seams + from opendox.default_columns import GateRecordsNotRegistered + base, caps, _repo = host + assert caps["actions"]["gate"] is False, caps + status, body, raw = _call(base, "POST", "/actions/gate/first-edit", + body=b"{}", token=caps["console_token"]) + assert status == 404 and body.get("error") == "unknown_action", raw + gate = column_seams.gate.current() + with pytest.raises(GateRecordsNotRegistered) as refused: + gate.build_gate_action_record( + actor="brett", action=gate.ACTION_EDIT_DOCUMENT, + at="2026-10-02T00:00:00Z", provenance=gate.HTTP_CONSOLE_TOKEN) + assert "opendox.column_seams.gate" in str(refused.value) + assert "opendox.column_seams.gate.register(" in str(refused.value) + + +def test_the_tiles_own_documents_are_exactly_the_editable_set(host) -> None: + """The projection the guard reads, over the same snapshot: the group's + two members, and nothing else of the corpus's eight.""" + from opendox import column_seams + from opendox.doxbench_scope_types import ScopeKey + base, _caps, repo = host + snapshot = json.loads((repo.parent / "out" / "snapshot.json") + .read_text(encoding="utf-8")) + projection = column_seams.scope.current().resolve_scope( + snapshot, ScopeKey(**TILE), source_root=repo) + own = ("notes-rain-barrel-leak.md", "notes-rain-barrel-overflow.md") + assert projection.context_paths == own + assert projection.editable_paths == own + assert projection.active_document_candidates == own + assert OUTSIDE not in projection.editable_paths From 1fab55f252c987e07ceb3b32fa71a224787fb64c Mon Sep 17 00:00:00 2001 From: Brett Heap <1513478+brettheap@users.noreply.github.com> Date: Fri, 2 Oct 2026 22:44:42 +0000 Subject: [PATCH 36/88] T084: pin the static bundle's content types ahead of the platform's table This is the holder's addition to T084, which is serve.py's last phase-3 writer. It comes from T075's finding on openDox-code#73 and realizes #1144 10.2, "reachable in a browser from an openDox-only install". The static route is SimpleHTTPRequestHandler's. Its `guess_type` reads the handler's `extensions_map` first and the platform's `mimetypes` table only for an extension that map lacks. On Linux and in CI the table answers `text/javascript` for `.js`. A host whose table differs can serve an ES module as `text/plain`, and a browser then refuses to run it. Windows reads its table from the registry. `serve.STATIC_CONTENT_TYPES` pins every extension the wheel's bundle carries, measured from a built wheel: 41 files under `opendox/web/`, 39 `.js`, one `.html` and one `.css`. It also pins the types of the bundle's other kinds of file: `.mjs`, `.json`, `.svg`, `.png`, `.ico` and `.woff2`. Each value is the standard library's built-in one. `.woff2`, which that table lacks, takes its registered type (RFC 8081). So a host whose table was already right serves what it served before. `DashboardHandler.extensions_map` carries the pin beside the stdlib's own compression entries, and any other extension still falls back to the platform table. tests/test_static_content_types.py (new): - under a hostile table, every bundle file keeps its pinned type; - an extension outside the pin still reads the platform table; - the pinned types are the stdlib's built-in ones; - every extension the shipped bundle carries is pinned. Mutant killed: drop the pin, 2 failed. Under the hostile table `index.html` and every `.js` were served `text/plain`. Lane: openxfactory-4 (openXfactory-4-openDox_extraction) Co-Authored-By: Claude Opus 5.5 (1M context) --- src/opendox/serve.py | 33 +++++++ tests/test_static_content_types.py | 150 +++++++++++++++++++++++++++++ 2 files changed, 183 insertions(+) create mode 100644 tests/test_static_content_types.py diff --git a/src/opendox/serve.py b/src/opendox/serve.py index ff9ef498..7a0fa439 100644 --- a/src/opendox/serve.py +++ b/src/opendox/serve.py @@ -465,6 +465,34 @@ ACTIONS_GATE_PREFIX = "/actions/gate/" ACTIONS_REFRESH_ROUTE = "/actions/refresh" +# THE STATIC BUNDLE'S CONTENT TYPES, PINNED (plan 034 T084, the holder's +# addition for #1144 10.2, "reachable in a browser from an openDox-only +# install", from T075's finding on openDox-code#73). The static route is +# `SimpleHTTPRequestHandler`'s, whose `guess_type` reads the handler's +# `extensions_map` FIRST and the platform's `mimetypes` table only for an +# extension that map lacks. The platform table is the host's: on Linux and +# in CI it answers `text/javascript` for `.js`, but a host whose table +# differs, and Windows reads its table from the registry, can serve an ES +# module as `text/plain`, which a browser refuses to run, so the console +# opens blank. Every extension the wheel's bundle carries is pinned here +# (measured at T084: 41 files under `opendox/web/`, 39 `.js`, one `.html` and +# one `.css`), with the types the bundle's own kinds of file take beside +# them. Each value is the one the standard library's built-in table gives +# (`.woff2`, which it lacks, takes its registered type, RFC 8081), so a host +# whose table was already right serves exactly what it served before. Any +# other extension still falls back to the platform table. +STATIC_CONTENT_TYPES: dict[str, str] = { + ".html": "text/html", + ".js": "text/javascript", + ".mjs": "text/javascript", + ".css": "text/css", + ".json": "application/json", + ".svg": "image/svg+xml", + ".png": "image/png", + ".ico": "image/vnd.microsoft.icon", + ".woff2": "font/woff2", +} + def answers_a_gate_verb(binding) -> bool: """Whether a contributed route binding answers `POST /actions/gate/` @@ -882,6 +910,11 @@ class DashboardHandler(serve_workbench.WorkbenchRoutes, # waits, and a healthy local client is orders of magnitude faster. timeout = 30 + # The static bundle's types, pinned ahead of the platform's table (see + # `STATIC_CONTENT_TYPES`). The stdlib's own compression entries stay. + extensions_map = {**http.server.SimpleHTTPRequestHandler.extensions_map, + **STATIC_CONTENT_TYPES} + checkout_root: Path = Path(".") snapshot_path: Path = Path("snapshot.json") snapshot_route: str = SNAPSHOT_ROUTE diff --git a/tests/test_static_content_types.py b/tests/test_static_content_types.py new file mode 100644 index 00000000..ff79ee27 --- /dev/null +++ b/tests/test_static_content_types.py @@ -0,0 +1,150 @@ +"""The static bundle is served with its own content types, whatever the +host's `mimetypes` table says (plan 034 T084, the holder's addition for #1144 +10.2, "reachable in a browser from an openDox-only install"; T075's finding on +openDox-code#73). + +`SimpleHTTPRequestHandler.guess_type` reads the handler's `extensions_map` +first and the platform's `mimetypes` table only for an extension that map +lacks. A host whose table maps `.js` to `text/plain` would serve every ES +module of the console as text, and a browser refuses to run a module served +so. Windows reads its table from the registry, which is how such a host +arises. `serve.STATIC_CONTENT_TYPES` pins the bundle's extensions. + +1. With a HOSTILE table (every guess `text/plain`), every file of the bundle + is served with its pinned type, the same type a sound table gives. +2. An extension the pin does not carry still falls back to the platform's + table: the pin narrows nothing else. +3. Every extension the shipped bundle carries is pinned, so a file of a new + kind added to `src/opendox/web/` fails here until it is. + +A CREATED FILE: no carve-manifest row (RULED OQ-C). +""" + +from __future__ import annotations + +import http.client +import mimetypes +import shutil +import threading +from pathlib import Path + +import pytest + +from standalone_child import fresh_repository, run_module + +ROOT = Path(__file__).resolve().parent.parent +PLAIN = ROOT / "tests" / "fixtures" / "plain-documents" +WEB = ROOT / "src" / "opendox" / "web" + +#: Files of the tree that the wheel does not ship (`pyproject.toml`'s +#: package data ships `web/**`, and setuptools leaves dotfiles out). +UNSHIPPED = {".gitkeep"} + + +def _bundle() -> list[Path]: + return sorted(p for p in WEB.rglob("*") + if p.is_file() and p.name not in UNSHIPPED) + + +def _hostile(monkeypatch) -> None: + """A platform table that answers `text/plain` for everything.""" + monkeypatch.setattr(mimetypes, "guess_type", + lambda *_a, **_k: ("text/plain", None)) + + +def _content_type(base, path: str) -> tuple[int, str]: + connection = http.client.HTTPConnection(*base, timeout=30) + try: + connection.request("GET", path) + response = connection.getresponse() + response.read() + return response.status, response.getheader("Content-Type") or "" + finally: + connection.close() + + +@pytest.fixture() +def serve_dir(tmp_path): + """`serve(web_dir)`: a server over `web_dir` and the plain fixture's + snapshot, on a thread. Yields the callable; returns `(host, port)`.""" + from opendox import serve + + repo = fresh_repository(PLAIN, tmp_path) + out = tmp_path / "out" / "snapshot.json" + child, status = run_module( + tmp_path, "opendox.cli", "generate", "--repo-root", str(repo), + "--repository", "fixture", "--output", str(out), "--no-validate") + assert status == 0, child.stderr_text() + servers = [] + + def start(web_dir: Path): + httpd = serve.build_server(web_dir, out, repo, port=0) + worker = threading.Thread(target=httpd.serve_forever, daemon=True) + worker.start() + servers.append((httpd, worker)) + return httpd.server_address[:2] + + try: + yield start + finally: + for httpd, worker in servers: + httpd.shutdown() + httpd.server_close() + worker.join(timeout=10) + + +def test_the_bundle_keeps_its_types_under_a_hostile_table( + serve_dir, monkeypatch) -> None: + from opendox import serve + _hostile(monkeypatch) + assert mimetypes.guess_type("app.js")[0] == "text/plain" # it is hostile + base = serve_dir(WEB) + served = {} + for path in _bundle(): + status, ctype = _content_type(base, "/" + path.relative_to(WEB).as_posix()) + assert status == 200, path + served[path.relative_to(WEB).as_posix()] = ctype + expected = {name: serve.STATIC_CONTENT_TYPES[Path(name).suffix] + for name in served} + assert served == expected + assert served["index.html"] == "text/html" + assert {served[n] for n in served if n.endswith(".js")} == {"text/javascript"} + + +#: The one pinned extension the standard library's built-in table lacks, +#: with its registered type (RFC 8081). +NOT_BUILT_IN = {".woff2": "font/woff2"} + + +def test_the_pinned_types_are_the_standard_librarys_own() -> None: + """Pinning moves nothing on a host whose table was already right: each + pinned type is the one the standard library's BUILT-IN table gives (a + fresh `MimeTypes()`, which reads no system file and no registry).""" + from opendox import serve + built_in = mimetypes.MimeTypes() + for ext, ctype in serve.STATIC_CONTENT_TYPES.items(): + expected = NOT_BUILT_IN.get(ext) or built_in.guess_type("file" + ext)[0] + assert ctype == expected, ext + + +def test_an_extension_outside_the_pin_still_reads_the_platform_table( + serve_dir, monkeypatch, tmp_path) -> None: + web = tmp_path / "web" + shutil.copytree(WEB, web) + (web / "notes.txt").write_text("plain\n", encoding="utf-8") + _hostile(monkeypatch) + base = serve_dir(web) + assert _content_type(base, "/notes.txt") == (200, "text/plain") + monkeypatch.setattr(mimetypes, "guess_type", + lambda *_a, **_k: ("text/x-from-the-table", None)) + assert _content_type(base, "/notes.txt") == (200, "text/x-from-the-table") + assert _content_type(base, "/index.html") == (200, "text/html") + + +def test_every_extension_the_bundle_ships_is_pinned() -> None: + from opendox import serve + shipped = {path.suffix for path in _bundle()} + assert shipped, "the bundle is empty" + assert shipped <= set(serve.STATIC_CONTENT_TYPES), ( + f"the bundle ships {sorted(shipped - set(serve.STATIC_CONTENT_TYPES))}, " + "which serve.STATIC_CONTENT_TYPES does not pin") From a9854078c0891d58e5483cae6f515077a9829987 Mon Sep 17 00:00:00 2001 From: Brett Heap <1513478+brettheap@users.noreply.github.com> Date: Fri, 2 Oct 2026 22:46:08 +0000 Subject: [PATCH 37/88] T072: the in-process local cases give themselves a short state directory The doxBench entrypoint fixture and test_projection_seams.py's empty-option case run `generate-and-open --local` in process, with the bundled server stood in. They scrubbed the runtime settings, and so they fell back to the runner's default state directory, under XDG_STATE_HOME or the home directory. When that default is too long for a Unix socket, configuration refuses it (13.1) before the case reaches what it tests. Measured with a 90-character XDG_STATE_HOME: 4 errors and 1 failure. Each now sets OPENDOX_STATE_DIR to a fresh, short directory under /tmp and removes it after the case. Nothing is made in it, because the database is stood in. - Under that long XDG_STATE_HOME, with OPENDOX_INSTALL_MODE=local exported, both modules pass whole, and both mutants without the private state directory are killed. - In a clean environment both modules pass whole. Lane: openxfactory-4 (openXfactory-4-openDox_extraction) Co-Authored-By: Claude Opus 5.5 (1M context) --- tests/test_doxbench_entrypoint.py | 22 +++++++++++++++++----- tests/test_projection_seams.py | 19 +++++++++++++++---- 2 files changed, 32 insertions(+), 9 deletions(-) diff --git a/tests/test_doxbench_entrypoint.py b/tests/test_doxbench_entrypoint.py index ade38784..e41659f2 100644 --- a/tests/test_doxbench_entrypoint.py +++ b/tests/test_doxbench_entrypoint.py @@ -51,6 +51,7 @@ import os import shutil import subprocess +import tempfile from pathlib import Path import pytest @@ -146,6 +147,14 @@ def _capture(*args, **kwargs): # from the shell would be refused beside the local mode, by design. for name in runtime_config.SETTING_NAMES: monkeypatch.delenv(name, raising=False) + # ITS OWN SHORT STATE DIRECTORY, so the configuration never reads the + # runner's: a local install's default is under `XDG_STATE_HOME` or the + # home directory, and one too long for a Unix socket is refused (13.1) + # before the entrypoint is reached. Nothing is made in it, because the + # database is stood in below, and it is removed after the case. + state = Path(tempfile.mkdtemp(prefix="odx-e-", + dir="/tmp" if os.path.isdir("/tmp") else None)) + monkeypatch.setenv(runtime_config.PREFIX + "STATE_DIR", str(state)) # AND THE LOCAL INSTALL'S DATABASE IS STOOD IN, with a tripwire of its own # (plan 034 T072). A local `generate-and-open` starts its bundled # PostgreSQL server before it serves, and these cases are about the model @@ -182,11 +191,14 @@ def report(self): "--model-session-root", str(session_root), "--no-validate", "--no-open", "--no-serve", ]) - rc = cli_mod.cmd_generate_and_open(args, opener=lambda url: None) - assert rc == 0, "the entrypoint did not complete" - assert len(bundles) == 1, "a LOCAL entrypoint run must own one database" - assert built, "the entrypoint never reached build_server" - yield _handler_class(built[-1]), spawned, session_root + try: + rc = cli_mod.cmd_generate_and_open(args, opener=lambda url: None) + assert rc == 0, "the entrypoint did not complete" + assert len(bundles) == 1, "a LOCAL entrypoint run must own one database" + assert built, "the entrypoint never reached build_server" + yield _handler_class(built[-1]), spawned, session_root + finally: + shutil.rmtree(state, ignore_errors=True) # -------------------------------------------------------------------------- diff --git a/tests/test_projection_seams.py b/tests/test_projection_seams.py index 77536de8..01bad88b 100644 --- a/tests/test_projection_seams.py +++ b/tests/test_projection_seams.py @@ -46,6 +46,7 @@ import stat import subprocess import sys +import tempfile import textwrap import threading import types @@ -1401,10 +1402,17 @@ def test_generate_and_open_refuses_an_empty_source_option_before_its_run_dir( bundled server (plan 034 T072): a refused option costs no database start. A tripwire stands in for the server, so a regression neither starts one nor passes.""" - from opendox.runtime.config import SETTING_NAMES + from opendox.runtime.config import PREFIX, SETTING_NAMES for name in SETTING_NAMES: monkeypatch.delenv(name, raising=False) + # Its own short state directory, so the configuration never reads the + # runner's state home, whose default may be too long for a Unix socket + # (13.1) and refused before the option is reached. The tripwire below + # means nothing is made in it. + state = Path(tempfile.mkdtemp(prefix="odx-e-", + dir="/tmp" if os.path.isdir("/tmp") else None)) + monkeypatch.setenv(PREFIX + "STATE_DIR", str(state)) started: list = [] class _Tripwire: @@ -1425,9 +1433,12 @@ def stop(self) -> None: run_dir = tmp_path / "run" # `--local`: the single-user install. Since plan 034 T070 an unflagged run # is HOSTED, and its issuer refusal would come first. - rc = cli.main(["generate-and-open", "--local", "--repo-root", str(repo), - "--repository", "garden", "--run-dir", str(run_dir), - "--no-open", "--no-serve", "--possibles", ""]) + try: + rc = cli.main(["generate-and-open", "--local", "--repo-root", str(repo), + "--repository", "garden", "--run-dir", str(run_dir), + "--no-open", "--no-serve", "--possibles", ""]) + finally: + shutil.rmtree(state, ignore_errors=True) assert rc == 1 assert ("generate-and-open refused: --possibles was given an empty path" in capsys.readouterr().err) From 55f6e8af16fd025dcadecb358c1fb7e535b21850 Mon Sep 17 00:00:00 2001 From: Brett Heap <1513478+brettheap@users.noreply.github.com> Date: Fri, 2 Oct 2026 22:53:54 +0000 Subject: [PATCH 38/88] T084: retire consumer_reach; the gate and projection columns are a host's (F4.1 whole) The two late columns, `consumer_reach.LateGateRoutes` and `LateProjectionRoutes`, leave `DashboardHandler`'s bases. openXdox's `serve_gate.GateRoutes` and `serve_projection.ProjectionRoutes` are a HOST's columns, so they are composed in at build time through the handler-contribution facet, beside the route bindings that name their methods (`_handle_gate_action`, `_serve_index`) (R1Q1 (a), 5817152735). A host that contributes a binding without its column is refused at wiring, before a socket (`route_extension.resolve_handlers`). openXdox contributes both columns that way at T086. With no reach left for it to stand in for, `src/opendox/consumer_reach.py` is deleted. F4.1 whole holds. `consumer_reach.py` is absent, and the scan prints "no deferred reach names the consumer or the publisher". Tests: - tests/test_projection_seams.py: `test_the_stand_ins_module_is_retired` checks that the file is absent and that `opendox.consumer_reach` has no spec. - tests/test_source_core_arm.py now holds the handler's bases to openDox's own. `_serve_index` stays the projection column's. - tests/test_consumer_reach.py: - keeps the direction census (every neutral module imports with openXdox blocked); - retires the stand-in section; - takes `opendox.consumer_reach` out of NEUTRAL_MODULES, with a note: the module no longer exists, so it can be no regression. - Docstrings that spoke of the module as present now speak of it as retired. The whole suite, locally: 3105 passed, 177 skipped. Lane: openxfactory-4 (openXfactory-4-openDox_extraction) Co-Authored-By: Claude Opus 5.5 (1M context) --- src/opendox/column_seams.py | 5 +- src/opendox/consumer_reach.py | 342 ----------------------------- src/opendox/generator_seam.py | 3 +- src/opendox/profile_proxy.py | 28 +-- src/opendox/projection_seams.py | 2 +- src/opendox/serve.py | 42 ++-- src/opendox/view_extension.py | 5 +- tests/test_consumer_reach.py | 316 ++------------------------ tests/test_profile_registration.py | 5 +- tests/test_projection_seams.py | 17 +- tests/test_reach_sweep.py | 6 +- tests/test_source_core_arm.py | 70 +++--- 12 files changed, 115 insertions(+), 726 deletions(-) delete mode 100644 src/opendox/consumer_reach.py diff --git a/src/opendox/column_seams.py b/src/opendox/column_seams.py index ad553c1a..fd4c6f8b 100644 --- a/src/opendox/column_seams.py +++ b/src/opendox/column_seams.py @@ -49,8 +49,9 @@ `gate_records_writable()` answers whether a HOST's gate is registered, so the model-intake surface can decline to start a flow whose approval openDox's -default would refuse (plan 034 T084, a holder reading under batch G and R1Q10 -(a) that Brett may overrule). +default would refuse (plan 034 T084; RULED by Brett Heap, 2026-10-02, "Refuse +by name, hide intake (Recommended)", confirmed at openxFactory#656 comment +`5961364221`, item 1). IMPORT WEIGHT: `opendox.projection_seams` only, which is stdlib-only, so this module names no sibling in an import. `register_defaults()` imports diff --git a/src/opendox/consumer_reach.py b/src/opendox/consumer_reach.py deleted file mode 100644 index a552c2aa..00000000 --- a/src/opendox/consumer_reach.py +++ /dev/null @@ -1,342 +0,0 @@ -"""Late-bound reaches from openDox INTO its consumer, openXdox. - -WHY THIS FILE EXISTS. openDox is the NEUTRAL product and openXdox is the layer -that pins it: `contracts/opendox-pin.yaml` in the openXdox assembly root names -openDox's commit and tree digest (`split-opendox-two-layer-product` § 4.2, -RULED OQ-2), and nothing in the chain points back. A module of THIS package may -therefore never require `openxdox` to be importable. `design.md`:243 states the -standard the carve is held to in one sentence: *"What must not survive is the -direction, not the calls."* - -Thirty-two calls survived the carve pointing the wrong way — **13 at import -time** and 19 deferred, over six modules, measured at `8e9ffa62` and recorded -as a per-module ratchet in openXdox-code's `tests/test_dependency_direction.py` -(`OPENDOX_BACK_IMPORTS`). They are not a defect of the carve: the manifest's -`import rewrites` class rewrote `ideation_dashboard.` to the package that -now owns ``, and for the gate column that package IS `openxdox`. The -rewrite was correct and the direction it produced is the thing the BUILD arc -removes. - -WHAT THIS MODULE DOES. It makes a surviving reach LATE, NAMED and REFUSABLE -instead of an import-time dependency on the consumer. `import opendox.workbench` -no longer requires openXdox to be installed; the verb that actually needs the -consumer's module resolves it on first use and, when it is absent, refuses with -the layering spelled out rather than raising `ModuleNotFoundError` from an -import line a thousand lines away from the call. - -WHAT IT DELIBERATELY IS NOT. It is not the § 2.4 extension points and it does -not replace them. A route or a subcommand openXdox CONTRIBUTES travels through -`route_extension.RouteBinding` / `subcommand_extension.SubcommandExtension` — -seams this repository already declares (`serve.build_server(route_extensions=)`, -`cli.build_parser(subcommand_extensions=)`) and openXdox-code already supplies -its half of (`serve_gate.routes()`, `serve_projection.routes()`, -`cli_gate.GateSubcommands.register()`). This module is for the OTHER class: a -neutral verb of openDox's own that calls a function living in the consumer's -column. Those calls are what `design.md`:243 permits to survive; their -DIRECTION at import time is what it does not. - -It is kept narrow on purpose so it cannot grow into a facade. A name is added -here only for a reach that is already in the tree and already counted in the -ratchet, and each carries the reason it is still pointing that way. - -A CREATED FILE: it has no row in openxFactory's -`docs/opendox-carve-manifest.yaml`, because the manifest declares what LEAVES -openxFactory and never what a destination assembles (RULED OQ-C). It sits under -a declared root, so the arrival verifier is told about it explicitly with -`--allow-created src/opendox/consumer_reach.py`. -""" - -from __future__ import annotations - -import importlib -from types import ModuleType -from typing import Any - -#: The consumer package. One spelling, unlike openXdox-code's two: openXdox is -#: an installed distribution (`pyproject.toml` at openXdox-code declares -#: `opendox` as a dependency and is itself installed as `openxdox`), never a -#: directory a runner happens to put on `sys.path`. A bare-name fallback here -#: would make an unrelated top-level `gate_console` on the path answer for the -#: consumer's, which is a worse failure than the absence it would paper over. -CONSUMER_PACKAGE = "openxdox" - - -def _names_the_candidate(missing: str, dotted: str) -> bool: - """Is `missing` the candidate itself, or a package on its dotted path? - - `import_module("openxdox.gate_console")` raises - `ModuleNotFoundError(name="openxdox")` when the PACKAGE is absent and - `name="openxdox.gate_console"` when only the submodule is — both mean "the - consumer is not here". `name="jsonschema"`, raised from inside a consumer - module that DID load, does not, and must not be swallowed: catching - `ImportError` wholesale would report a broken openXdox as a layering - problem and send the reader to the wrong repository. - """ - return dotted == missing or dotted.startswith(f"{missing}.") - - -class ConsumerReachUnavailable(RuntimeError): - """A verb of openDox reached its consumer's column and it is absent. - - Raised instead of `ModuleNotFoundError` so the failure names the LAYERING - rather than a module path: openDox does not ship, pin or depend on - openXdox — the pin runs the other way — so "openXdox is not installed" is - the NORMAL state of a neutral openDox, and a caller that needs this verb is - responsible for assembling a server that has it. - """ - - -class _LateConsumerModule: - """A stand-in for a consumer module, resolved on first use. - - Attribute access — and nothing earlier — performs the import. The resolved - module is cached, so the cost is paid once and `is` identity holds across - accesses, which is what lets a caller `monkeypatch.setattr` the real module - and be seen by openDox's verbs. - """ - - __slots__ = ("_name", "_reason", "_module") - - def __init__(self, name: str, *, reason: str) -> None: - self._name = name - self._reason = reason - self._module: ModuleType | None = None - - @property - def name(self) -> str: - """The dotted name this stand-in resolves, e.g. `openxdox.snapshot`.""" - return f"{CONSUMER_PACKAGE}.{self._name}" - - def resolve(self) -> ModuleType: - """Import the consumer module, or refuse naming the layering. - - An ABSENT consumer refuses; a consumer that is PRESENT and raises while - executing re-raises untouched (`_names_the_candidate`). - """ - if self._module is not None: - return self._module - dotted = self.name - try: - self._module = importlib.import_module(dotted) - except ModuleNotFoundError as exc: - if exc.name is None or not _names_the_candidate(exc.name, dotted): - raise - raise ConsumerReachUnavailable( - f"{dotted!r} belongs to openXdox, the layer that PINS this " - f"one, and openDox does not supply it: {self._reason}. " - "openXdox pins openDox by commit and tree digest " - "(split-opendox § 4.2, RULED OQ-2) and openDox pins nothing " - "back, so a neutral openDox with no openXdox installed is the " - "normal case and this reach is the exception. THE REMEDY IS " - "TO INSTALL openXdox — and only that, today. The § 2.4 " - "extension points are NOT an alternative here and this " - "message will not offer one: `build_server(route_extensions=)` " - "contributes route bindings and " - "`build_parser(subcommand_extensions=)` contributes " - "subcommands, and neither injects a MODULE, so a caller who " - "followed them would arrive back at this same refusal. The " - "injection that would make this reach disappear — openDox " - "naming a protocol and being handed an implementation — does " - "not exist yet and is BUILD-arc work " - "(split-opendox § 3.5/3.6, § 4.3)") from exc - return self._module - - def __getattr__(self, attr: str) -> Any: - # Dunder lookups must not resolve the consumer: `copy`, `pickle`, - # `inspect` and pytest's own assertion rewriting all probe for dunders - # on arbitrary objects, and resolving openXdox because something asked - # for `__wrapped__` would make the reach fire at a moment no verb chose. - if attr.startswith("__") and attr.endswith("__"): - raise AttributeError(attr) - return getattr(self.resolve(), attr) - - def __repr__(self) -> str: - state = "resolved" if self._module is not None else "unresolved" - return f"" - - -class _LateConsumerColumn: - """The BASE-CLASS member of this family: one handler-method column of the - consumer, reached on first CALL instead of at class-definition time. - - THE SITE, AND WHY NOTHING NARROWER WOULD DO. `serve.DashboardHandler` named - `serve_gate.GateRoutes` and `serve_projection.ProjectionRoutes` as MIXIN - BASES. A base expression is evaluated when the class statement runs, which - is when the module loads, so those two lines alone made - `import opendox.serve` require openXdox — and a base, unlike a call, cannot - be deferred by the module stand-in above: a class needs its bases to exist - before its first instance does. - - So the column is replaced by a base openDox OWNS, carrying one method per - name the consumer's column defines. Each forwards - `getattr(, name)(self, *args, **kwargs)` — the SAME function - object, with the SAME `self`, which is the live request handler — so the - handler behaves exactly as it did when it inherited: a contributed binding - naming `_handle_gate_action` or `_serve_index` still resolves - against the bound class at wiring time - (`route_extension.resolve_handlers`), which is the check that refuses a - route that cannot be served BEFORE a socket. - - `design.md`:243 is the standard again: *"What must not survive is the - direction, not the calls."* The call into openXdox's column survives, and it - is the same call; what goes is the import that used to make it at load time. - - WHY THE METHOD NAMES ARE RESTATED HERE. The same reason `defaults.py` - restates three literals: the alternative is reading them off the consumer, - which is the import this removes. `LATE_COLUMN` publishes the triple - (consumer module, class, method names) so openXdox-code's - `tests/test_dependency_direction.py` can hold the two surfaces together and - refuse if either side moves without the other — the drift guard is the - invariant, the restatement is the spelling. - - NOT A GENERAL SUBCLASS PROXY. There is no `__getattr__` here, deliberately: - a handler instance is probed for absent attributes constantly (`hasattr(self, - "do_PUT")` in `http.server`'s own dispatch, `copy`, `pickle`, pytest), and a - base that answered those by importing openXdox would fire the reach at a - moment no verb chose — and, worse, would raise this module's - `ConsumerReachUnavailable` where the caller was testing for `AttributeError`. - A NAMED method list answers exactly the names the column has and nothing - else, and an absent consumer refuses at the call with the layering spelled - out. - """ - - #: `(consumer module, class, method names)`. Set on each generated subclass. - LATE_COLUMN: tuple[str, str, tuple[str, ...]] = ("", "", ()) - - -def _column_forwarder(holder: _LateConsumerModule, class_name: str, attr: str): - """One forwarding method: resolve the column's class, then call through it.""" - - def forward(self, *args: Any, **kwargs: Any) -> Any: - column = getattr(holder.resolve(), class_name) - return getattr(column, attr)(self, *args, **kwargs) - - forward.__name__ = attr - forward.__qualname__ = f"Late{class_name}.{attr}" - forward.__doc__ = ( - f"`{holder.name}.{class_name}.{attr}`, resolved on first call " - f"(consumer_reach._LateConsumerColumn).") - return forward - - -def route_column(holder: _LateConsumerModule, class_name: str, - methods: tuple[str, ...]) -> type: - """A mixin base standing in for `.`.""" - if not methods: - raise ValueError( - "a late column with no methods stands in for nothing; name the " - f"methods {holder.name}.{class_name} defines") - namespace: dict[str, Any] = { - name: _column_forwarder(holder, class_name, name) for name in methods} - namespace["LATE_COLUMN"] = (holder.name, class_name, tuple(methods)) - namespace["__doc__"] = ( - f"Late stand-in for `{holder.name}.{class_name}` as a mixin base. " - "See `consumer_reach._LateConsumerColumn`.") - return type(f"Late{class_name}", (_LateConsumerColumn,), namespace) - - -def module(name: str, *, reason: str) -> _LateConsumerModule: - """A late stand-in for `openxdox.`. `name` carries no package prefix.""" - if name.startswith(f"{CONSUMER_PACKAGE}."): - raise ValueError( - f"consumer_reach.module() takes the module name WITHOUT the " - f"{CONSUMER_PACKAGE!r} prefix; got {name!r}") - return _LateConsumerModule(name, reason=reason) - - -# -------------------------------------------------------------------------- -# The reaches this package still makes, each with the reason it still makes it -# -------------------------------------------------------------------------- - -#: The gate console — openXdox's gate-and-commission loop (`design.md` § D3, -#: the gate column). `cli`'s human-gate plumbing and `branch_session`'s session -#: refusals read `GateConsole`, `GateRefused`, `Provenance`, `PRESENCE_*`, -#: `SURFACE_CLI` and `require_human_gate` off it. The injection that removes -#: the reach altogether is § 4.3/§ 4.5 work at openXdox-code, not a direction -#: fix, so the call stays and only its direction at import time goes. -gate_console = module( - "gate_console", - reason="the gate-and-commission loop is openXdox's column (design.md § D3) " - "and openDox contributes no gate of its own") - -#: THE PROJECTION MECHANISM'S FOUR STAND-INS AND THEIR SIX NAMES ARE GONE -#: (plan 034 T055; #1144 5.5 and 4.3 in part; R1Q10 (a), openxFactory#656 -#: comment 5850003126). `snapshot`, `snapshot_registry`, `corpus_root` and -#: `generator` stood here as module stand-ins, with `find_validator`, -#: `corpus_root_refusal`, `generate_snapshot`, `is_rfc3339_datetime`, -#: `hosted_ref_refused` and the one constant stand-in, `scanned_roots`, bound -#: over them. Each was a neutral verb of openDox's reaching the consumer's -#: projection column, so a lone openDox could neither generate nor build a -#: server (plan 034, research R7). They are now read from DECLARED SEAMS, -#: `opendox.projection_seams` (the snapshot registry and source, the -#: corpus-root predicate, the writer and the validator lookup) and -#: `opendox.generator_seam` (the generate operation), each with openDox's own -#: default, which the entry points register where no host has. The consumer -#: contributes its governed mechanism through the same seams (T059). -#: `is_rfc3339_datetime` is openDox's own (`opendox.rfc3339`), and -#: `hosted_ref_refused` is `serve.py`'s own, over the registry seam's -#: `is_publishable_ref`. With no binding left for them, the callable and value -#: stand-ins (`function`, `constant`) went too. - -#: The projection's HTTP column. Named here only to carry -#: `LateProjectionRoutes` below: `serve.py` holds no other reference to it -#: since plan 034 T055. -serve_projection = module( - "serve_projection", - reason="the snapshot index route is openXdox's column (design.md § D3); " - "this core dispatches it through the § 2.4 route extension point " - "instead of inheriting it") - -#: `resolve_source_path` STOOD HERE and is `serve.py`'s own definition since -#: § 3.4 slice S6 (RULED Q4, openxFactory#656 comment 5642758731). It is the -#: single-root entry point to the `/source` pass-through's containment, and that -#: pass-through is now a FIXED CORE ARM of the neutral product — so a stand-in -#: forwarding into the layer that PINS openDox was the wrong shape for it. The -#: RULE is the snapshot registry's `resolve_within`, which the entry point -#: calls through `serve.py`'s `registry_mod`, the registry seam's proxy since -#: plan 034 T055. `notebook_action.py`:52 still imports the name `from .serve`, -#: and gets a real function rather than a forwarder. - -#: The gate console's HTTP column — the door the gate verbs are posted through. -#: Named here only to carry `LateGateRoutes` below; `serve.py` holds no other -#: reference to it. -serve_gate = module( - "serve_gate", - reason="the gate console's route column is openXdox's (design.md § D3); " - "this core dispatches its one contributed route through the § 2.4 " - "route extension point") - -#: `serve_gate.GateRoutes` as a mixin base — `DashboardHandler`'s third base -#: until slice 2b. Two methods: the contributed `POST /actions/gate/` handler -#: the § 2.4 binding names, and the failure logger it calls. -LateGateRoutes = route_column(serve_gate, "GateRoutes", - ("_handle_gate_action", "_log_gate_failure")) - -#: `serve_projection.ProjectionRoutes` as a mixin base — `DashboardHandler`'s -#: fourth base until slice 2b. ONE method since plan 034 T055: `_serve_index`, -#: which the column's own § 2.4 binding for `/snapshot-index.json` names, and -#: which T084 hands to the handler-contribution facet with the column. -#: -#: IT WAS EIGHT UNTIL § 3.4 SLICE S6 AND FIVE UNTIL T055. S6 (RULED Q4, -#: openxFactory#656 comment 5642758731) made `_keyed_source`, `_serve_source` -#: and `_refuse_bare_source` `serve.py`'s own, because the route they answer is -#: the neutral product's fixed core arm. T055 did the same for the core -#: `/snapshot.json` arm's four, `_query_key`, `_read_snapshot`, -#: `_serve_snapshot` and `_hosted_entry_refused`: forwarded here, a standalone -#: server refused every `/snapshot.json`. The rules they consult (FR-048's -#: hosted-plane confinement, the registry's per-entry resolution) are the -#: snapshot registry's, reached through its seam. -LateProjectionRoutes = route_column( - serve_projection, "ProjectionRoutes", ("_serve_index",)) - -__all__ = [ - "CONSUMER_PACKAGE", - "ConsumerReachUnavailable", - "gate_console", - "LateGateRoutes", - "LateProjectionRoutes", - "module", - "route_column", - "serve_gate", - "serve_projection", -] diff --git a/src/opendox/generator_seam.py b/src/opendox/generator_seam.py index c7edcd30..674fc0f4 100644 --- a/src/opendox/generator_seam.py +++ b/src/opendox/generator_seam.py @@ -9,7 +9,8 @@ contributes it *"through a DECLARED GENERATOR SEAM"*. That seam *"SHALL BE DECLARED BY THIS ARC, naming the operation it hands over, the registration point, and what a conformant implementation must satisfy"*. `consumer_reach.py` -names the same gap from the other side. The injection that would retire its +named the same gap from the other side (until plan 034 T084 retired it). The +injection that would retire its `generator` reach, *"openDox naming a protocol and being handed an implementation"*, *"does not exist yet and is BUILD-arc work"*. This module is that declaration (plan 034's T052). diff --git a/src/opendox/profile_proxy.py b/src/opendox/profile_proxy.py index 6b74534c..92f59019 100644 --- a/src/opendox/profile_proxy.py +++ b/src/opendox/profile_proxy.py @@ -15,17 +15,18 @@ `Dox` profile may want; not now"). This module is that proxy, and `domain_profile.py` beside it is the registration it resolves through. -THE PATTERN IS THIS REPOSITORY'S OWN. `consumer_reach.py` (§ 4.1, landed) -already defers a name to first use — `_LateConsumerModule` defers an attribute -read, `_LateConsumerValue` defers the first OPERATION on a value — and refuses -with the layering spelled out instead of raising `ModuleNotFoundError` from an -import line a thousand lines away from the call. `_LateProfile` below is the -same shape pointed at a different question. It is NOT in `consumer_reach.py`, -deliberately: that module is for reaches into `openxdox`, the package that PINS -openDox, and every name in it is counted in a ratchet that must reach zero. A -host profile is not a reach into the consumer at all — the host may be an -openxFactory, a `MedxDox`, or a test — so filing it there would corrupt the one -number `tests/test_consumer_reach.py` exists to hold. +THE PATTERN IS THIS REPOSITORY'S OWN. `consumer_reach.py` (§ 4.1; retired at +plan 034 T084, when its last reaches became declared seams) deferred a name to +first use — `_LateConsumerModule` deferred an attribute read, +`_LateConsumerValue` the first OPERATION on a value — and refused with the +layering spelled out instead of raising `ModuleNotFoundError` from an import +line a thousand lines away from the call. `_LateProfile` below is the same +shape pointed at a different question. It was NOT in `consumer_reach.py`, +deliberately: that module was for reaches into `openxdox`, the package that +PINS openDox, and every name in it was counted in a ratchet that had to reach +zero, as it did at T084. A host profile is not a reach into the consumer at +all — the host may be an openxFactory, a `MedxDox`, or a test — so filing it +there would have corrupted the one number that ratchet held. WHAT IT RESOLVES, AND WHEN. Nothing at import time. `import opendox.profile_proxy` performs no lookup, touches no registry and cannot fail @@ -166,8 +167,9 @@ def __getattr__(self, attr: str) -> Any: # because something asked for `__wrapped__` would fire the composition # point at a moment no caller chose — and would raise # `ProfileNotRegistered` where the prober was testing for - # `AttributeError`. `consumer_reach._LateConsumerModule` holds the same - # line for the same reason. + # `AttributeError`. `projection_seams._SeamProxy` holds the same line + # for the same reason (as `consumer_reach._LateConsumerModule` did, + # until plan 034 T084 retired it). if attr.startswith("__") and attr.endswith("__"): raise AttributeError(attr) # A composition point's read of the facet it composes from IS the build diff --git a/src/opendox/projection_seams.py b/src/opendox/projection_seams.py index 76638bde..11e5fdfc 100644 --- a/src/opendox/projection_seams.py +++ b/src/opendox/projection_seams.py @@ -10,7 +10,7 @@ validator (`openxdox.snapshot`). With openXdox absent, which is the normal state of a neutral openDox, each of those reaches refused, so a server could not be BUILT standalone (plan 034, research R7) and a generate verb could not write. -`consumer_reach` names the gap itself: the injection that would retire a reach, +`consumer_reach` named the gap itself: the injection that would retire a reach, *"openDox naming a protocol and being handed an implementation"*, *"does not exist yet and is BUILD-arc work"*. This module is that injection for the four. diff --git a/src/opendox/serve.py b/src/opendox/serve.py index 7a0fa439..f810e18f 100644 --- a/src/opendox/serve.py +++ b/src/opendox/serve.py @@ -148,7 +148,6 @@ # evaluated where the `def` sits, at import time. openDox owns those two # values (`defaults.py`), and openXdox-code's drift guard holds the literals # together. -from opendox import consumer_reach # noqa: E402 from opendox import defaults # noqa: E402 from opendox import projection_seams # noqa: E402 from opendox import column_seams # noqa: E402 @@ -873,22 +872,19 @@ def _head_of(checkout_root: Path, git=None) -> str | None: class DashboardHandler(serve_workbench.WorkbenchRoutes, serve_project.ProjectRoutes, - # BUILD slice 2b: these two read `serve_gate.GateRoutes` - # and `serve_projection.ProjectionRoutes` — openXdox - # classes, and a base expression is evaluated when the - # class statement runs, so these two lines alone made - # `import opendox.serve` require the layer that PINS - # openDox. The stand-ins carry the same method names and - # forward to the same functions with the same `self` on - # first call, so every contributed binding behaves - # exactly as before. Since plan 034 T055 the core - # `/snapshot.json` arm's handlers are THIS class's own - # (`_serve_snapshot` below), and the projection stand-in - # forwards one method, `_serve_index`, the one its - # contributed `/snapshot-index.json` binding names - # (T084 hands the column to the handler facet). - consumer_reach.LateGateRoutes, - consumer_reach.LateProjectionRoutes, + # Plan 034 T084 (#1144 4.3; R1Q1 (a), openxFactory#656 + # comment 5817152735): openXdox's gate and projection + # columns, `serve_gate.GateRoutes` and + # `serve_projection.ProjectionRoutes`, stood here, as + # `consumer_reach`'s late stand-ins since BUILD slice 2b + # and as the classes themselves before it. They are a + # HOST's columns, so they are composed in at build + # time through the handler-contribution facet, beside + # the route bindings that name their methods + # (`_handle_gate_action`, `_serve_index`). A host that + # contributes a binding without its column is refused + # at wiring, before a socket (`route_extension. + # resolve_handlers`). A lone openDox carries neither. # Plan 034 T011 (#1144 task 2.2): openxFactory's # `serve_openxfactory_lanes.LaneRoutes` stood here. # It is a descendant's column in a package openDox @@ -1252,12 +1248,12 @@ def do_HEAD(self): # noqa: N802 # at § 2.4 PR 3, because it tests `path == self.snapshot_route`, a # per-server keyword a frozen `RouteBinding.pattern` cannot carry, while # its handlers travelled to openXdox's projection column and were reached - # through `consumer_reach.LateProjectionRoutes`. So a standalone server - # refused every `/snapshot.json`. The four methods below are that route's - # handlers, answering from the REGISTERED snapshot source: the query key, - # the active snapshot's bytes, the route itself, and FR-048's per-entry - # hosted refusal, which `_serve_source` asks too. The rules they consult - # are the registry's, through its seam. + # through a late stand-in for it (`consumer_reach`, retired at T084). So + # a standalone server refused every `/snapshot.json`. The four methods + # below are that route's handlers, answering from the REGISTERED snapshot + # source: the query key, the active snapshot's bytes, the route itself, + # and FR-048's per-entry hosted refusal, which `_serve_source` asks too. + # The rules they consult are the registry's, through its seam. def _query_key(self) -> tuple[str | None, str | None]: """The optional `?repository=&ref=` of a read route. No repository means the ACTIVE entry, which is what a query-less request asks for.""" diff --git a/src/opendox/view_extension.py b/src/opendox/view_extension.py index e7ed67f4..a5df59d7 100644 --- a/src/opendox/view_extension.py +++ b/src/opendox/view_extension.py @@ -37,8 +37,9 @@ openXdox CONSUMES it, and openXdox already pins openDox (`openXdox-code` `af15f712`'s `opendox` pin -> `a99eba03`). A consumer importing the product it pins is the direction the split is FOR; it is the reverse — the -product reaching its consumer — that `consumer_reach.py` exists to make late, -named and refusable. So `from opendox import view_extension` is a legal downward +product reaching its consumer — that `consumer_reach.py` existed to make late, +named and refusable, and that declared seams carry since plan 034 T084 retired +it. So `from opendox import view_extension` is a legal downward import for a contributing column, and no replica is needed. THE THREE THINGS A CONTRIBUTED VIEW BRINGS WITH IT, and why each is a field diff --git a/tests/test_consumer_reach.py b/tests/test_consumer_reach.py index 874e98c3..0d0ab652 100644 --- a/tests/test_consumer_reach.py +++ b/tests/test_consumer_reach.py @@ -59,9 +59,9 @@ def _candidates(..., records_dir: str = gate_console.DEFAULT_RECORDS_DIR): import pytest -#: The package openDox must not require. Spelled once here rather than -#: imported from `opendox.consumer_reach`, because this file must hold even if -#: that module is the thing that broke. +#: The package openDox must not require. Spelled once here, and never imported +#: from the package under test, because this file must hold even if that +#: module is the thing that broke. CONSUMER_PACKAGE = "openxdox" ROOT = Path(__file__).resolve().parent.parent @@ -103,7 +103,8 @@ def _import_in_subprocess(module: str, *, consumer_blocked: bool, NEUTRAL_MODULES = ( "opendox.workbench", "opendox.serve_workbench", - "opendox.consumer_reach", + # `opendox.consumer_reach` stood here, the late seam itself, until plan + # 034 T084 deleted it: no reach is left for it to stand in for. # BUILD slice 2b. NINE default-argument reads of # `gate_console.DEFAULT_RECORDS_DIR` (:1740, :1859, :1878, :4179, :4217, # :4254, :4287, :4504, :4991) — evaluated where the `def` sits, so no @@ -521,302 +522,17 @@ def test_the_runtime_extras_modules_are_the_runtimes_own() -> None: # 2 — the seam itself, exercised at run time # -------------------------------------------------------------------------- -def test_importing_the_seam_resolves_nothing() -> None: - """Constructing a stand-in performs no import. - - The whole value of the module is that `import opendox.consumer_reach` is - free; a stand-in that resolved eagerly would be an import statement wearing - a different hat. - """ - done = _import_in_subprocess("opendox.consumer_reach", consumer_blocked=True) - assert done.returncode == 0, done.stderr - - -def test_first_attribute_access_refuses_naming_the_layering() -> None: - from opendox import consumer_reach - - absent = consumer_reach.module("no_such_column", reason="a test's own") - with pytest.raises(consumer_reach.ConsumerReachUnavailable) as caught: - absent.anything - message = str(caught.value) - assert "openxdox.no_such_column" in message - assert "RULED OQ-2" in message, ( - "the refusal must name the LAYERING — which way the pin runs — rather " - "than reading as a missing-module accident") - assert isinstance(caught.value.__cause__, ModuleNotFoundError), ( - "the original ModuleNotFoundError is chained, so a reader still gets " - "the import machinery's own account beneath the layering one") - - -def test_a_consumer_module_that_exists_and_raises_is_re_raised_untouched() -> None: - """`except ImportError` wholesale would blame the layering for a bug. - - A consumer module that IS present and fails while executing — because one - of ITS dependencies is missing — must surface as that failure, not as - `ConsumerReachUnavailable`, or the reader is sent to the wrong repository. - """ - from opendox import consumer_reach - - reach = consumer_reach.module("cheerfully_broken", reason="a test's own") - broken = ModuleNotFoundError("No module named 'jsonschema'", name="jsonschema") - - def _raise(_dotted: str): - raise broken - - original = consumer_reach.importlib.import_module - consumer_reach.importlib.import_module = _raise - try: - with pytest.raises(ModuleNotFoundError) as caught: - reach.anything - finally: - consumer_reach.importlib.import_module = original - assert caught.value is broken - assert not isinstance(caught.value, consumer_reach.ConsumerReachUnavailable) - - -def test_resolution_forwards_to_the_real_module_and_caches(tmp_path: Path) -> None: - from opendox import consumer_reach - - module_object = type(sys)("openxdox.pretend") - module_object.ANSWER = 42 - module_object.verb = lambda x: x * 2 - reach = consumer_reach.module("pretend", reason="a test's own") - sys.modules["openxdox.pretend"] = module_object - # The fake PARENT is removed again below only if this test created it. - # Leaving an empty `openxdox` package in `sys.modules` would make every - # later test in the process see an importable-but-empty consumer instead - # of normal import behaviour — including this file's own seam tests. - parent_was_created = "openxdox" not in sys.modules - if parent_was_created: - sys.modules["openxdox"] = type(sys)("openxdox") - try: - assert reach.ANSWER == 42 - assert reach.resolve() is module_object - assert reach.resolve() is module_object, "the resolved module is cached" - assert reach.verb(3) == 6, ( - "a resolved module's function is the module's own, called through") - finally: - sys.modules.pop("openxdox.pretend", None) - if parent_was_created: - sys.modules.pop("openxdox", None) - - -def test_a_dunder_lookup_does_not_resolve_the_consumer() -> None: - """`copy`, `pickle`, `inspect` and pytest all probe for dunders. - - Resolving openXdox because something asked for `__wrapped__` would fire the - reach at a moment no verb chose — and, with the consumer absent, would turn - an innocuous introspection into `ConsumerReachUnavailable`. - """ - from opendox import consumer_reach - - reach = consumer_reach.module("never_resolved", reason="a test's own") - with pytest.raises(AttributeError): - reach.__wrapped__ - assert "unresolved" in repr(reach) - - -# THE VALUE AND CALLABLE STAND-INS ARE RETIRED (plan 034 T055). `constant` -# stood for ONE site, `cli.py`'s `SCANNED_ROOTS`, and `function` for six -# callables (`find_validator`, `corpus_root_refusal`, `generate_snapshot`, -# `is_rfc3339_datetime`, `hosted_ref_refused` among them); every one of them is -# read from a declared seam now, or is openDox's own, so the four cases that -# held `_LateConsumerValue` to the tuple it stood for went with it. -# `tests/test_projection_seams.py` holds the seams that replaced them. - - -@pytest.fixture() -def pretend_column(): - """A stand-in for a consumer module carrying one handler-method COLUMN. - - The methods are written the way the real columns are — plain functions on a - class, called with the live request handler as `self` — so what the test - exercises is the forwarding contract and not a mock's idea of it. - """ - module_object = type(sys)("openxdox.pretend_routes") - - class PretendRoutes: - def _serve_thing(self, path, *, keyed=False): - # Reads state off `self`, which is the whole point: the forwarder - # must pass the HANDLER, not the column, as `self`. - return f"{self.marker}:{path}:{keyed}" - - def _refuse_thing(self): - return f"{self.marker}:refused" - - module_object.PretendRoutes = PretendRoutes - sys.modules["openxdox.pretend_routes"] = module_object - parent_was_created = "openxdox" not in sys.modules - if parent_was_created: - sys.modules["openxdox"] = type(sys)("openxdox") - try: - yield module_object - finally: - sys.modules.pop("openxdox.pretend_routes", None) - if parent_was_created: - sys.modules.pop("openxdox", None) - - -def _late_handler(consumer_reach, methods=("_serve_thing", "_refuse_thing")): - """A `DashboardHandler`-shaped class over a late column, as `serve.py` builds one.""" - column = consumer_reach.route_column( - consumer_reach.module("pretend_routes", reason="a test's own"), - "PretendRoutes", methods) - - class Handler(column): - marker = "handler" - - return column, Handler - - -def test_a_late_column_is_built_without_resolving_the_consumer() -> None: - """The class statement runs at IMPORT time — this is the whole reason the - - column member exists. A base that resolved while being built would defer - nothing: `DashboardHandler`'s bases are evaluated when `serve.py` loads. - """ - from opendox import consumer_reach - - column, Handler = _late_handler(consumer_reach) - assert Handler.marker == "handler" - assert column.LATE_COLUMN == ("openxdox.pretend_routes", "PretendRoutes", - ("_serve_thing", "_refuse_thing")), ( - "the triple openXdox-code's drift guard reads to hold the two surfaces " - "together must name the module, the class and the method list") - assert column._serve_thing.__name__ == "_serve_thing", ( - "the forwarder keeps the method's NAME, because a contributed binding " - "is resolved against the bound class BY NAME at wiring time") - - -def test_the_forwarders_call_the_consumer_with_the_handler_as_self( - pretend_column) -> None: - """The contract: same function object, same `self`, same arguments. - - `route_extension.resolve_handlers` refuses a route that cannot be served - before a socket is opened, and it resolves the handler by name against the - BOUND CLASS — so a wrong method list or a forwarding signature that dropped - an argument would leave imports green and break requests, which is exactly - what this test is here to stop. - """ - from opendox import consumer_reach - - _column, Handler = _late_handler(consumer_reach) - handler = Handler() - - assert handler._serve_thing("/a/b") == "handler:/a/b:False", ( - "positional arguments forward, and `self` is the HANDLER — the column's " - "method reads `self.marker`, which only the handler has") - assert handler._serve_thing("/a/b", keyed=True) == "handler:/a/b:True", ( - "keyword arguments forward too") - assert handler._refuse_thing() == "handler:refused" - assert handler._serve_thing.__func__ is not \ - pretend_column.PretendRoutes._serve_thing, ( - "the BOUND method is the forwarder, not the column's function") - - -def test_a_late_column_answers_only_the_names_it_was_given(pretend_column) -> None: - """No `__getattr__`, deliberately, and the absence is asserted. - - A handler instance is probed for absent attributes constantly — `http.server` - asks `hasattr(self, "do_PUT")`, and `copy`, `pickle` and pytest all probe — - so a base that answered those by importing openXdox would fire the reach at - a moment no verb chose, and would raise `ConsumerReachUnavailable` where the - caller was testing for `AttributeError`. - """ - from opendox import consumer_reach - - _column, Handler = _late_handler(consumer_reach, methods=("_serve_thing",)) - handler = Handler() - - assert handler._serve_thing("/x") == "handler:/x:False" - with pytest.raises(AttributeError): - handler.do_PUT - with pytest.raises(AttributeError): - # Present on the consumer's column, absent from the NAMED list: a name - # left out of the list is left out of the class, not silently proxied. - handler._refuse_thing - assert not hasattr(handler, "_refuse_thing") - - -def test_a_late_column_with_no_consumer_refuses_at_the_call() -> None: - """Construction succeeds, the call refuses, and the refusal names the layering.""" - from opendox import consumer_reach - - column = consumer_reach.route_column( - consumer_reach.module("no_such_column", reason="a test's own"), - "NoRoutes", ("_serve_thing",)) - - class Handler(column): - marker = "handler" - - with pytest.raises(consumer_reach.ConsumerReachUnavailable) as caught: - Handler()._serve_thing("/x") - message = str(caught.value) - assert "openxdox.no_such_column" in message - assert "RULED OQ-2" in message - assert isinstance(caught.value.__cause__, ModuleNotFoundError) - - -def test_a_column_standing_in_for_nothing_is_refused() -> None: - """An empty method list would build a base that inherits nothing and hides it.""" - from opendox import consumer_reach - - with pytest.raises(ValueError, match="name the methods"): - consumer_reach.route_column( - consumer_reach.module("pretend_routes", reason="a test's own"), - "PretendRoutes", ()) - - -def test_the_two_live_columns_name_the_methods_serve_dispatches() -> None: - """The real bindings, held against the names `serve.py` and § 2.4 rely on. - - `_serve_index` is the projection column's ONE forwarded method since plan - 034 T055: the § 2.4 binding for `/snapshot-index.json` names it. The core - `/snapshot.json` arm's handlers, which travelled to the column at § 2.4 - PR 3 while their dispatch ARM stayed core, are `serve.py`'s own again. - """ - from opendox import consumer_reach - - gate_module, gate_class, gate_methods = \ - consumer_reach.LateGateRoutes.LATE_COLUMN - assert (gate_module, gate_class) == ("openxdox.serve_gate", "GateRoutes") - assert "_handle_gate_action" in gate_methods, ( - "the route the § 2.4 gate binding declares") - - proj_module, proj_class, proj_methods = \ - consumer_reach.LateProjectionRoutes.LATE_COLUMN - assert (proj_module, proj_class) == ("openxdox.serve_projection", - "ProjectionRoutes") - assert proj_methods == ("_serve_index",), ( - "the projection column forwards the ONE method its contributed " - "`/snapshot-index.json` binding names. The core `/snapshot.json` " - "arm's handlers are serve.py's own since plan 034 T055") - # § 3.4 SLICE S6, RULED Q4 (openxFactory#656 comment 5642758731): the - # `/source` pair is `serve.py`'s own FIXED CORE ARM now, so the three - # methods that answer it must NOT be forwarded into the consumer. Asserted - # as an ABSENCE and not left as silence: a forwarder left behind here would - # be invisible — the route would keep working wherever openXdox happens to - # be installed, which is every developer machine and neither claim this - # slice makes. - for departed in ("_keyed_source", "_serve_source", "_refuse_bare_source"): - assert departed not in proj_methods, ( - f"{departed} answers /source, which RULED Q4 makes openDox's own " - "core arm; it must be defined in serve.py, not forwarded to " - "openxdox.serve_projection") - # PLAN 034 T055: the same for the core `/snapshot.json` arm. Forwarded, - # its four handlers refused every `/snapshot.json` of a standalone server. - for departed in ("_query_key", "_read_snapshot", "_serve_snapshot", - "_hosted_entry_refused"): - assert departed not in proj_methods, ( - f"{departed} answers /snapshot.json, the neutral product's own core " - "arm; since plan 034 T055 it is defined in serve.py") - - -def test_the_prefix_is_refused_rather_than_doubled() -> None: - from opendox import consumer_reach - - with pytest.raises(ValueError, match="WITHOUT"): - consumer_reach.module("openxdox.gate_console", reason="a test's own") +# RETIRED BY PLAN 034 T084 (#1144 4.3, F4.1 whole). This section exercised +# `opendox.consumer_reach`: the late module stand-in, its refusal naming the +# layering, its caching, its dunder discipline, and the late route columns +# `LateGateRoutes` and `LateProjectionRoutes` with the method names +# `serve.py` dispatched through them. Every reach it stood in for is now read +# from a declared seam (`opendox.projection_seams`, `opendox.generator_seam`, +# `opendox.column_seams`), and the two columns are a host's, composed in +# through the handler-contribution facet (R1Q1 (a)), so the module is deleted. +# `tests/test_projection_seams.py`'s `test_the_stand_ins_module_is_retired` +# holds its absence, and `tests/test_source_core_arm.py` holds the handler's +# bases to openDox's own. # -------------------------------------------------------------------------- diff --git a/tests/test_profile_registration.py b/tests/test_profile_registration.py index 29780fd1..70bd1484 100644 --- a/tests/test_profile_registration.py +++ b/tests/test_profile_registration.py @@ -10,8 +10,9 @@ 1. NOTHING RESOLVES AT IMPORT TIME. The whole value of a lazy proxy is that importing it cannot fail for want of a host, so the check is an import in a - SUBPROCESS that then reads the registry — `consumer_reach`'s own - `test_importing_the_seam_resolves_nothing` for the same reason. + SUBPROCESS that then reads the registry, as `consumer_reach`'s + `test_importing_the_seam_resolves_nothing` did for the same reason, until + plan 034 T084 retired that seam. 2. THE UNREGISTERED READ REFUSES, AND THE MESSAGE NAMES THE CALL. A refusal whose text does not name the fix is a stack trace with extra steps, so the assertion is on the CONTENT — the registration call, the ruling, the runbook, diff --git a/tests/test_projection_seams.py b/tests/test_projection_seams.py index 371d6c7e..7a634f89 100644 --- a/tests/test_projection_seams.py +++ b/tests/test_projection_seams.py @@ -66,7 +66,6 @@ from opendox import serve from opendox import workbench from opendox import branch_session as bs -from opendox import consumer_reach from opendox.boundary import BoundaryViolation, OutputBoundary ROOT = Path(__file__).resolve().parent.parent @@ -1797,14 +1796,14 @@ def test_no_proxy_over_a_seam_is_read_at_import_time() -> None: "serve_workbench.py": (["registry_mod"], [])}, found -def test_the_retired_stand_ins_are_gone_from_consumer_reach() -> None: - for name in ("snapshot", "snapshot_registry", "corpus_root", "generator", - "find_validator", "corpus_root_refusal", "generate_snapshot", - "is_rfc3339_datetime", "hosted_ref_refused", "scanned_roots", - "function", "constant"): - assert not hasattr(consumer_reach, name), name - assert name not in consumer_reach.__all__, name - assert consumer_reach.LateProjectionRoutes.LATE_COLUMN[2] == ("_serve_index",) +def test_the_stand_ins_module_is_retired() -> None: + """T055 retired the projection mechanism's stand-ins, and T084 the last + three: the gate console's module stand-in and the gate and projection + columns' late bases. So `consumer_reach` itself is gone (F4.1 whole: "the + file is absent"), and nothing in the package can import it.""" + import importlib.util + assert not (PACKAGE / "consumer_reach.py").exists() + assert importlib.util.find_spec("opendox.consumer_reach") is None # --------------------------------------------------------------------------- diff --git a/tests/test_reach_sweep.py b/tests/test_reach_sweep.py index ad993a96..c4c72cc2 100644 --- a/tests/test_reach_sweep.py +++ b/tests/test_reach_sweep.py @@ -47,7 +47,8 @@ because it could hide a reach into openxFactory. So is a star import from a package a sibling lives under (`from scripts import *`), which may import any submodule the package's `__all__` lists. A computed name is not refused, -because `consumer_reach`'s seam imports one. A relative call is read against +because a late seam may import one (`consumer_reach`'s did, until plan 034 +T084 retired it). A relative call is read against `globals()` or `__package__` as the module's own only where the module never rebinds either; where it does, the call is refused too. A name in a comment, a docstring or a string is not an import. The openxFactory ban goes one step @@ -587,7 +588,8 @@ def test_a_deferred_reach_into_the_consumer_passes_all_three(monkeypatch, tmp_pa def test_an_importing_call_it_cannot_read_is_refused(monkeypatch, tmp_path): """A spread that hides an importing call's module could hide a reach into openxFactory, so the sweep refuses it. A computed name is another - matter (`consumer_reach`'s seam imports one), and is not refused.""" + matter (a late seam may import one, as `consumer_reach`'s did until plan + 034 T084), and is not refused.""" (tmp_path / "src").mkdir() (tmp_path / "src" / "spreading.py").write_text( "import importlib\n\n\ndef verb(names):\n" diff --git a/tests/test_source_core_arm.py b/tests/test_source_core_arm.py index 745f9d57..4d579781 100644 --- a/tests/test_source_core_arm.py +++ b/tests/test_source_core_arm.py @@ -35,8 +35,9 @@ So this module is the RUNNABLE half, in the shape its four neighbours on the explicit list `validate` ran until plan 034 T036 already use (`test_leg_shape.py`, `test_consumer_reach.py`, `test_web_boundary.py`, `test_view_registry.py`): it -PARSES `src/opendox/serve.py` and reads the live `opendox.consumer_reach`, and -it imports `opendox.serve` nowhere. It was written while `opendox.serve` could +PARSES `src/opendox/serve.py`, and it imports `opendox.serve` nowhere +(it read the live `opendox.consumer_reach` too, until plan 034 T084 retired +that module). It was written while `opendox.serve` could not be imported at either leg: `from ideation_dashboard import serve_openxfactory_lanes` named openxFactory's PRE-CARVE package, a `stays_openxfactory_adapter` row (RULING DQ-1) present at neither destination, @@ -51,11 +52,12 @@ 1. THE ROUTE IS DECLARED HERE, with the pre-carve strings. Two constants and three methods, defined in `serve.py` rather than forwarded — the difference between "openDox owns this route" and "openDox can reach a leg that does". -2. THE COLUMN NO LONGER CARRIES THEM. Asserted on the LIVE - `consumer_reach.LateProjectionRoutes`, as an absence: a forwarder left behind - would be invisible, because the route would go on working wherever openXdox - happens to be installed — which is every developer machine, and neither claim - this slice makes. +2. THE COLUMN NO LONGER CARRIES THEM. Asserted as an absence, on the + handler's own bases since plan 034 T084 retired `consumer_reach`'s + `LateProjectionRoutes` (it was asserted on that live stand-in before): a + forwarder left behind would be invisible, because the route would go on + working wherever openXdox happens to be installed — which is every + developer machine, and neither claim this slice makes. 3. THE ORDER IS THE ONE THE BINDINGS HAD. `collect_bindings` groups every EXACT binding ahead of every PREFIX one, so `/source` refused with a message and `/source/` (empty tail) 404'd with divergence headers and a zero-length body. @@ -99,9 +101,9 @@ `--noconftest` SAFE, deliberately, like its neighbours on the explicit list `validate` ran until plan 034 T036: -nothing here needs a fixture, a path insertion or an installed consumer, and the -one import (`opendox.consumer_reach`) is the module whose whole point is that -importing it resolves nothing. +nothing here needs a fixture, a path insertion or an installed consumer, and it +imports nothing of the package's (its one import, `opendox.consumer_reach`, +went with that module at plan 034 T084). A CREATED file: no carve-manifest row (RULED OQ-C) — it declares what a destination assembles, which the manifest never carries. @@ -194,33 +196,43 @@ def test_the_three_handlers_are_defined_on_this_handler(name): "layer that PINS openDox") -@pytest.mark.parametrize("name", SOURCE_METHODS) -def test_the_consumer_column_no_longer_forwards_them(name): - """The absence, on the LIVE seam — see this module's point 2.""" - from opendox import consumer_reach +#: `DashboardHandler`'s bases since plan 034 T084: openDox's own two route +#: mixins and the stdlib handler. The gate and projection columns' late +#: stand-ins (`consumer_reach.LateGateRoutes`, `LateProjectionRoutes`) are +#: gone with `consumer_reach`, and those columns are a host's, composed in +#: through the handler-contribution facet (R1Q1 (a)). +HANDLER_BASES = ("serve_workbench.WorkbenchRoutes", "serve_project.ProjectRoutes", + "http.server.SimpleHTTPRequestHandler") + - _module, _cls, methods = consumer_reach.LateProjectionRoutes.LATE_COLUMN - assert name not in methods, ( - f"consumer_reach.LateProjectionRoutes still forwards {name} into " - "openxdox.serve_projection. A forwarder left behind keeps the route " - "working on any machine that happens to have openXdox installed, which " - "is how a move looks complete and is not") +@pytest.mark.parametrize("name", SOURCE_METHODS) +def test_no_consumer_column_is_a_base_to_forward_them(name): + """The absence, on the handler's own bases (this module's point 2): with + no late column among them, nothing can forward `name` into openXdox's + projection column, so the route is answered here or nowhere.""" + bases = tuple(ast.unparse(base) for base in _handler_class().bases) + assert bases == HANDLER_BASES, ( + f"DashboardHandler's bases are {bases}. A late consumer column among " + f"them could forward {name} into openxdox.serve_projection, which keeps " + "the route working on any machine that happens to have openXdox " + "installed: how a move looks complete and is not") + assert _method(name) is not None SNAPSHOT_METHODS = ("_query_key", "_read_snapshot", "_serve_snapshot", "_hosted_entry_refused") -def test_the_column_keeps_only_its_own_contributed_route(): +def test_the_contributed_index_route_stays_the_columns(): """`_serve_index` stays the projection column's: its contributed - `/snapshot-index.json` binding names it. Since plan 034 T055 it is the ONE - method forwarded. The core `/snapshot.json` arm's four handlers, which S6 - left on the column, are this handler's own, because forwarded they refused - every `/snapshot.json` of a standalone server.""" - from opendox import consumer_reach - - _module, _cls, methods = consumer_reach.LateProjectionRoutes.LATE_COLUMN - assert methods == ("_serve_index",), methods + `/snapshot-index.json` binding names it, and since plan 034 T084 the + column arrives with the binding through the handler-contribution facet, + not as a base here. The core `/snapshot.json` arm's four handlers, which S6 + left on the column, are this handler's own (T055), because forwarded they + refused every `/snapshot.json` of a standalone server.""" + assert _method("_serve_index") is None, ( + "DashboardHandler defines _serve_index, which is the projection " + "column's handler for its own contributed /snapshot-index.json binding") @pytest.mark.parametrize("name", SNAPSHOT_METHODS) From 0b0c37d599bbbc42ddd210df33466867cf3c3267 Mon Sep 17 00:00:00 2001 From: Brett Heap <1513478+brettheap@users.noreply.github.com> Date: Fri, 2 Oct 2026 22:55:35 +0000 Subject: [PATCH 39/88] T084: the rail's thread read and a standalone own-document turn, held Two cases over a standalone `python -m opendox.serve` child. 1. tests/test_capability_honesty.py: the chat rail's thread read, with the exact query the rail sends on opening a document: `repository=fixture&ref=main&tile_kind=cluster&tile_id=barrel-rain&document=` (`views/staging-workbench.js` `loadThread`). T095's AT-R1 harness (openDox-code#75) found that the route reached `from openxdox import doxbench_scope` (`serve_workbench.py:548` at 047bb4fa) and dropped the connection. A query-less GET stops at the 400 check first, which is why batch L's measurement missed it. This task's routing commit already sends the live-session question through `column_seams.scope`. The case holds it: the answer is the stated no-session absence (`thread_capability_unavailable`, `NO_LIVE_SESSION_CAUSE`). Before (a39e0201): RemoteDisconnected. 2. tests/test_neutral_turn_scope.py, case 4: with a `model-binding add` binding configured, a turn over the tile's own document is answered and is never refused at the guard. Until T085 (#71) merges into this branch, the plane answers at its validators step (`model_capability_unavailable`); after it, the model step answers (`model_unavailable`). The merge commit narrows the case to that one answer, as accepted by the holder. Lane: openxfactory-4 (openXfactory-4-openDox_extraction) Co-Authored-By: Claude Opus 5.5 (1M context) --- tests/test_capability_honesty.py | 33 ++++++++++++++++++ tests/test_neutral_turn_scope.py | 59 +++++++++++++++++++++++++++++++- 2 files changed, 91 insertions(+), 1 deletion(-) diff --git a/tests/test_capability_honesty.py b/tests/test_capability_honesty.py index ae5c1f97..3dd4a69d 100644 --- a/tests/test_capability_honesty.py +++ b/tests/test_capability_honesty.py @@ -469,6 +469,39 @@ def test_the_three_crash_sites_answer_a_standalone_server( assert child.refused() == [], child.refused() +def test_the_rails_thread_read_answers_a_standalone_server( + tmp_path, monkeypatch) -> None: + """The chat rail's thread read, with the exact query the rail sends on + opening a document (`views/staging-workbench.js` `loadThread`, called from + `views/doxbench-chat.js`): a group tile and one of its members. Found by + T095's AT-R1 harness (openDox-code#75): the route reached + `from openxdox import doxbench_scope` (`serve_workbench.py:548` at + `047bb4fa`) and dropped the connection. A query-less GET stops at the 400 + check first, which is why batch L's measurement missed it. The live-session + question now goes through `column_seams.scope`, and a checkout with no open + session answers the stated no-session absence.""" + from opendox import serve_workbench + from opendox.serve_wire import DOXBENCH_ERR_THREAD_CAPABILITY_UNAVAILABLE + _clean_environment(monkeypatch) + repo = _repository(tmp_path, identity=True) + child, base, caps = _standalone(tmp_path, repo) + try: + token = caps.get("console_token") + assert caps["actions"]["session"] is True and token, caps + status, body, raw = _call( + base, "GET", + "/workbench/thread?repository=fixture&ref=main&tile_kind=cluster" + "&tile_id=barrel-rain&document=notes-rain-barrel-leak.md", + token=token) + assert _structured((status, body, raw)), (status, raw) + assert body["error"] == DOXBENCH_ERR_THREAD_CAPABILITY_UNAVAILABLE, raw + assert body.get("cause") == serve_workbench.NO_LIVE_SESSION_CAUSE, body + assert child.interrupt() == 0, child.stderr_text() + finally: + child.kill() + assert child.refused() == [], child.refused() + + #: The binding `opendox model-binding add` declares for the cases below, as #: `tests/test_model_provider_broker.py` declares its own. Nothing is spawned #: and nothing is contacted: no case dispatches a turn. diff --git a/tests/test_neutral_turn_scope.py b/tests/test_neutral_turn_scope.py index 7d823e6c..147d6d92 100644 --- a/tests/test_neutral_turn_scope.py +++ b/tests/test_neutral_turn_scope.py @@ -33,6 +33,14 @@ `unknown_action`. And the record a Save writes is a governed gate-action record, which the gate seam's default refuses by name. +4. STANDALONE, `python -m opendox.serve` as a child with neither sibling + importable, over the same checkout and binding: a turn over the tile's own + document is answered, never dropped, and is not refused at the guard. + Until T085's validators merge into this branch, the plane answers at its + validators step (`model_capability_unavailable`), before scope. Once they + do, the turn reaches the model step (`model_unavailable`) as case 1 does, + and the merge narrows this case to that one answer. + A CREATED FILE: no carve-manifest row (RULED OQ-C). """ @@ -40,12 +48,14 @@ import http.client import json +import os +import re import threading from pathlib import Path import pytest -from standalone_child import fresh_repository, git, run_module +from standalone_child import Child, fresh_repository, git, run_module ROOT = Path(__file__).resolve().parent.parent PLAIN = ROOT / "tests" / "fixtures" / "plain-documents" @@ -220,3 +230,50 @@ def test_the_tiles_own_documents_are_exactly_the_editable_set(host) -> None: assert projection.editable_paths == own assert projection.active_document_candidates == own assert OUTSIDE not in projection.editable_paths + + +_SERVE_URL = re.compile(r"^serving ideation dashboard at " + r"(http://([0-9.]+):([0-9]+))/index\.html$") + + +def test_a_standalone_turn_over_the_tiles_own_document_is_answered( + tmp_path, monkeypatch) -> None: + """Case 4. The child's environment carries no `GIT_*` and no `XF_*`, so + its actor is the one its repository's identity names (the suite's own + roster of several principals would resolve none).""" + from opendox.serve_wire import (DOXBENCH_ERR_MODEL_CAPABILITY_UNAVAILABLE, + DOXBENCH_ERR_MODEL_UNAVAILABLE) + for name in list(os.environ): + if name.startswith(("GIT_", "XF_")): + monkeypatch.delenv(name) + monkeypatch.setenv("GIT_CONFIG_GLOBAL", os.devnull) + monkeypatch.setenv("GIT_CONFIG_SYSTEM", os.devnull) + repo = fresh_repository(PLAIN, tmp_path) + git(repo, "config", "user.name", "fixture") + git(repo, "config", "user.email", "fixture@example.invalid") + added, status = run_module(tmp_path, "opendox.cli", "model-binding", "add", + "--repo-root", str(repo), *BINDING) + assert status == 0, added.stderr_text() + out = tmp_path / "out" / "snapshot.json" + generated, status = run_module( + tmp_path, "opendox.cli", "generate", "--repo-root", str(repo), + "--repository", "fixture", "--output", str(out), "--no-validate") + assert status == 0, generated.stderr_text() + child = Child(tmp_path, "opendox.serve", "--snapshot", str(out), + "--checkout-root", str(repo), "--port", "0") + try: + match = child.wait_for_line(_SERVE_URL) + base = (match.group(2), int(match.group(3))) + status, caps, raw = _call(base, "GET", "/capabilities") + assert status == 200 and caps["actions"]["session"] is True, raw + status, body, raw = _call(base, "POST", "/actions/workbench/chat-turn", + body=_turn(repo, OWN), + token=caps["console_token"]) + # the validators step until T085 merges in, the model step after it; + # never the guard's `turn_scope_refused`, never a dropped connection + assert body.get("error") in (DOXBENCH_ERR_MODEL_CAPABILITY_UNAVAILABLE, + DOXBENCH_ERR_MODEL_UNAVAILABLE), (status, raw) + assert child.interrupt() == 0, child.stderr_text() + finally: + child.kill() + assert child.refused() == [], child.refused() From e214477dc58727ddaafcda745b65d330790011ca Mon Sep 17 00:00:00 2001 From: Brett Heap <1513478+brettheap@users.noreply.github.com> Date: Fri, 2 Oct 2026 23:02:02 +0000 Subject: [PATCH 40/88] Adversarial review M1: a comma in the state directory is refused MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit PostgreSQL splits -k (unix_socket_directories) on commas. libpq splits a host on them once the DSN's percent-encoding is decoded. So a state directory such as "…/c2,…/ev" made the server put its sockets in two directories nothing had checked, one of them 0777, while the checked 0700 socket directory stayed empty (reproduced by the review). config.database_bundle, the one place every bundle is derived (from OPENDOX_STATE_DIR, XDG_STATE_HOME or HOME), now refuses a comma at configuration. The refusal names where the directory came from and does not repeat the value. New case: the refusal from each of the three sources, through load_settings and load_migration_settings, and the message never holds the value. A mutant without the refusal is killed. The review's own state directory is now refused before anything starts. Lane: openxfactory-4 (openXfactory-4-openDox_extraction) Co-Authored-By: Claude Opus 5.5 (1M context) --- src/opendox/runtime/config.py | 21 ++++++++++++++++++++- tests_runtime/test_bundled_postgres.py | 22 ++++++++++++++++++++++ 2 files changed, 42 insertions(+), 1 deletion(-) diff --git a/src/opendox/runtime/config.py b/src/opendox/runtime/config.py index e160b5e3..c1d9ea56 100644 --- a/src/opendox/runtime/config.py +++ b/src/opendox/runtime/config.py @@ -1778,8 +1778,27 @@ def state_dir(env: Mapping[str, str] | None = None) -> Path: def database_bundle(state: Path) -> DatabaseBundle: - """The bundle under `state`, refusing a socket path the kernel cannot bind.""" + """The bundle under `state`, refusing a socket path the kernel cannot bind. + + A COMMA IS REFUSED, wherever the state directory came from + (`OPENDOX_STATE_DIR`, `XDG_STATE_HOME` or the home directory), because + both ends read the socket directory as a LIST. PostgreSQL splits `-k` + (`unix_socket_directories`) on commas, and libpq splits a `host` on them + once the DSN's percent-encoding is decoded. So `…/a,…/b` made the server + put its sockets in two directories nothing here had checked, one of + them anyone could write, while the checked 0700 one stayed empty + (adversarial review of openDox-code#69). The value is not repeated: + the refusal names where it came from, and that is enough to find it. + """ bundle = DatabaseBundle(state_dir=state) + if "," in str(state): + raise ConfigurationError( + "the state directory's path holds a `,`. PostgreSQL reads its " + "socket directories, and libpq its hosts, as comma-separated " + "lists, so a comma would split this install's one socket " + "directory into two it never checked. Choose a state directory " + f"without one: {PREFIX}STATE_DIR or, where that is unset, " + "XDG_STATE_HOME or HOME") length = len(os.fsencode(str(bundle.socket_path))) if length > UNIX_SOCKET_PATH_MAX: raise ConfigurationError( diff --git a/tests_runtime/test_bundled_postgres.py b/tests_runtime/test_bundled_postgres.py index e650275f..2ec65b4d 100644 --- a/tests_runtime/test_bundled_postgres.py +++ b/tests_runtime/test_bundled_postgres.py @@ -301,6 +301,28 @@ def test_a_state_dir_too_long_for_a_unix_socket_is_refused_naming_it() -> None: assert STATE in str(caught.value) and "socket" in str(caught.value) +@pytest.mark.parametrize("variable", [STATE, "XDG_STATE_HOME", "HOME"]) +def test_a_comma_in_the_state_dir_is_refused_without_repeating_it( + monkeypatch, variable: str) -> None: + """PostgreSQL splits its socket directories, and libpq its hosts, on a + comma, so one would split the socket's one checked directory into two + unchecked ones (adversarial review of #69). Refused at configuration, + from whichever setting it came, and the value is not repeated.""" + value = "/tmp/odx-c1,/tmp/odx-c2-secretish" + env = {MODE: "local"} + if variable == "HOME": + monkeypatch.setenv("HOME", value) # `Path.home()` reads the process's + else: + env[variable] = value + with pytest.raises(config.ConfigurationError) as caught: + config.load_settings(env) + message = str(caught.value) + assert "`,`" in message and STATE in message, message + assert "secretish" not in message and value not in message, message + with pytest.raises(config.ConfigurationError): + config.load_migration_settings(env) + + def test_a_relative_state_dir_is_refused_naming_it() -> None: with pytest.raises(config.ConfigurationError) as caught: config.load_settings({MODE: "local", STATE: "var/state"}) From c4f5df3c0fd956b6373b513787834a7693d26142 Mon Sep 17 00:00:00 2001 From: Brett Heap <1513478+brettheap@users.noreply.github.com> Date: Fri, 2 Oct 2026 23:05:10 +0000 Subject: [PATCH 41/88] Adversarial review L1: the carrier is pinned below 0.7, and another major is refused by name pixeltable-pgserver>=0.6.0 had no ceiling. bundle.py runs the PostgreSQL 16 under pginstall/, upstream's default is already 18, and a cluster is opened only by the major that made it. Brett has ruled that release 1 publishes to PyPI, which makes an unbounded server dependency a property of every user's data. - pyproject.toml's local extra is now pixeltable-pgserver>=0.6.0,<0.7. The lock's ==0.6.0 sits inside that range. The local-extra case asserts both. - Before an existing cluster is used, BundledServer asks the server's own `postgres --version` against the cluster's PG_VERSION. Another major, or a server that does not say, is the named refusal, before anything is written into the data directory. New case: an existing PostgreSQL 16 cluster, with a stand-in server that says 17.2 (refused, both majors named), 16.14 (it goes on to the launch), or nothing (refused). All four mutants are killed: the check skipped, another major accepted, a silent server accepted, the ceiling dropped. Lane: openxfactory-4 (openXfactory-4-openDox_extraction) Co-Authored-By: Claude Opus 5.5 (1M context) --- pyproject.toml | 15 ++++++++--- src/opendox/runtime/bundle.py | 35 ++++++++++++++++++++++++++ tests_runtime/test_bundled_postgres.py | 8 ++++-- tests_runtime/test_local_lifecycle.py | 28 +++++++++++++++++++++ 4 files changed, 80 insertions(+), 6 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 22c71a72..c8794fb1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -136,12 +136,19 @@ test = [ # * about 24.7 MB per wheel, because it carries the two server majors; # * Apache-2.0 for the package; the PostgreSQL License for the server. # `>=0.6.0` is the release this package has been exercised against, the same -# rule every floor in this file is set by. Its own requirements (fasteners, -# platformdirs, psutil, typing-extensions) arrive with it and are imported by -# nothing here. +# rule every floor in this file is set by. `<0.7` IS A CEILING, which no other +# requirement here carries, because this one is a DATABASE SERVER whose major +# is a property of the user's data: `bundle.py` runs the PostgreSQL 16 under +# `pginstall/`, and a cluster is opened only by the major that made it. +# Upstream's own default is already 18, so a later release may move what +# `pginstall/` holds; the ceiling keeps that a deliberate act of this package +# (adversarial review of openDox-code#69), and `bundle.py` still refuses, by +# name, a server whose major disagrees with the cluster's `PG_VERSION`. Its +# own requirements (fasteners, platformdirs, psutil, typing-extensions) arrive +# with it and are imported by nothing here. local = [ "opendox[runtime]", - "pixeltable-pgserver>=0.6.0", + "pixeltable-pgserver>=0.6.0,<0.7", ] # THE RUNTIME EXTRA — `split-opendox-two-layer-product` § 3.5, RULED Q2 diff --git a/src/opendox/runtime/bundle.py b/src/opendox/runtime/bundle.py index 2da965d3..e872c00f 100644 --- a/src/opendox/runtime/bundle.py +++ b/src/opendox/runtime/bundle.py @@ -70,6 +70,7 @@ import ctypes import importlib.util import os +import re import shutil import signal import stat @@ -722,6 +723,7 @@ def _initialize(self, binaries: Path) -> None: """ data = self.bundle.data_dir if (data / "PG_VERSION").is_file(): + self._refuse_another_major(binaries) return if data.exists(): try: @@ -748,6 +750,39 @@ def _initialize(self, binaries: Path) -> None: shutil.rmtree(attempt, ignore_errors=True) raise + def _refuse_another_major(self, binaries: Path) -> None: + """An existing cluster is opened only by the major that made it. + + PostgreSQL refuses another major's data directory itself, but only + from inside a launch, where the reason reaches nobody but the log. + The carrier's `pginstall/` is PostgreSQL 16 today, and upstream's + default is already 18 (adversarial review of openDox-code#69). So + the server's own `postgres --version` is asked first, against the + cluster's `PG_VERSION`, and a disagreement is the named refusal, + before anything is written into the data directory. + """ + data = self.bundle.data_dir + cluster = (data / "PG_VERSION").read_text(encoding="utf-8").strip() + done = subprocess.run( + [str(binaries / "postgres"), "--version"], env=_child_environment(), + capture_output=True, text=True, timeout=START_TIMEOUT_SECONDS) + found = re.search(r"\(PostgreSQL\) (\d+)", done.stdout or "") + if found is None: + raise BundleRefused( + f"the bundled `postgres` under {binaries} does not say which " + f"PostgreSQL it is (`postgres --version` exited " + f"{done.returncode}), so it is not given {data}, a " + f"PostgreSQL {cluster} cluster; reinstall the `local` extra") + if found.group(1) != cluster: + raise BundleRefused( + f"{data} holds a PostgreSQL {cluster} cluster and the bundled " + f"server is PostgreSQL {found.group(1)}: a cluster is opened " + "only by the major version that made it. Reinstall the " + f"`local` extra this install was made with (`{SERVER_DISTRIBUTION}` " + "is pinned below 0.7 for this reason), or move the data " + "directory aside, and lose its coordination state, to start " + "a new one") + def _remove_abandoned_attempts(self) -> None: """Every initialization attempt whose process no longer exists.""" for candidate in self.bundle.data_dir.parent.glob( diff --git a/tests_runtime/test_bundled_postgres.py b/tests_runtime/test_bundled_postgres.py index 2ec65b4d..ce87ed02 100644 --- a/tests_runtime/test_bundled_postgres.py +++ b/tests_runtime/test_bundled_postgres.py @@ -258,12 +258,16 @@ def test_the_local_extra_carries_the_runtime_and_the_server_and_test_joins_it( # the carrier RULED on openxFactory#656 `5916000030` item 2 assert any(req.startswith("pixeltable-pgserver") for req in extras["local"]) assert not any(req.startswith("pgserver") for req in extras["local"]) + # WITH A CEILING (adversarial review of #69): the carrier's `pginstall/` + # major is a property of the user's data, so no later release moves it + # without this package's say. The lock's pin sits inside the range. + assert "pixeltable-pgserver>=0.6.0,<0.7" in extras["local"], extras["local"] assert "opendox[local]" in extras["test"], ( "F9.1 installs `.[test]` alone; without the local extra there, this " "suite could not start the server it tests") lock = (ROOT / "constraints-cpython312-linux.txt").read_text() - assert re.search(r"(?m)^pixeltable-pgserver==", lock), \ - "the lock does not pin pixeltable-pgserver" + assert re.search(r"(?m)^pixeltable-pgserver==0\.6\.\d+$", lock), \ + "the lock does not pin pixeltable-pgserver inside >=0.6.0,<0.7" assert not re.search(r"(?m)^pgserver==", lock), "the lock still pins pgserver" files = project["tool"]["setuptools"]["data-files"] assert files == {"share/opendox/migrations": ["migrations/*.sql"]} diff --git a/tests_runtime/test_local_lifecycle.py b/tests_runtime/test_local_lifecycle.py index 7bc74535..ea1bdfa0 100644 --- a/tests_runtime/test_local_lifecycle.py +++ b/tests_runtime/test_local_lifecycle.py @@ -890,6 +890,34 @@ def _target_of_initdb() -> str: return 'while [ "$1" != "-D" ]; do shift; done; T="$2"' +@pytest.mark.parametrize("says", ["17.2", "16.14", ""]) +def test_an_existing_cluster_is_opened_only_by_its_own_major( + monkeypatch, tmp_path: Path, short_state: Path, says: str) -> None: + """The server's `postgres --version` against the cluster's `PG_VERSION` + (adversarial review of #69). Another major, or a server that does not + say, is the named refusal, before anything is written into the data + directory. The same major goes on to the launch, which the stand-in + fails on purpose.""" + data = short_state / "postgres" / "data" + data.mkdir(parents=True, mode=0o700) + (short_state / "postgres").chmod(0o700) + (data / "PG_VERSION").write_text("16\n", encoding="utf-8") + answer = f'echo "postgres (PostgreSQL) {says}"' if says else "true" + server = _server(monkeypatch, tmp_path, short_state, initdb="exit 1", + postgres=f'if [ "$1" = --version ]; then {answer}; exit 0; fi\nexit 3') + with pytest.raises(bundle_mod.BundleRefused) as caught: + server.start() + message = str(caught.value) + if says == "17.2": + assert "PostgreSQL 16 cluster" in message and "PostgreSQL 17:" in message, message + elif says == "": + assert "does not say which PostgreSQL it is" in message, message + else: + assert "exited during start (exit 3)" in message, message + if says != "16.14": + assert not (data / "pg_hba.conf").exists(), "wrote into another major's cluster" + + def test_an_initdb_that_dies_midway_leaves_no_data_directory( monkeypatch, tmp_path: Path, short_state: Path) -> None: """The half-built cluster: `PG_VERSION` written, then the run fails. The From b2d80e94fd4fcc4af7afc3caba11ca095d42c3f4 Mon Sep 17 00:00:00 2001 From: Brett Heap <1513478+brettheap@users.noreply.github.com> Date: Fri, 2 Oct 2026 23:06:12 +0000 Subject: [PATCH 42/88] Adversarial review L3: the directory creation starts from is judged by its descriptor _make_private_directories opened its base with O_DIRECTORY, which follows a link (allowed on the configured path), and never judged the descriptor. So its own guard did not cover the very directory it writes into, which is round 11's rule ("nothing is made through a path that was not judged first"), leaving that to the path-wise check in front of it. The opened base is now fstat-judged with the tree check's own question, _unsafe_because, before the first mkdir. It must be this user's alone where it is the state directory or under it. Otherwise it must be this user's or root's, with any write by others only behind the sticky bit. A new keyword, `state`, tells the function which is which. New case: the function is asked directly, with no path-wise check in front, for a 0777 base reached through a link, a 0777 base, a link that is itself the state directory to a 0777 one, and a sticky state directory. Each is refused, and nothing is made. Two mutants are killed: the base not judged, and the state directory judged as an ancestor (killed by the sticky-state shape). Lane: openxfactory-4 (openXfactory-4-openDox_extraction) Co-Authored-By: Claude Opus 5.5 (1M context) --- src/opendox/runtime/bundle.py | 20 ++++++++++++++++-- tests_runtime/test_local_lifecycle.py | 30 +++++++++++++++++++++++++++ 2 files changed, 48 insertions(+), 2 deletions(-) diff --git a/src/opendox/runtime/bundle.py b/src/opendox/runtime/bundle.py index e872c00f..4ee1827d 100644 --- a/src/opendox/runtime/bundle.py +++ b/src/opendox/runtime/bundle.py @@ -346,7 +346,7 @@ def _preexec() -> None: # pragma: no cover - runs in the child BUNDLE_TREE = BUNDLE_SOCKET_DIR.parts -def _make_private_directories(leaf: Path) -> None: +def _make_private_directories(leaf: Path, *, state: Path) -> None: """`leaf` and every missing directory above it, each born exactly 0700. `Path.mkdir(parents=True)` gives the directories it creates on the way @@ -369,6 +369,16 @@ def _make_private_directories(leaf: Path) -> None: or written through: a symbolic link, something that is not a directory, or a directory that is not this user's alone. + AND THE DIRECTORY IT STARTS FROM IS JUDGED BY ITS DESCRIPTOR, before + the first `mkdir` (adversarial review of openDox-code#69). Opening it + follows a link, which this install allows on the configured path, so + what the descriptor names is asked the tree check's own question + (`_unsafe_because`): this user's alone where it is the state directory + or under it, and otherwise this user's or root's, with any write by + others only behind the sticky bit. The path-wise check before it asks + the same question; this one asks it of the very directory that is + written into. + THE UMASK IS PROCESS-WIDE, and it is narrowed only for these few `mkdir`s and then put back. A file another thread creates meanwhile can only come out more private than asked, never less. @@ -382,6 +392,11 @@ def _make_private_directories(leaf: Path) -> None: if not missing: return descriptor = os.open(base, os.O_RDONLY | os.O_DIRECTORY) + own = base == state or state in base.parents + reason = _unsafe_because(os.fstat(descriptor), uid=uid, own=own) + if reason is not None: + os.close(descriptor) + raise BundledServer._unsafe(base, reason) path = base previous = os.umask(0o077) try: @@ -622,7 +637,8 @@ def _prepare_directories(self) -> None: # BEFORE THE CHMOD, which follows a symbolic link: a `run` placed # there as a link would otherwise have its TARGET re-moded. self._refuse_an_unsafe_tree(existing_only=True) - _make_private_directories(self.bundle.socket_dir) + _make_private_directories(self.bundle.socket_dir, + state=self.bundle.state_dir) self._refuse_an_unsafe_tree() os.chmod(self.bundle.socket_dir, 0o700) diff --git a/tests_runtime/test_local_lifecycle.py b/tests_runtime/test_local_lifecycle.py index ea1bdfa0..68bbdba8 100644 --- a/tests_runtime/test_local_lifecycle.py +++ b/tests_runtime/test_local_lifecycle.py @@ -589,6 +589,36 @@ def test_nothing_is_made_through_a_path_the_tree_check_refuses( assert not os.path.lexists(made), f"{made} was made before the refusal" +@pytest.mark.parametrize("shape", ["link-to-open", "open", "link-to-open-state", + "sticky-state"]) +def test_the_directory_creation_starts_from_judges_its_own_descriptor( + tmp_path: Path, short_state: Path, shape: str) -> None: + """`_make_private_directories` judges the directory it opens, by its + descriptor, before its first `mkdir` (adversarial review of #69). It is + asked here directly, with no path-wise check in front of it, so its own + guard is what is measured: a base that every user can write, reached + through a link or not, makes nothing beneath it. The state directory is + judged as the install's OWN, so even a sticky one is refused, where a + sticky ANCESTOR (`/tmp`'s shape) is accepted.""" + watched = short_state / "open" + watched.mkdir() + watched.chmod(0o1777 if shape == "sticky-state" else 0o777) + if shape == "link-to-open": + (short_state / "link").symlink_to(watched) + state = short_state / "link" / "state" + elif shape == "open": + state = watched / "state" + elif shape == "link-to-open-state": + (short_state / "link").symlink_to(watched) + state = short_state / "link" + else: + state = watched + with pytest.raises(bundle_mod.BundleRefused) as caught: + bundle_mod._make_private_directories(state / "postgres" / "run", state=state) + assert "writable by every user" in str(caught.value), caught.value + assert list(watched.iterdir()) == [], "made beneath an unsafe base" + + @pytest.mark.parametrize("shape", ["link", "foreign-directory"]) def test_a_name_put_in_the_way_first_is_refused_never_followed( monkeypatch, tmp_path: Path, short_state: Path, shape: str) -> None: From 9e2f303029dd9a874c2ec12c308a3bd001c335bd Mon Sep 17 00:00:00 2001 From: Brett Heap <1513478+brettheap@users.noreply.github.com> Date: Fri, 2 Oct 2026 23:13:57 +0000 Subject: [PATCH 43/88] Adversarial review L2: the local verbs judge their socket before connecting to it `runtime status`, `migrate` and `reset` of a local install connected, as the served role or as the owner, to whatever answered at the bundle's socket path. Nothing had checked that path. The review reproduced it with a 0777 state directory whose postgres/run linked to another bundle's socket directory: status and migrate ran as the owner role against the other server, while a start refused the same tree. Reproduced here too: status on such a tree reported the other bundle's applied migrations and exited 0. - bundle.refuse_an_unsafe_tree is the start's tree check lifted out of BundledServer (which now delegates to it), so a verb can ask it without a server. - bundle.refusal_before_connecting asks two things and writes nothing: - the tree check, of what exists; - a live server of THIS data directory behind the socket. Its own postmaster.pid must name a live pid, which (where /proc can say) is a postgres whose cwd is this data directory, and which listens at this socket directory (the lock file's fifth line). - runtime/cli.py asks it in all three local verbs before any connection. status reports `database: "not probed: "` (its database block is re-indented under the new branch; `git diff -w` shows only the branch). migrate and reset refuse as "local-bundle-unverified". A hosted install is not judged here. test_install_mode's local status case with no server now expects "not probed: no bundled server is running" where it read "unreachable": the same answer, reached without a connection. New cases: - a real one in test_bundled_postgres.py: a linked run directory, an open state directory, and a valid tree with no server, each against a running bundle. All three verbs are refused by name, and the running bundle's status stays reachable with its schema intact. - a hermetic one in test_local_lifecycle.py: no lock, a dead pid, a live pid that is not a postgres, a socket line that differs and one that matches (both without /proc), and an open state directory. The real case fails against the previous runtime/cli.py. Seven mutants are all killed. Lane: openxfactory-4 (openXfactory-4-openDox_extraction) Co-Authored-By: Claude Opus 5.5 (1M context) --- src/opendox/runtime/bundle.py | 192 ++++++++++++++++--------- src/opendox/runtime/cli.py | 171 +++++++++++++--------- tests_runtime/test_bundled_postgres.py | 57 ++++++++ tests_runtime/test_install_mode.py | 6 +- tests_runtime/test_local_lifecycle.py | 46 ++++++ 5 files changed, 337 insertions(+), 135 deletions(-) diff --git a/src/opendox/runtime/bundle.py b/src/opendox/runtime/bundle.py index 4ee1827d..c6aceee1 100644 --- a/src/opendox/runtime/bundle.py +++ b/src/opendox/runtime/bundle.py @@ -266,6 +266,62 @@ def _remove_a_proven_stale_lock(bundle: DatabaseBundle) -> None: (bundle.data_dir / "postmaster.pid").unlink(missing_ok=True) +def refusal_before_connecting(bundle: DatabaseBundle) -> str | None: + """Why a local verb must not connect to `bundle`'s socket, or `None`. + + `runtime status`, `migrate` and `reset` of a LOCAL install connect, as + the served role or as the OWNER, to whatever answers at the bundle's + socket path. A start judges that path, and these verbs did not: a state + directory every user could write, whose `postgres/run` was a link to + another bundle's socket directory, had `status` and `migrate` run as the + owner role against that other server while a start refused the same + tree (adversarial review of openDox-code#69). So before any client + connection, two things are asked, and neither writes anything: + + * THE TREE CHECK a start asks, of what exists (`refuse_an_unsafe_tree` + with `existing_only`): a socket directory that is a link, or that + another user could replace, is refused here as it is there; + * A LIVE SERVER OF THIS DATA DIRECTORY behind the socket. The lock + file in this data directory, `postmaster.pid`, names a live process + that, where the platform can say (`_serves`), is a `postgres` whose + working directory is this data directory, and it names THIS socket + directory as the one the server listens on (its fifth line). Where + `/proc` cannot say what the pid is, the other answers still bind + the socket to this tree. + """ + try: + refuse_an_unsafe_tree(bundle, existing_only=True) + except BundleRefused as exc: + return str(exc) + except OSError as exc: + return (f"the bundled server's path under {bundle.state_dir} could " + f"not be judged ({type(exc).__name__})") + lock = bundle.data_dir / "postmaster.pid" + try: + lines = lock.read_text(encoding="utf-8").splitlines() + pid = int(lines[0].strip()) + except (OSError, IndexError, ValueError): + return (f"no bundled server is running on {bundle.data_dir}: it has no " + "readable postmaster.pid. A local install's server is started " + f"by `opendox generate-and-open {LOCAL_FLAG}`, which owns it") + try: + os.kill(pid, 0) + except (ProcessLookupError, PermissionError): + return (f"no bundled server is running on {bundle.data_dir}: its " + f"postmaster.pid names pid {pid}, which is not a live process " + "of this user") + if _serves(pid, bundle) is False: + return (f"pid {pid}, named by {lock}, is not the postgres serving " + f"{bundle.data_dir}, so what answers at {bundle.socket_dir} " + "is not this install's server") + listens = lines[4].strip() if len(lines) > 4 else "" + if listens != str(bundle.socket_dir): + return (f"the server on {bundle.data_dir} listens at " + f"{listens or 'no Unix socket'}, not at {bundle.socket_dir}, so " + "what answers there is not this install's server") + return None + + def report(bundle: DatabaseBundle) -> dict[str, Any]: """The `database_bundle` block `runtime status` prints (#1144 13.1).""" return {"data_dir": str(bundle.data_dir), @@ -540,6 +596,75 @@ def _unsafe_because(info: os.stat_result, *, uid: int, own: bool) -> str | None: return None +def refuse_an_unsafe_tree(bundle: DatabaseBundle, *, + existing_only: bool = False) -> None: + """The socket's whole path is this user's to change, or it is refused. + + The socket's directory is how this install's clients find ITS + server, so the 0700 on it is worth only what the path above it is + worth. Peer authentication keeps other users out of the server, but + not a substitute socket out of the path: whoever could replace `run` + could stand up a server of their own for this install's clients to + talk to. A directory entry is controlled by its PARENT: a parent that + another user can write lets them rename `run` away, or put a symbolic + link in its place, after the mode is set. A symbolic link on the way there + can be pointed elsewhere by whoever owns it, or by whoever can write + the directory it sits in (Copilot review of openDox-code#69). So: + + * THE INSTALL'S OWN TREE, as the configured path resolves: the + state directory, `postgres/` and `run/` must be real directories, + owned by this user and writable by no one else; + * EVERY DIRECTORY ABOVE IT, on the configured path and on the path + it resolves to, must be owned by this user or by root. One that + anyone else can write, a group included, must be sticky, as + `/tmp` is, so nobody can rename what is not theirs; + * EVERY SYMBOLIC LINK on the configured path must be this user's or + root's. + + `OPENDOX_STATE_DIR` never holds `..` (`config.state_dir` refuses it), + so the configured path's components are the ones the kernel walks. + + With `existing_only`, the same rules are asked of only what exists + yet. `_prepare_directories` asks that BEFORE it creates anything, + so the links are checked first, a broken one included (Copilot + review of openDox-code#69). + """ + uid = os.getuid() + configured = bundle.state_dir + + def present(path: Path) -> bool: + return not existing_only or os.path.lexists(path) + + for component in (configured, *configured.parents): + if not present(component): + continue + info = os.lstat(component) + if stat.S_ISLNK(info.st_mode) and info.st_uid not in (uid, 0): + raise BundledServer._unsafe( + component, f"is a symbolic link owned by uid {info.st_uid}, " + "neither this user nor root, who could point it elsewhere") + state = configured.resolve() + tree = [state, state / BUNDLE_TREE[0], state / BUNDLE_TREE[0] / BUNDLE_TREE[1]] + # AND THE DATA DIRECTORY, where one exists already, a broken link + # included (Copilot review of openDox-code#69). A `data` placed there + # as a link to a cluster elsewhere would otherwise be launched, and + # given this install's authentication files, outside the state tree. + # A fresh one needs no check: `_initialize` renames it into place. + data = state / BUNDLE_DATA_DIR + if os.path.lexists(data): + tree.append(data) + checks = [(path, True) for path in tree] + [ + (path, False) for path in dict.fromkeys( + [*state.parents, *configured.parents])] + for directory, mine in checks: + if not present(directory): + continue + info = os.lstat(directory) if mine else os.stat(directory) + reason = _unsafe_because(info, uid=uid, own=mine) + if reason is not None: + raise BundledServer._unsafe(directory, reason) + + class BundledServer: """One local install's PostgreSQL server: started as this process's child. @@ -643,71 +768,8 @@ def _prepare_directories(self) -> None: os.chmod(self.bundle.socket_dir, 0o700) def _refuse_an_unsafe_tree(self, *, existing_only: bool = False) -> None: - """The socket's whole path is this user's to change, or it is refused. - - The socket's directory is how this install's clients find ITS - server, so the 0700 on it is worth only what the path above it is - worth. Peer authentication keeps other users out of the server, but - not a substitute socket out of the path: whoever could replace `run` - could stand up a server of their own for this install's clients to - talk to. A directory entry is controlled by its PARENT: a parent that - another user can write lets them rename `run` away, or put a symbolic - link in its place, after the mode is set. A symbolic link on the way there - can be pointed elsewhere by whoever owns it, or by whoever can write - the directory it sits in (Copilot review of openDox-code#69). So: - - * THE INSTALL'S OWN TREE, as the configured path resolves: the - state directory, `postgres/` and `run/` must be real directories, - owned by this user and writable by no one else; - * EVERY DIRECTORY ABOVE IT, on the configured path and on the path - it resolves to, must be owned by this user or by root. One that - anyone else can write, a group included, must be sticky, as - `/tmp` is, so nobody can rename what is not theirs; - * EVERY SYMBOLIC LINK on the configured path must be this user's or - root's. - - `OPENDOX_STATE_DIR` never holds `..` (`config.state_dir` refuses it), - so the configured path's components are the ones the kernel walks. - - With `existing_only`, the same rules are asked of only what exists - yet. `_prepare_directories` asks that BEFORE it creates anything, - so the links are checked first, a broken one included (Copilot - review of openDox-code#69). - """ - uid = os.getuid() - configured = self.bundle.state_dir - - def present(path: Path) -> bool: - return not existing_only or os.path.lexists(path) - - for component in (configured, *configured.parents): - if not present(component): - continue - info = os.lstat(component) - if stat.S_ISLNK(info.st_mode) and info.st_uid not in (uid, 0): - raise self._unsafe( - component, f"is a symbolic link owned by uid {info.st_uid}, " - "neither this user nor root, who could point it elsewhere") - state = configured.resolve() - tree = [state, state / BUNDLE_TREE[0], state / BUNDLE_TREE[0] / BUNDLE_TREE[1]] - # AND THE DATA DIRECTORY, where one exists already, a broken link - # included (Copilot review of openDox-code#69). A `data` placed there - # as a link to a cluster elsewhere would otherwise be launched, and - # given this install's authentication files, outside the state tree. - # A fresh one needs no check: `_initialize` renames it into place. - data = state / BUNDLE_DATA_DIR - if os.path.lexists(data): - tree.append(data) - checks = [(path, True) for path in tree] + [ - (path, False) for path in dict.fromkeys( - [*state.parents, *configured.parents])] - for directory, mine in checks: - if not present(directory): - continue - info = os.lstat(directory) if mine else os.stat(directory) - reason = _unsafe_because(info, uid=uid, own=mine) - if reason is not None: - raise self._unsafe(directory, reason) + """`refuse_an_unsafe_tree`, for this server's own bundle.""" + refuse_an_unsafe_tree(self.bundle, existing_only=existing_only) @staticmethod def _unsafe(directory: Path, reason: str) -> BundleRefused: diff --git a/src/opendox/runtime/cli.py b/src/opendox/runtime/cli.py index b5d22207..605f65ad 100644 --- a/src/opendox/runtime/cli.py +++ b/src/opendox/runtime/cli.py @@ -491,6 +491,24 @@ def cmd_init(args: argparse.Namespace) -> int: "next": "opendox-runtime runtime migrate"}, ok=True) +def _local_bundle_refusal(settings) -> str | None: + """For a LOCAL install, why its socket must not be connected to, or `None`. + + `bundle.refusal_before_connecting`: the tree check a start asks, and a + live server of THIS data directory behind the socket, both before any + client connection (adversarial review of openDox-code#69). A hosted + install's DSN is the operator's, and is not judged here. + """ + if settings.install_mode != INSTALL_MODE_LOCAL: + return None + from opendox.runtime import bundle as bundle_mod + from opendox.runtime.config import database_bundle + + reason = bundle_mod.refusal_before_connecting( + database_bundle(settings.state_dir)) + return None if reason is None else _safe_message(reason) + + def cmd_migrate(args: argparse.Namespace) -> int: """Apply the ordered SQL, or with `--plan` report what would be applied. @@ -526,6 +544,10 @@ def cmd_migrate(args: argparse.Namespace) -> int: "package with the `runtime` extra: " "pip install '.[runtime]'"}, ok=False) + refusal = _local_bundle_refusal(settings) + if refusal is not None: + return _emit({"verb": "migrate", "refusal": "local-bundle-unverified", + "message": refusal}, ok=False) runner_db = Database(dsn, application_name="opendox-runtime-migrate", checkout_timeout=args.connect_timeout) # THE OUTCOME IS COMPUTED INSIDE THE CONTEXT AND EMITTED OUTSIDE IT. A @@ -742,76 +764,85 @@ def cmd_status(args: argparse.Namespace) -> int: report["runtime_extra"] = "present" connected = False - try: - # INSIDE the context, all of it. `runner.applied()` and `runner.plan()` - # each check a connection out of the pool, so calling them after the - # `with` had closed it raised `PoolClosed` and this verb reported a - # perfectly reachable database as unreachable (Copilot review of - # openDox-code#25, and it is the kind of defect only a live database - # shows — every unreachable-database test passed). - with Database(settings.database_url, - checkout_timeout=args.probe_timeout) as db: - with db.connection() as conn: - conn.execute("select 1") - # THE CONNECTIVITY ANSWER IS RECORDED THE MOMENT IT IS TRUE, so a - # failure in the queries BELOW cannot rewrite it — see the generic - # handler at the end of this block. - connected = True - # THE SAME CANONICAL GATE `apply()` AND `/readyz` RUN. Without - # it an EMPTY migrations directory reports `pending: []` on a - # fresh database — nothing pending, nothing drifted, everything - # fine — for an install with no coordination schema at all - # (Copilot review of openDox-code#25, round 7). `status` is the - # verb an operator believes. - migrations.verify_canonical_digest(settings.migrations_dir) - runner = migrations.MigrationRunner( - db, migrations_dir=settings.migrations_dir) - applied = [row.version for row in runner.applied()] - pending = [m.version for m in runner.plan()] - drifted = runner.drift() - report["database"] = "reachable" - report["applied_migrations"] = applied - report["pending_migrations"] = pending - # NOTHING PENDING IS NOT THE SAME AS MATCHING THIS TREE: a migration - # whose file changed, or vanished, is invisible to `plan()` and is - # refused by `apply()`. See `MigrationRunner.drift`. - report["migration_drift"] = drifted - # PENDING IS UNHEALTHY, exactly as `/readyz` treats it. This reported - # the versions and left `ok` true, so a reachable but UNMIGRATED - # database exited 0 while the readiness probe on the same install - # refuses traffic — two answers to one question, and the CLI's was the - # comforting one (Copilot review of openDox-code#25, round 7). - if drifted or pending: - ok = False - # a status verb reports, never raises - except migrations.MigrationError as exc: - # THE DATABASE ANSWERED; THE TREE DID NOT. `select 1` has already - # succeeded by the time the runner is asked anything, so reporting - # `database: unreachable` for a missing or malformed migrations - # directory pointed the operator at the wrong dependency entirely - # (Copilot review of openDox-code#25, round 7). - report.setdefault("database", "reachable") - report["migrations"] = ( - f"unreadable: {type(exc).__name__}: {_safe_message(exc)}") + # A LOCAL INSTALL'S SOCKET IS JUDGED BEFORE IT IS CONNECTED TO + # (adversarial review of openDox-code#69): the tree a start checks, and a + # live server of this data directory behind it. Otherwise `status` asks + # whatever answers at that path, as the served role. + refusal = _local_bundle_refusal(settings) + if refusal is not None: + report["database"] = f"not probed: {refusal}" ok = False - except Exception as exc: # noqa: BLE001 - # THE SAME DISTINCTION THE BRANCH ABOVE MAKES, for the failures that - # are not the runner's own. Once `select 1` has answered, the database - # IS reachable, and a later failure — the served role without `select` - # on the ledger, a schema the search path does not reach, a query that - # errors — is a privilege or schema problem reported as one. Reporting - # `database: unreachable` for it pointed the operator at the network - # and hid the real fault, which is the defect round 7 fixed for - # `MigrationError` and left in place one handler down (Copilot review - # of openDox-code#25, round 10, suppressed). - if connected: + else: + try: + # INSIDE the context, all of it. `runner.applied()` and `runner.plan()` + # each check a connection out of the pool, so calling them after the + # `with` had closed it raised `PoolClosed` and this verb reported a + # perfectly reachable database as unreachable (Copilot review of + # openDox-code#25, and it is the kind of defect only a live database + # shows — every unreachable-database test passed). + with Database(settings.database_url, + checkout_timeout=args.probe_timeout) as db: + with db.connection() as conn: + conn.execute("select 1") + # THE CONNECTIVITY ANSWER IS RECORDED THE MOMENT IT IS TRUE, so a + # failure in the queries BELOW cannot rewrite it — see the generic + # handler at the end of this block. + connected = True + # THE SAME CANONICAL GATE `apply()` AND `/readyz` RUN. Without + # it an EMPTY migrations directory reports `pending: []` on a + # fresh database — nothing pending, nothing drifted, everything + # fine — for an install with no coordination schema at all + # (Copilot review of openDox-code#25, round 7). `status` is the + # verb an operator believes. + migrations.verify_canonical_digest(settings.migrations_dir) + runner = migrations.MigrationRunner( + db, migrations_dir=settings.migrations_dir) + applied = [row.version for row in runner.applied()] + pending = [m.version for m in runner.plan()] + drifted = runner.drift() report["database"] = "reachable" - report["schema_queries"] = ( - f"failed: {type(exc).__name__}: {_safe_message(exc)}") - else: - report["database"] = ( - f"unreachable: {type(exc).__name__}: {_safe_message(exc)}") - ok = False + report["applied_migrations"] = applied + report["pending_migrations"] = pending + # NOTHING PENDING IS NOT THE SAME AS MATCHING THIS TREE: a migration + # whose file changed, or vanished, is invisible to `plan()` and is + # refused by `apply()`. See `MigrationRunner.drift`. + report["migration_drift"] = drifted + # PENDING IS UNHEALTHY, exactly as `/readyz` treats it. This reported + # the versions and left `ok` true, so a reachable but UNMIGRATED + # database exited 0 while the readiness probe on the same install + # refuses traffic — two answers to one question, and the CLI's was the + # comforting one (Copilot review of openDox-code#25, round 7). + if drifted or pending: + ok = False + # a status verb reports, never raises + except migrations.MigrationError as exc: + # THE DATABASE ANSWERED; THE TREE DID NOT. `select 1` has already + # succeeded by the time the runner is asked anything, so reporting + # `database: unreachable` for a missing or malformed migrations + # directory pointed the operator at the wrong dependency entirely + # (Copilot review of openDox-code#25, round 7). + report.setdefault("database", "reachable") + report["migrations"] = ( + f"unreadable: {type(exc).__name__}: {_safe_message(exc)}") + ok = False + except Exception as exc: # noqa: BLE001 + # THE SAME DISTINCTION THE BRANCH ABOVE MAKES, for the failures that + # are not the runner's own. Once `select 1` has answered, the database + # IS reachable, and a later failure — the served role without `select` + # on the ledger, a schema the search path does not reach, a query that + # errors — is a privilege or schema problem reported as one. Reporting + # `database: unreachable` for it pointed the operator at the network + # and hid the real fault, which is the defect round 7 fixed for + # `MigrationError` and left in place one handler down (Copilot review + # of openDox-code#25, round 10, suppressed). + if connected: + report["database"] = "reachable" + report["schema_queries"] = ( + f"failed: {type(exc).__name__}: {_safe_message(exc)}") + else: + report["database"] = ( + f"unreachable: {type(exc).__name__}: {_safe_message(exc)}") + ok = False # A LOCAL INSTALL HAS NO BROKER TO PROBE (plan 034 T070; #1144 13.4), # and that is its configuration rather than a fault: reported by name, and @@ -865,6 +896,10 @@ def cmd_reset(args: argparse.Namespace) -> int: except ImportError as exc: # pragma: no cover - the extra is absent return _emit({"verb": "reset", "refusal": "runtime-extra-missing", "message": _safe_message(exc)}, ok=False) + refusal = _local_bundle_refusal(settings) + if refusal is not None: + return _emit({"verb": "reset", "refusal": "local-bundle-unverified", + "message": refusal}, ok=False) try: with Database(dsn, application_name="opendox-runtime-reset", checkout_timeout=args.connect_timeout) as db: diff --git a/tests_runtime/test_bundled_postgres.py b/tests_runtime/test_bundled_postgres.py index ce87ed02..462251f9 100644 --- a/tests_runtime/test_bundled_postgres.py +++ b/tests_runtime/test_bundled_postgres.py @@ -53,6 +53,7 @@ import pytest from opendox.runtime import bundle as bundle_mod +from opendox.runtime import cli from opendox.runtime import config from opendox.runtime.config import PREFIX @@ -620,6 +621,62 @@ def test_a_cluster_whose_configuration_points_elsewhere_runs_on_its_own_files( assert method == f"peer:{bundle_mod.os_user()}", method +def _verb(state: Path, *verb: str) -> tuple[int, dict]: + """`OPENDOX_INSTALL_MODE=local opendox-runtime runtime `, a SECOND process.""" + done = subprocess.run( + [sys.executable, "-m", "opendox.runtime.cli", "runtime", *verb], + env=_clean_env(**{MODE: "local", STATE: str(state)}), cwd=ROOT, + capture_output=True, text=True, timeout=60) + return done.returncode, json.loads(done.stdout) + + +@pytest.mark.parametrize("shape", ["linked-run", "open-state", "no-server"]) +def test_the_local_verbs_connect_only_to_their_own_verified_server( + state_dir: Path, shape: str) -> None: + """The adversarial review of #69 (L2): a second state tree whose + `postgres/run` is a link to a RUNNING bundle's socket directory, in a + state directory every user could write. `status` and `migrate` ran on + it as the owner role, against the other bundle's server, while a start + refused the same tree. Each verb now asks the tree check and a live + server of its OWN data directory before any connection. A valid tree + with no server is answered the same way, by name and unconnected.""" + settings = config.load_settings({MODE: "local", STATE: str(state_dir)}) + other = Path(tempfile.mkdtemp(prefix="odx-o-", dir="/tmp" if os.path.isdir("/tmp") else None)) + try: + (other / "postgres" / "data").mkdir(parents=True, mode=0o700) + (other / "postgres").chmod(0o700) + if shape == "no-server": + (other / "postgres" / "run").mkdir(mode=0o700) + else: + (other / "postgres" / "run").symlink_to( + config.DatabaseBundle(state_dir).socket_dir) + if shape == "open-state": + other.chmod(0o777) + with bundle_mod.BundledServer(settings): + status_code, status = _status(other) + migrate_code, migrate = _verb(other, "migrate") + reset_code, reset = _verb(other, "reset", "--confirm", + cli.RESET_CONFIRMATION) + # the positive control: the running bundle's own verbs connect, + # and its schema was not dropped through the other tree's link + own_code, own = _status(state_dir) + finally: + other.chmod(0o700) + shutil.rmtree(other, ignore_errors=True) + expected = {"linked-run": "is a symbolic link", + "open-state": "writable by every user", + "no-server": "no bundled server is running"}[shape] + assert status_code == 1 and status["database"].startswith("not probed: "), status + assert expected in status["database"], status + assert migrate_code == 1, migrate + assert migrate["refusal"] == "local-bundle-unverified", migrate + assert expected in migrate["message"], migrate + assert reset_code == 1 and reset["refusal"] == "local-bundle-unverified", reset + assert expected in reset["message"], reset + assert own["database"] == "reachable" and own_code == 0, own + assert own["applied_migrations"] and not own["pending_migrations"], own + + def test_migrate_under_the_local_mode_uses_the_bundle_and_refuses_a_dsn( state_dir: Path) -> None: """`runtime migrate` is part of the same install: it reads the bundle's diff --git a/tests_runtime/test_install_mode.py b/tests_runtime/test_install_mode.py index 52047299..3cbbf4bb 100644 --- a/tests_runtime/test_install_mode.py +++ b/tests_runtime/test_install_mode.py @@ -334,8 +334,10 @@ def _no_broker(_settings): assert evidence["settings"][PREFIX + "OIDC_JWKS_URL"] == "" # the database half is the ONLY reason `ok` is false: no bundled server # is running on this (nonexistent) state directory, and `status` reports - # that rather than starting one - assert evidence["database"].startswith("unreachable"), evidence + # that rather than starting one. It says so WITHOUT connecting: a local + # socket is judged before it is asked (adversarial review of #69). + assert evidence["database"].startswith( + "not probed: no bundled server is running"), evidence assert evidence["database_bundle"]["pid"] is None, evidence assert code == 1 diff --git a/tests_runtime/test_local_lifecycle.py b/tests_runtime/test_local_lifecycle.py index 68bbdba8..96c8a479 100644 --- a/tests_runtime/test_local_lifecycle.py +++ b/tests_runtime/test_local_lifecycle.py @@ -948,6 +948,52 @@ def test_an_existing_cluster_is_opened_only_by_its_own_major( assert not (data / "pg_hba.conf").exists(), "wrote into another major's cluster" +def _a_tree(state: Path) -> config.DatabaseBundle: + """A valid bundle tree under `state`, every directory 0700, no server.""" + bundle = config.DatabaseBundle(state) + for directory in (state / "postgres", bundle.data_dir, bundle.socket_dir): + directory.mkdir(mode=0o700, exist_ok=True) + directory.chmod(0o700) + return bundle + + +@pytest.mark.parametrize("shape", ["no-lock", "dead-pid", "not-postgres", + "no-proc-other-socket", "no-proc-this-socket", + "open-state"]) +def test_a_local_verb_connects_only_behind_a_verified_server( + monkeypatch, short_state: Path, shape: str) -> None: + """`bundle.refusal_before_connecting` (adversarial review of #69, L2): + the tree a start checks, then a live server of THIS data directory, + named by its own lock file, listening at THIS socket directory. Where + `/proc` cannot say what the pid is, the lock file's socket line still + binds the socket to the tree.""" + bundle = _a_tree(short_state) + live = os.getpid() # alive, and not a postgres + dead = 2 ** 22 + 17 # above the default pid_max + lock = bundle.data_dir / "postmaster.pid" + socket_line = (str(bundle.socket_dir) if shape != "no-proc-other-socket" + else "/tmp/somewhere-else") + if shape != "no-lock": + pid = dead if shape == "dead-pid" else live + lock.write_text(f"{pid}\n{bundle.data_dir}\n1\n5432\n{socket_line}\n\n", + encoding="utf-8") + if shape.startswith("no-proc"): + monkeypatch.setattr(bundle_mod, "PROC", short_state / "no-proc") + if shape == "open-state": + short_state.chmod(0o777) + reason = bundle_mod.refusal_before_connecting(bundle) + expected = {"no-lock": "no readable postmaster.pid", + "dead-pid": "not a live process", + "not-postgres": "is not the postgres serving", + "no-proc-other-socket": "listens at /tmp/somewhere-else", + "no-proc-this-socket": None, + "open-state": "writable by every user"}[shape] + if expected is None: + assert reason is None, reason + else: + assert reason is not None and expected in reason, reason + + def test_an_initdb_that_dies_midway_leaves_no_data_directory( monkeypatch, tmp_path: Path, short_state: Path) -> None: """The half-built cluster: `PG_VERSION` written, then the run fails. The From 21bde1af699d1129a84d472fc538b60cd2c348e9 Mon Sep 17 00:00:00 2001 From: Brett Heap <1513478+brettheap@users.noreply.github.com> Date: Fri, 2 Oct 2026 23:15:18 +0000 Subject: [PATCH 44/88] Adversarial review note: no replication connection, logical or physical authentication_files' docstring said "a replication connection is refused". That holds for a PHYSICAL one, which matches no pg_hba.conf rule. A LOGICAL one (replication=database) names a database, so the one local rule read it as an ordinary connection. It was accepted as peer:, and IDENTIFY_SYSTEM answered (measured here, as the review found). The docstring was not just reworded, because no pg_hba.conf rule can tell the two apart. The launch now sets max_wal_senders=0, so the server starts no WAL sender at all, and the docstring says so. New real-server case: replication=database and replication=true are both refused as the owner role, while an ordinary connection still is not and reads max_wal_senders as 0. The mutant that leaves WAL senders on is killed. Lane: openxfactory-4 (openXfactory-4-openDox_extraction) Co-Authored-By: Claude Opus 5.5 (1M context) --- src/opendox/runtime/bundle.py | 15 +++++++++++++-- tests_runtime/test_bundled_postgres.py | 25 +++++++++++++++++++++++++ 2 files changed, 38 insertions(+), 2 deletions(-) diff --git a/src/opendox/runtime/bundle.py b/src/opendox/runtime/bundle.py index c6aceee1..18853154 100644 --- a/src/opendox/runtime/bundle.py +++ b/src/opendox/runtime/bundle.py @@ -520,7 +520,14 @@ def authentication_files(user: str) -> dict[str, str]: * Every HOST connection is rejected. The server also listens on no TCP address at all (`listen_addresses` is empty), so these lines never match. They are written so that the file says what the install is. - * No replication line, so a replication connection is refused. + * No replication line, so a PHYSICAL replication connection matches + no rule and is refused. A LOGICAL one (`replication=database`) + names a database, so `pg_hba.conf` reads it as the ordinary local + connection it resembles, and no rule here can tell the two apart: + it was accepted as `peer:` (adversarial review of + openDox-code#69). So the launch sets `max_wal_senders=0`, and the + server starts no WAL sender for either kind. A replication + connection is refused by the server, whatever the files say. """ header = ("# Written by opendox.runtime.bundle before every start of this local\n" "# install's bundled server (plan 034 T072). Changes here are replaced.\n") @@ -908,7 +915,11 @@ def _launch(self, binaries: Path) -> None: # the data directory and the two files written above. "-c", f"data_directory={self.bundle.data_dir}", "-c", f"hba_file={self.bundle.data_dir / 'pg_hba.conf'}", - "-c", f"ident_file={self.bundle.data_dir / 'pg_ident.conf'}"], + "-c", f"ident_file={self.bundle.data_dir / 'pg_ident.conf'}", + # NO REPLICATION, physical or logical: see + # `authentication_files`, whose rules cannot refuse a + # logical one (adversarial review of openDox-code#69). + "-c", "max_wal_senders=0"], stdin=subprocess.DEVNULL, stdout=log, stderr=subprocess.STDOUT, env=_child_environment(), # ITS OWN SESSION, so a terminal's Ctrl-C reaches this process diff --git a/tests_runtime/test_bundled_postgres.py b/tests_runtime/test_bundled_postgres.py index 462251f9..45432e45 100644 --- a/tests_runtime/test_bundled_postgres.py +++ b/tests_runtime/test_bundled_postgres.py @@ -566,6 +566,31 @@ def test_a_role_outside_the_map_is_refused_even_for_this_os_user( assert "peer authentication failed" in str(caught.value).lower(), caught.value +@pytest.mark.parametrize("mode", ["database", "true"]) +def test_a_replication_connection_is_refused_logical_or_physical( + state_dir: Path, mode: str) -> None: + """A PHYSICAL replication connection (`replication=true`) matches no + rule in `pg_hba.conf`. A LOGICAL one (`replication=database`) names a + database, and the one local rule admitted it as `peer:`, where + `IDENTIFY_SYSTEM` answered (adversarial review of #69). The server + starts no WAL sender (`max_wal_senders=0`), so both are refused, as the + owner role and over the same socket.""" + import psycopg + from psycopg.conninfo import make_conninfo + + settings = config.load_settings({MODE: "local", STATE: str(state_dir)}) + with bundle_mod.BundledServer(settings) as server: + with pytest.raises(psycopg.OperationalError) as caught: + psycopg.connect(make_conninfo(server.bundle.migration_dsn, + replication=mode)).close() + with _owner(server) as conn: # an ordinary one still is + senders = conn.execute("show max_wal_senders").fetchone()[0] + message = str(caught.value).lower() + assert ("max_wal_senders" in message if mode == "database" + else ("max_wal_senders" in message or "no pg_hba.conf entry" in message)), message + assert senders == "0", senders + + def test_an_older_trust_cluster_is_brought_back_to_peer_on_start( state_dir: Path) -> None: """A data directory an earlier build initialized with `trust` (or a file From b1db1965c57ab67305fa1f70148a0162bcab8436 Mon Sep 17 00:00:00 2001 From: Brett Heap <1513478+brettheap@users.noreply.github.com> Date: Fri, 2 Oct 2026 23:26:14 +0000 Subject: [PATCH 45/88] T084 fix round 1: a seam registration with the names but not their shape is refused (Copilot review) `projection_seams._Seam` gains an optional `shape` check. It runs after the name probe, in `register()` and `register_default()`, and refuses a registration whose further defects it answers, as a `TypeError` naming the call. Two seams use it: - `column_seams.gate`: `GateRefused` must be an exception class, because openDox's verbs catch it (`except .GateRefused`). A callable that is not one would pass the name probe and raise `TypeError` at the first refusal (review r4170607959). - `column_seams.register`: `CrossReferenceIndexAdapter` must carry a callable `discover`, because `branch_session` calls `CrossReferenceIndexAdapter.discover()` (review r4170608011). tests/test_column_seams.py: both refusals, and the module docstring now describes the default scope as the ruling made it, a tile's own documents editable (review r4170608087). Before (897029d8): the two new cases fail. Mutant killed: skip the shape check, 2 failed. The whole suite, locally: 3114 passed, 177 skipped. Lane: openxfactory-4 (openXfactory-4-openDox_extraction) Co-Authored-By: Claude Opus 5.5 (1M context) --- src/opendox/column_seams.py | 25 +++++++++++++++++++++-- src/opendox/projection_seams.py | 36 ++++++++++++++++++++++++++++++--- tests/test_column_seams.py | 31 +++++++++++++++++++++++++--- 3 files changed, 84 insertions(+), 8 deletions(-) diff --git a/src/opendox/column_seams.py b/src/opendox/column_seams.py index fd4c6f8b..05d6f47d 100644 --- a/src/opendox/column_seams.py +++ b/src/opendox/column_seams.py @@ -133,11 +133,32 @@ #: What a cross-reference register registration must carry. REGISTER_CALLABLES: tuple[str, ...] = ("CrossReferenceIndexAdapter",) +def _gate_shape(registration) -> list[str]: + """`GateRefused` is CAUGHT (`except gate_console.GateRefused`), so it must + be an exception class: a callable that is not one would pass the name + probe and then raise `TypeError` from the first `except` that reads it.""" + refused = getattr(registration, "GateRefused", None) + if isinstance(refused, type) and issubclass(refused, BaseException): + return [] + return ["GateRefused must be an exception class, because openDox's verbs " + "catch it (`except .GateRefused`)"] + + +def _register_shape(registration) -> list[str]: + """openDox calls `CrossReferenceIndexAdapter.discover()`, so the + adapter must carry a callable `discover`, not only be callable itself.""" + adapter = getattr(registration, "CrossReferenceIndexAdapter", None) + if callable(getattr(adapter, "discover", None)): + return [] + return ["CrossReferenceIndexAdapter must carry a callable `discover`, " + "because openDox calls CrossReferenceIndexAdapter.discover()"] + + gate = projection_seams._Seam( "gate", what="gate primitives", callables=GATE_CALLABLES, values=GATE_VALUES, default="opendox.default_columns.GATE", consequence="no gate action can be guarded, stamped or recorded", - module=_MODULE) + module=_MODULE, shape=_gate_shape) scope = projection_seams._Seam( "scope", what="doxBench scope authority", callables=SCOPE_CALLABLES, @@ -156,7 +177,7 @@ "register", what="cross-reference register", callables=REGISTER_CALLABLES, default="opendox.default_columns.REGISTER", consequence="no cross-reference register can be read", - module=_MODULE) + module=_MODULE, shape=_register_shape) def register_defaults() -> None: diff --git a/src/opendox/projection_seams.py b/src/opendox/projection_seams.py index 11e5fdfc..06a62466 100644 --- a/src/opendox/projection_seams.py +++ b/src/opendox/projection_seams.py @@ -83,7 +83,7 @@ import threading from dataclasses import dataclass from pathlib import Path -from typing import Any +from typing import Any, Callable __all__ = [ "CORPUS_ROOT_CALLABLES", @@ -239,14 +239,23 @@ class _Seam: `module` names the module that declares the seam, in every refusal and in the call a host makes. It is this module's own by default, and `opendox.column_seams` declares its four seams through the same class - (plan 034 T084), so every seam of openDox's keeps one discipline.""" + (plan 034 T084), so every seam of openDox's keeps one discipline. + + `shape`, where a seam's consumers rely on more than a name being present + and callable, answers the registration's further defects as sentences, an + empty list for none. A registration with any is refused at registration, + as one lacking a name is, rather than at the first consumer that relies on + it (an exception class a consumer catches, say, or a member of a member it + calls).""" def __init__(self, name: str, *, what: str, callables: tuple[str, ...], values: tuple[str, ...] = (), default: str, consequence: str, - module: str = "opendox.projection_seams") -> None: + module: str = "opendox.projection_seams", + shape: Callable[[Any], list[str]] | None = None) -> None: self.name = name self.module = module + self._shape = shape #: The module's own name without the package, as a refusal names a call. self._short = module.rsplit(".", 1)[-1] self.what = what @@ -282,6 +291,8 @@ def register(self, registration: Any) -> Any: return registration _probe(registration, self.callables, self.values, f"{self._short}.{self.name}.register()", f"the host's {self.what}") + self._probe_shape(registration, f"{self._short}.{self.name}.register()", + f"the host's {self.what}") with self._lock: held = self._registered if held is registration: @@ -326,11 +337,30 @@ def register_default(self, registration: Any) -> Any: _probe(registration, self.callables, self.values, f"{self._short}.{self.name}.register_default()", f"openDox's own default {self.what}") + self._probe_shape(registration, + f"{self._short}.{self.name}.register_default()", + f"openDox's own default {self.what}") with self._lock: if self._registered is None: self._begin(registration, is_default=True) return self._registered + def _probe_shape(self, registration: Any, call: str, what: str) -> None: + """Refuse a registration whose `shape` answers a defect (see the class + docstring). Raised as `_probe` raises, a `TypeError` naming the call.""" + if self._shape is None: + return + try: + defects = list(self._shape(registration)) + except Exception as exc: # noqa: BLE001 - a shape it cannot show is a defect + raise TypeError( + f"{call} takes {what}, and the shape of " + f"{name_of(registration)} could not be read: {exc}") from exc + if defects: + raise TypeError( + f"{call} takes {what}, and {name_of(registration)} " + f"carries the names but not their shape: {'; '.join(defects)}.") + def unregister(self) -> None: """Drop the registration, a host's or the default, and its records. For test isolation and for a host tearing down.""" diff --git a/tests/test_column_seams.py b/tests/test_column_seams.py index 6b47313d..cd50a453 100644 --- a/tests/test_column_seams.py +++ b/tests/test_column_seams.py @@ -18,9 +18,13 @@ refuses the governed record functions and `GateConsole` by name, as `GateRecordsNotRegistered` (a `GateRefused`). `gate_records_writable()` is true only with a host's gate. -4. THE SCOPE DEFAULT projects each kind of tile read-only, confines every - path, and answers no session-created path; the kickoff and register defaults - answer nothing dispatched and no possibles. +4. THE SCOPE DEFAULT projects each kind of tile with ITS OWN documents + editable and nothing else (RULED `5961651355`, "Tile's own documents + editable"), confines every path, and answers no session-created path; the + kickoff and register defaults answer nothing dispatched and no possibles. +5. A REGISTRATION THAT HAS THE NAMES BUT NOT THEIR SHAPE is refused at + registration: a gate whose `GateRefused` is not an exception class, and a + register whose adapter carries no callable `discover`. A CREATED FILE: no carve-manifest row (RULED OQ-C). """ @@ -162,6 +166,27 @@ class _Partial: assert "column_seams.scope.register()" in str(caught.value) +def test_a_gate_whose_refusal_is_not_an_exception_class_is_refused( + isolated) -> None: + """`except gate.GateRefused` needs a class: a function would pass the name + probe and raise `TypeError` at the first refusal a verb catches.""" + host = _HostGate() + host.GateRefused = lambda *args: None + with pytest.raises(TypeError, match="GateRefused must be an exception class"): + cs.gate.register(host) + assert cs.gate.is_registered() is False + + +def test_a_register_whose_adapter_cannot_discover_is_refused(isolated) -> None: + class _NoDiscover: + CrossReferenceIndexAdapter = staticmethod(lambda *args: None) + + with pytest.raises(TypeError, match="callable `discover`") as caught: + cs.register.register(_NoDiscover()) + assert "column_seams.register.register()" in str(caught.value) + assert cs.register.is_registered() is False + + def test_gate_records_are_writable_only_with_a_hosts_gate(isolated) -> None: assert cs.gate_records_writable() is False # nothing at all cs.register_defaults() From 28e195b98d6751d22b7fc33013b112cb8a5bd501 Mon Sep 17 00:00:00 2001 From: Brett Heap <1513478+brettheap@users.noreply.github.com> Date: Fri, 2 Oct 2026 23:43:56 +0000 Subject: [PATCH 46/88] Fix round 13: the server is found as the installed distribution's own files, never by import precedence (Copilot review) server_binaries found pixeltable_pgserver with importlib.util.find_spec, which follows sys.path. Under `python -m opendox.cli`, sys.path starts with the working directory, and a corpus checkout is where that runs. So a checkout holding an executable pixeltable_pgserver/pginstall/bin/postgres was run as this install's database server. The carrier is now looked up by distribution name (importlib.metadata) on sys.path without the working directory, whether it appears as '' or as its own absolute path. Both binaries must be: - files the distribution's RECORD lists; - located inside the distribution after symlinks are resolved; - executable. Anything else is the named refusal. Still nothing imports pixeltable_pgserver. Cases in tests_runtime/test_local_lifecycle.py: - the reworked lookup case: no distribution, a RECORD naming no binaries, binaries linked outside it file by file or through a linked bin directory, and binaries that are not executable; - a new working-directory case: an importable pixeltable_pgserver with executable binaries AND a forged .dist-info, both in the working directory, which is also on sys.path as '' and as its path. The server is not taken from there. Six of the eight fail against the find_spec lookup. Four mutants are killed. One is equivalent ('' resolves to the working directory, which the next check drops). A redundant parent-directory check was removed. Lane: openxfactory-4 (openXfactory-4-openDox_extraction) Co-Authored-By: Claude Opus 5.5 (1M context) --- src/opendox/runtime/bundle.py | 72 ++++++++++++++++----- tests_runtime/test_local_lifecycle.py | 93 +++++++++++++++++++++++---- 2 files changed, 136 insertions(+), 29 deletions(-) diff --git a/src/opendox/runtime/bundle.py b/src/opendox/runtime/bundle.py index 18853154..9bc6c025 100644 --- a/src/opendox/runtime/bundle.py +++ b/src/opendox/runtime/bundle.py @@ -19,8 +19,9 @@ THE SERVER'S OWN PACKAGE IS `pixeltable-pgserver` (pyproject.toml's `local` extra; RULED openxFactory#656 `5916000030` item 2), and only its BINARIES are used: PostgreSQL 16's `initdb` and `postgres` from the wheel's `pginstall/bin`, -found by `importlib.util.find_spec` without importing `pixeltable_pgserver` at -all. Its Python manager is deliberately not used. It daemonizes the server +found through the INSTALLED distribution's own file list +(`importlib.metadata`), never by import precedence, and without importing +`pixeltable_pgserver` at all (see `server_binaries`). Its Python manager is deliberately not used. It daemonizes the server through `pg_ctl`, which re-parents it away from this process (against (i)). It shares one server between processes by reference count and stops it from `atexit`, which a SIGTERM never runs (against (iv)). And it may put the socket @@ -68,7 +69,7 @@ import contextlib import ctypes -import importlib.util +from importlib import metadata import os import re import shutil @@ -125,32 +126,73 @@ class BundleRefused(Exception): """ +def _distribution_search_path() -> list[str]: + """`sys.path` without the working directory, which no install is. + + `python -m opendox.cli` and `python -c` put the directory they were + started in at the front of `sys.path` (as `''`, or as its absolute + path), and a corpus repository is exactly where a user runs them from. + """ + try: + here = Path.cwd().resolve() + except OSError: # a working directory since removed + here = None + kept = [] + for entry in sys.path: + if not entry: + continue + try: + if here is not None and Path(entry).resolve() == here: + continue + except (OSError, RuntimeError): + continue + kept.append(entry) + return kept + + def server_binaries() -> Path: """The directory holding the bundled `initdb` and `postgres`, or a refusal. Found WITHOUT importing `pixeltable_pgserver`: its package initializer imports its manager, which this module does not use and which registers an `atexit` handler and reaches for the user's runtime directory. + + AND FOUND AS THE INSTALLED DISTRIBUTION'S OWN FILES, never by import + precedence (Copilot review of openDox-code#69). `importlib.util.find_spec` + follows `sys.path`, whose first entry under `python -m opendox.cli` is + the working directory. So a checkout holding an executable + `pixeltable_pgserver/pginstall/bin/postgres` was run as this install's + database server. The distribution is now looked up by its name + (`importlib.metadata`), on `sys.path` WITHOUT the working directory, and + both binaries must be files its RECORD lists, located inside it. """ - spec = importlib.util.find_spec(SERVER_PACKAGE) - # THE FIRST LOCATION, OR NONE, read without an index, so no path reaches - # a subscript that could raise (SonarCloud S6466 on openDox-code#69). - location = next(iter(spec.submodule_search_locations or ()), None) \ - if spec else None - if location is None: + candidates = list(metadata.distributions( + name=SERVER_DISTRIBUTION, path=_distribution_search_path())) + if not candidates: raise BundleRefused( "the local install's PostgreSQL server is not installed: it " "arrives with the `local` extra, `pip install \"opendox[local]\"` " "(R1Q16 (iii)). A local install brings its own database and never " "borrows one") - binaries = Path(location) / "pginstall" / "bin" - missing = [name for name in ("initdb", "postgres") - if not os.access(binaries / name, os.X_OK)] + distribution = candidates[0] + suffix = ".exe" if os.name == "nt" else "" + listed = {str(entry).replace("\\", "/"): entry + for entry in (distribution.files or ())} + root = Path(distribution.locate_file("")).resolve() + binaries = Path(distribution.locate_file(f"{SERVER_PACKAGE}/pginstall/bin")) + missing = [] + for name in ("initdb", "postgres"): + entry = listed.get(f"{SERVER_PACKAGE}/pginstall/bin/{name}{suffix}") + located = (Path(distribution.locate_file(entry)).resolve() + if entry is not None else None) + if (located is None or not located.is_relative_to(root) + or not os.access(located, os.X_OK)): + missing.append(name) if missing: raise BundleRefused( - f"the `{SERVER_DISTRIBUTION}` package is installed but carries no " - f"executable {' or '.join(missing)} under {binaries}; reinstall " - "the `local` extra") + f"the `{SERVER_DISTRIBUTION}` package is installed but its own " + f"file list carries no executable {' or '.join(missing)} under " + f"{binaries}; reinstall the `local` extra") return binaries diff --git a/tests_runtime/test_local_lifecycle.py b/tests_runtime/test_local_lifecycle.py index 96c8a479..89db724a 100644 --- a/tests_runtime/test_local_lifecycle.py +++ b/tests_runtime/test_local_lifecycle.py @@ -742,23 +742,88 @@ def test_both_classes_together_name_both_reasons() -> None: assert "hunter2" not in message -@pytest.mark.parametrize("found", ["absent", "no-locations"]) +@pytest.mark.parametrize("found", ["absent", "no-binaries", "outside-it", + "dir-outside-it", "not-executable"]) def test_a_missing_server_package_is_the_named_refusal( - monkeypatch: pytest.MonkeyPatch, found: str) -> None: - """No `pixeltable_pgserver` at all, or a spec with no location: both are - the one refusal naming the `local` extra, never an `IndexError`.""" - import importlib.machinery - - spec = None - if found == "no-locations": - spec = importlib.machinery.ModuleSpec(bundle_mod.SERVER_PACKAGE, None, - is_package=True) - spec.submodule_search_locations = [] - monkeypatch.setattr(bundle_mod.importlib.util, "find_spec", - lambda name: spec) + monkeypatch: pytest.MonkeyPatch, tmp_path: Path, found: str) -> None: + """No `pixeltable-pgserver` distribution at all; one whose file list + names no binaries; or one whose listed binaries resolve outside it, + file by file or through a linked `bin` directory; or binaries that are + listed and inside it but cannot be executed. + Each is the one refusal naming the `local` extra, never a traceback, + and nothing is taken from anywhere but the distribution's own files + (Copilot review of #69).""" + root = tmp_path / "site" + bin_dir = root / bundle_mod.SERVER_PACKAGE / "pginstall" / "bin" + bin_dir.mkdir(parents=True) + elsewhere = tmp_path / "elsewhere" + elsewhere.mkdir() + for name in ("initdb", "postgres"): + for directory in (bin_dir, elsewhere): + (directory / name).write_text("#!/bin/sh\nexit 0\n", encoding="utf-8") + (directory / name).chmod(0o755) + if found == "outside-it": + for name in ("initdb", "postgres"): + (bin_dir / name).unlink() + (bin_dir / name).symlink_to(elsewhere / name) + elif found == "dir-outside-it": + shutil.rmtree(bin_dir) + bin_dir.symlink_to(elsewhere) + elif found == "not-executable": + for name in ("initdb", "postgres"): + (bin_dir / name).chmod(0o644) + + class _Distribution: + files = ([] if found == "no-binaries" else + [f"{bundle_mod.SERVER_PACKAGE}/pginstall/bin/{name}" + for name in ("initdb", "postgres")]) + + def locate_file(self, entry): + return root / str(entry) + + monkeypatch.setattr(bundle_mod.metadata, "distributions", + lambda **kwargs: [] if found == "absent" else [_Distribution()]) with pytest.raises(bundle_mod.BundleRefused) as caught: bundle_mod.server_binaries() - assert 'opendox[local]' in str(caught.value), caught.value + message = str(caught.value) + assert ('opendox[local]' if found == "absent" else "reinstall the `local` extra") \ + in message, message + + +def test_the_server_is_never_taken_from_the_working_directory( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + """`python -m opendox.cli` puts the working directory first on + `sys.path`, and a corpus checkout is where it runs (Copilot review of + #69). A checkout holding an importable `pixeltable_pgserver` with + executable binaries, AND a forged `.dist-info` naming them, is still not + where the server comes from: the search path leaves the working + directory out, as `''` and as its own path.""" + shadow = tmp_path / "checkout" + package = shadow / bundle_mod.SERVER_PACKAGE + bin_dir = package / "pginstall" / "bin" + bin_dir.mkdir(parents=True) + (package / "__init__.py").write_text("", encoding="utf-8") + for name in ("initdb", "postgres"): + (bin_dir / name).write_text("#!/bin/sh\necho planted\n", encoding="utf-8") + (bin_dir / name).chmod(0o755) + info = shadow / "pixeltable_pgserver-99.0.dist-info" + info.mkdir() + (info / "METADATA").write_text( + "Metadata-Version: 2.1\nName: pixeltable-pgserver\nVersion: 99.0\n", + encoding="utf-8") + (info / "RECORD").write_text("".join( + f"{bundle_mod.SERVER_PACKAGE}/pginstall/bin/{name},,\n" + for name in ("initdb", "postgres")), encoding="utf-8") + monkeypatch.chdir(shadow) + monkeypatch.syspath_prepend(str(shadow)) + monkeypatch.setattr(bundle_mod.sys, "path", ["", *bundle_mod.sys.path]) + assert str(shadow) not in bundle_mod._distribution_search_path() + assert "" not in bundle_mod._distribution_search_path() + try: + found = bundle_mod.server_binaries() + except bundle_mod.BundleRefused: + return # no carrier installed: nothing taken either + assert not found.resolve().is_relative_to(shadow.resolve()), found # -- peer authentication, hermetic ---------------------------------------------- From dd015d1ce8c86bed360dae98e46ab941b27048d4 Mon Sep 17 00:00:00 2001 From: Brett Heap <1513478+brettheap@users.noreply.github.com> Date: Fri, 2 Oct 2026 23:55:24 +0000 Subject: [PATCH 47/88] T084: `opendox --help` names the installed command and openDox only MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit T099's PyPI writer found the problem. The installed `opendox --help` printed `usage: ideation-dashboard` and `cli.py`'s module docstring, which is openxFactory's pre-carve history ("validates it against the pinned openxFactory validator", `python3 -m ideation_dashboard.cli`). It would be the product's public description of itself on PyPI. T084 is `cli.py`'s last phase-3 writer, so it fixes it here. - `cli.PROG = "opendox"`, the console script `pyproject.toml` installs. - `cli.PARSER_DESCRIPTION` and `cli.PARSER_EPILOG` are neutral and name openDox only. - Two subcommand help lines shown in the same listing lose their internal words. `create` now scaffolds "a new header-compliant document", not "ideation doc". `runtime` (src/opendox/runtime/cli.py) drops "(split-opendox § 3.5)". No open PR touches either line. tests/test_installed_help.py (new) runs the INSTALLED console script, after checking that its entry point is `opendox.cli:main`. It asserts: - `usage: opendox` and `openDox`; - no openxFactory, xFactory, ideation-dashboard, split-opendox or `scripts/` wording; - the parser's prog, description and epilog in process. Before (94dc15e6): 3 failed. The help named ideation-dashboard, ideation_dashboard, openxFactory, scripts/ and split-opendox. Mutants killed: - the prog reverted: 3 failed; - the description back to `__doc__`: 3 failed. The whole suite, locally: 3138 passed, 177 skipped. Lane: openxfactory-4 (openXfactory-4-openDox_extraction) Co-Authored-By: Claude Opus 5.5 (1M context) --- src/opendox/cli.py | 21 +++++++-- src/opendox/runtime/cli.py | 2 +- tests/test_installed_help.py | 84 ++++++++++++++++++++++++++++++++++++ 3 files changed, 103 insertions(+), 4 deletions(-) create mode 100644 tests/test_installed_help.py diff --git a/src/opendox/cli.py b/src/opendox/cli.py index 610720c8..ac9bf92f 100644 --- a/src/opendox/cli.py +++ b/src/opendox/cli.py @@ -1291,6 +1291,21 @@ def _default_home_factory(root): corpus_adapter.CorpusRef(name="home", location=str(root))) +#: THE INSTALLED COMMAND'S OWN NAME AND WORDS (plan 034 T084; found by T099's +#: PyPI writer). `opendox --help` is what a published install prints, so the +#: usage line names the console script `pyproject.toml` installs, `opendox`, +#: and the description and epilog name openDox only. They used to print +#: `usage: ideation-dashboard` and this module's docstring, which is +#: openxFactory's pre-carve history, not a user's help. +PROG = "opendox" +PARSER_DESCRIPTION = ( + "openDox, a document workbench over a corpus of documents: regenerate " + "the corpus's deterministic snapshot, serve it locally and open it in a " + "browser, create and edit its documents, declare the model providers a " + "chat may use, and run the identity and coordination runtime.") +PARSER_EPILOG = "Run `opendox --help` for a command's own options." + + def build_parser(*, subcommand_extensions: tuple = ()) -> argparse.ArgumentParser: """The command line, plus whatever this invocation was ASSEMBLED with. @@ -1354,8 +1369,8 @@ def build_parser(*, subcommand_extensions: tuple = ()) -> argparse.ArgumentParse # R1Q10 (a)): the gate primitives, the doxBench scope, kickoff and # the cross-reference register, the same way. column_seams.register_defaults() - parser = argparse.ArgumentParser(prog="ideation-dashboard", description=__doc__, - formatter_class=argparse.RawDescriptionHelpFormatter) + parser = argparse.ArgumentParser(prog=PROG, description=PARSER_DESCRIPTION, + epilog=PARSER_EPILOG) sub = parser.add_subparsers(dest="command", required=True) gen = sub.add_parser("generate", help="regenerate the deterministic snapshot") @@ -1401,7 +1416,7 @@ def build_parser(*, subcommand_extensions: tuple = ()) -> argparse.ArgumentParse gao.set_defaults(func=cmd_generate_and_open) create = sub.add_parser( - "create", help="scaffold a new header-compliant ideation doc and open it for editing") + "create", help="scaffold a new header-compliant document and open it for editing") create.add_argument("--repo-root", required=True, help="repository to scaffold into") create.add_argument("--area", default=authoring_mod.DEFAULT_AREA, help=f"target ideation area (default: {authoring_mod.DEFAULT_AREA})") diff --git a/src/opendox/runtime/cli.py b/src/opendox/runtime/cli.py index 605f65ad..ad15d616 100644 --- a/src/opendox/runtime/cli.py +++ b/src/opendox/runtime/cli.py @@ -1136,7 +1136,7 @@ def register(subparsers: Any) -> None: """ runtime = subparsers.add_parser( "runtime", - help="the identity and coordination runtime (split-opendox § 3.5)", + help="the identity and coordination runtime", description="Lifecycle verbs for the openDox runtime: FastAPI + " "Postgres holding identity and coordination (RULING Q1), " "OIDC through the Keycloak broker (RULING Q2).") diff --git a/tests/test_installed_help.py b/tests/test_installed_help.py new file mode 100644 index 00000000..6578f296 --- /dev/null +++ b/tests/test_installed_help.py @@ -0,0 +1,84 @@ +"""The installed `opendox --help` names openDox and nothing else (plan 034 +T084; found by T099's PyPI writer). + +`opendox --help` is the first thing a published install prints. It used to +print `usage: ideation-dashboard` and `cli.py`'s module docstring, which is +openxFactory's pre-carve history: "validates it against the pinned openxFactory +validator", `python3 -m ideation_dashboard.cli`, `scripts/ideation_dashboard/`. +On PyPI that text would be the product's own description of itself. + +The case runs the INSTALLED console script, the file `pip` wrote into the +interpreter's scripts directory from `pyproject.toml`'s `[project.scripts]`, +as a user runs it: not `opendox.cli.main()` in process, which would pass +whatever the entry point string said. The suite always runs against an +installed package (`validate.yml` installs it with `pip install -e`), so a +missing script fails the case rather than skipping it. + +A CREATED FILE: no carve-manifest row (RULED OQ-C). +""" + +from __future__ import annotations + +import importlib.metadata +import os +import re +import subprocess +import sys +import sysconfig +from pathlib import Path + +#: Words that name the governed host or the pre-carve package, none of which +#: belongs in the neutral product's own help. +FOREIGN = re.compile(r"openxfactory|xfactory|ideation[-_ ]dashboard|" + r"ideation_dashboard|split-opendox|\bscripts/", + re.IGNORECASE) + + +def _console_script() -> Path: + """The installed `opendox` script, after checking that the entry point + `pip` wrote it from is `opendox.cli:main`.""" + points = [ep for ep in importlib.metadata.entry_points(group="console_scripts") + if ep.name == "opendox"] + assert [ep.value for ep in points] == ["opendox.cli:main"], points + name = "opendox.exe" if os.name == "nt" else "opendox" + for directory in (Path(sysconfig.get_path("scripts")), + Path(sys.executable).parent): + if (directory / name).is_file(): + return directory / name + raise AssertionError( + f"no installed `{name}` script beside {sys.executable}: the suite runs " + "against an installed package (`pip install -e .`)") + + +def _help() -> str: + done = subprocess.run([str(_console_script()), "--help"], + capture_output=True, text=True, timeout=120, + env={**os.environ, "COLUMNS": "100"}) + assert done.returncode == 0, done.stderr + assert done.stderr == "", done.stderr + return done.stdout + + +def test_the_installed_help_names_the_installed_command() -> None: + out = _help() + assert out.startswith("usage: opendox "), out.splitlines()[0] + assert "openDox" in out, out + + +def test_the_installed_help_names_no_host_and_no_pre_carve_package() -> None: + out = _help() + found = sorted({match.group(0) for match in FOREIGN.finditer(out)}) + assert found == [], ( + f"`opendox --help` names {found}, which is not openDox's own " + f"vocabulary:\n{out}") + + +def test_the_parser_carries_the_neutral_name_and_words() -> None: + """The same, in process, so a failure names the attribute that moved.""" + from opendox import cli + parser = cli.build_parser() + assert parser.prog == cli.PROG == "opendox" + assert parser.description == cli.PARSER_DESCRIPTION + assert parser.epilog == cli.PARSER_EPILOG + for text in (cli.PARSER_DESCRIPTION, cli.PARSER_EPILOG): + assert not FOREIGN.search(text), text From bf9bcb08ba7951b2b2fbaef04333e01e5542ec7b Mon Sep 17 00:00:00 2001 From: Brett Heap <1513478+brettheap@users.noreply.github.com> Date: Sat, 3 Oct 2026 00:11:08 +0000 Subject: [PATCH 48/88] Fix round 14: a named platform gate, resolution failures as reasons, no pid behind a refused tree (Copilot review) Three threads from Copilot's review at 21bde1af. - The bundle is a POSIX design, but the carrier ships Windows wheels. There a start failed as a generic AttributeError phase refusal, and status raised one. bundle.unsupported_platform names the missing primitives: os.getuid, O_DIRECTORY, O_NOFOLLOW, os.fchmod, mkdir with dir_fd (read once at import as MKDIR_TAKES_DIR_FD), and socket.AF_UNIX. BundledServer.start and refusal_before_connecting ask it first. It follows the pattern of local_git_adapter's refuse_without_the_no_follow_walk. - In refusal_before_connecting, a symlink loop makes Path.resolve() raise RuntimeError (Python 3.12), and an embedded NUL makes the os calls raise ValueError. Both escaped as the CLI's generic failure. They are now a named reason, beside OSError. - bundle.report printed running_pid(bundle) before the tree was verified. So a postgres/data linked to another live bundle reported that server's pid, while the connection guard refused the same tree. The pid is now reported only where refusal_before_connecting has nothing to say. New cases: - tests_runtime/test_local_lifecycle.py: four missing primitives, each refused by start and by the socket check, with nothing made; a symlink loop answered as a reason; and no pid behind a refused tree. - tests_runtime/test_bundled_postgres.py: the real verbs case gains a linked-data shape, every refused shape reports a null pid, and the running bundle's own status reports its pid. All eight mutants are killed. The unverified-pid mutant is killed by the real linked-data case as well. Lane: openxfactory-4 (openXfactory-4-openDox_extraction) Co-Authored-By: Claude Opus 5.5 (1M context) --- src/opendox/runtime/bundle.py | 68 ++++++++++++++++++++++++-- tests_runtime/test_bundled_postgres.py | 20 ++++++-- tests_runtime/test_local_lifecycle.py | 53 ++++++++++++++++++++ 3 files changed, 131 insertions(+), 10 deletions(-) diff --git a/src/opendox/runtime/bundle.py b/src/opendox/runtime/bundle.py index 9bc6c025..0dbb2e6b 100644 --- a/src/opendox/runtime/bundle.py +++ b/src/opendox/runtime/bundle.py @@ -74,6 +74,7 @@ import re import shutil import signal +import socket import stat import subprocess import sys @@ -126,6 +127,42 @@ class BundleRefused(Exception): """ +#: Whether `os.mkdir` takes `dir_fd` here, read ONCE at import: a case that +#: stands a wrapper in for `os.mkdir` must not read as another platform. +MKDIR_TAKES_DIR_FD = os.mkdir in os.supports_dir_fd + + +def unsupported_platform() -> str | None: + """Why this platform cannot run the local install's bundled server, or `None`. + + The bundle is a POSIX design, and every one of its guarantees rests on a + POSIX primitive. The socket is a Unix socket, and authentication is peer, + by the kernel's uid. The directories are judged by uid and made without + following a link (`os.getuid`, `O_DIRECTORY`, `O_NOFOLLOW`, a `dir_fd` + `mkdir`, `fchmod`). The carrier ships wheels for Windows too, and there + a start failed as a generic `AttributeError` and `status` raised one + (Copilot review of openDox-code#69). So the gap is named first, as + `runtime/local_git_adapter.py`'s `refuse_without_the_no_follow_walk` + names its own. + """ + missing = [name for name, present in ( + ("os.getuid", hasattr(os, "getuid")), + ("os.O_DIRECTORY", hasattr(os, "O_DIRECTORY")), + ("os.O_NOFOLLOW", hasattr(os, "O_NOFOLLOW")), + ("os.fchmod", hasattr(os, "fchmod")), + ("mkdir with dir_fd", MKDIR_TAKES_DIR_FD), + ("socket.AF_UNIX", hasattr(socket, "AF_UNIX")), + ) if not present] + if not missing: + return None + return (f"the local install's bundled PostgreSQL server needs a POSIX " + f"platform, and this one ({sys.platform}) lacks " + f"{', '.join(missing)}: its socket is a Unix socket authenticated " + "by peer, and its directories are judged by owner and made " + "without following a link. Use a hosted install here " + f"({PREFIX}INSTALL_MODE=hosted, with an operator's database)") + + def _distribution_search_path() -> list[str]: """`sys.path` without the working directory, which no install is. @@ -331,13 +368,21 @@ def refusal_before_connecting(bundle: DatabaseBundle) -> str | None: `/proc` cannot say what the pid is, the other answers still bind the socket to this tree. """ + gap = unsupported_platform() + if gap is not None: + return gap try: refuse_an_unsafe_tree(bundle, existing_only=True) except BundleRefused as exc: return str(exc) - except OSError as exc: + except (OSError, RuntimeError, ValueError) as exc: + # A SYMBOLIC-LINK LOOP OR A NUL IS A REASON TOO (Copilot review of + # openDox-code#69): `Path.resolve()` raises `RuntimeError` for a loop + # (Python 3.12) and the `os` calls `ValueError` for an embedded NUL, + # and either would otherwise escape as the CLI's generic failure. return (f"the bundled server's path under {bundle.state_dir} could " - f"not be judged ({type(exc).__name__})") + f"not be judged ({type(exc).__name__}): it is not a tree " + "this install can verify") lock = bundle.data_dir / "postmaster.pid" try: lines = lock.read_text(encoding="utf-8").splitlines() @@ -365,10 +410,19 @@ def refusal_before_connecting(bundle: DatabaseBundle) -> str | None: def report(bundle: DatabaseBundle) -> dict[str, Any]: - """The `database_bundle` block `runtime status` prints (#1144 13.1).""" + """The `database_bundle` block `runtime status` prints (#1144 13.1). + + THE PID ONLY BEHIND A VERIFIED TREE (Copilot review of openDox-code#69). + `running_pid` reads the lock file through whatever `postgres/data` is, + so a `data` linked to another live bundle answered with THAT server's + pid, while the connection guard refused the same tree. A pid is + reported only where `refusal_before_connecting` has nothing to say, so + `status` never claims a server it would not connect to. + """ + verified = refusal_before_connecting(bundle) is None return {"data_dir": str(bundle.data_dir), "socket_dir": str(bundle.socket_dir), - "pid": running_pid(bundle)} + "pid": running_pid(bundle) if verified else None} def _child_environment() -> dict[str, str]: @@ -742,6 +796,9 @@ def report(self) -> dict[str, Any]: # -- start ------------------------------------------------------------- def start(self) -> BundledServer: + gap = unsupported_platform() + if gap is not None: + raise BundleRefused(gap) if hasattr(os, "geteuid") and os.geteuid() == 0: raise BundleRefused( "the bundled PostgreSQL server refuses to run as root, and so " @@ -1115,5 +1172,6 @@ def __exit__(self, *exc: object) -> None: __all__ = ["BUNDLE_PORT", "BundleRefused", "BundledServer", "authentication_files", "isolated_from_libpq_environment", - "os_user", "report", "running_pid", "server_binaries", + "os_user", "refusal_before_connecting", "refuse_an_unsafe_tree", + "report", "running_pid", "server_binaries", "unsupported_platform", "write_authentication"] diff --git a/tests_runtime/test_bundled_postgres.py b/tests_runtime/test_bundled_postgres.py index 45432e45..1b70aa81 100644 --- a/tests_runtime/test_bundled_postgres.py +++ b/tests_runtime/test_bundled_postgres.py @@ -655,7 +655,8 @@ def _verb(state: Path, *verb: str) -> tuple[int, dict]: return done.returncode, json.loads(done.stdout) -@pytest.mark.parametrize("shape", ["linked-run", "open-state", "no-server"]) +@pytest.mark.parametrize("shape", ["linked-run", "linked-data", "open-state", + "no-server"]) def test_the_local_verbs_connect_only_to_their_own_verified_server( state_dir: Path, shape: str) -> None: """The adversarial review of #69 (L2): a second state tree whose @@ -664,13 +665,19 @@ def test_the_local_verbs_connect_only_to_their_own_verified_server( it as the owner role, against the other bundle's server, while a start refused the same tree. Each verb now asks the tree check and a live server of its OWN data directory before any connection. A valid tree - with no server is answered the same way, by name and unconnected.""" + with no server is answered the same way, by name and unconnected. And + `status` reports no pid for a tree it refuses, a `data` linked to the + running bundle's included (Copilot review of #69).""" settings = config.load_settings({MODE: "local", STATE: str(state_dir)}) other = Path(tempfile.mkdtemp(prefix="odx-o-", dir="/tmp" if os.path.isdir("/tmp") else None)) try: - (other / "postgres" / "data").mkdir(parents=True, mode=0o700) - (other / "postgres").chmod(0o700) - if shape == "no-server": + (other / "postgres").mkdir(mode=0o700) + if shape == "linked-data": + (other / "postgres" / "data").symlink_to( + config.DatabaseBundle(state_dir).data_dir) + else: + (other / "postgres" / "data").mkdir(mode=0o700) + if shape in {"no-server", "linked-data"}: (other / "postgres" / "run").mkdir(mode=0o700) else: (other / "postgres" / "run").symlink_to( @@ -689,9 +696,11 @@ def test_the_local_verbs_connect_only_to_their_own_verified_server( other.chmod(0o700) shutil.rmtree(other, ignore_errors=True) expected = {"linked-run": "is a symbolic link", + "linked-data": "is a symbolic link", "open-state": "writable by every user", "no-server": "no bundled server is running"}[shape] assert status_code == 1 and status["database"].startswith("not probed: "), status + assert status["database_bundle"]["pid"] is None, status["database_bundle"] assert expected in status["database"], status assert migrate_code == 1, migrate assert migrate["refusal"] == "local-bundle-unverified", migrate @@ -699,6 +708,7 @@ def test_the_local_verbs_connect_only_to_their_own_verified_server( assert reset_code == 1 and reset["refusal"] == "local-bundle-unverified", reset assert expected in reset["message"], reset assert own["database"] == "reachable" and own_code == 0, own + assert own["database_bundle"]["pid"] is not None, own["database_bundle"] assert own["applied_migrations"] and not own["pending_migrations"], own diff --git a/tests_runtime/test_local_lifecycle.py b/tests_runtime/test_local_lifecycle.py index 89db724a..134381a3 100644 --- a/tests_runtime/test_local_lifecycle.py +++ b/tests_runtime/test_local_lifecycle.py @@ -1059,6 +1059,59 @@ def test_a_local_verb_connects_only_behind_a_verified_server( assert reason is not None and expected in reason, reason +@pytest.mark.parametrize("lacking", ["O_NOFOLLOW", "getuid", "AF_UNIX", "dir_fd"]) +def test_a_platform_without_the_posix_primitives_is_the_named_refusal( + monkeypatch, tmp_path: Path, short_state: Path, lacking: str) -> None: + """The bundle rests on POSIX primitives, and the carrier ships Windows + wheels too (Copilot review of #69). Lacking any one, a start and the + local verbs' socket check both name the gap, never an `AttributeError`.""" + import socket as socket_mod + + if lacking == "AF_UNIX": + monkeypatch.delattr(socket_mod, "AF_UNIX") + elif lacking == "dir_fd": + monkeypatch.setattr(bundle_mod, "MKDIR_TAKES_DIR_FD", False) + else: + monkeypatch.delattr(bundle_mod.os, lacking) + expected = {"dir_fd": "mkdir with dir_fd", "AF_UNIX": "socket.AF_UNIX"}.get( + lacking, f"os.{lacking}") + server = _prepared(monkeypatch, tmp_path, short_state) + with pytest.raises(bundle_mod.BundleRefused) as caught: + server.start() + assert "needs a POSIX platform" in str(caught.value), caught.value + assert expected in str(caught.value), caught.value + assert not (short_state / "postgres").exists(), "made directories first" + reason = bundle_mod.refusal_before_connecting(config.DatabaseBundle(short_state)) + assert reason is not None and expected in reason, reason + + +def test_a_symlink_loop_in_the_state_path_is_a_reason_not_a_traceback( + tmp_path: Path, short_state: Path) -> None: + """`Path.resolve()` raises for a symbolic-link loop (`RuntimeError` on + Python 3.12), and the local verbs' socket check must still answer with + a reason (Copilot review of #69).""" + loop = short_state / "loop" + loop.symlink_to(loop) + reason = bundle_mod.refusal_before_connecting( + config.DatabaseBundle(loop / "state")) + assert reason is not None, reason + assert "could not be judged" in reason or "symbolic link" in reason, reason + + +def test_status_reports_no_pid_behind_a_tree_it_refuses( + monkeypatch, short_state: Path) -> None: + """A pid is reported only behind a verified tree (Copilot review of + #69): `running_pid` alone would answer for whatever `postgres/data` is, + another live bundle's included.""" + bundle = _a_tree(short_state) + monkeypatch.setattr(bundle_mod, "running_pid", lambda b: 4242) + monkeypatch.setattr(bundle_mod, "refusal_before_connecting", + lambda b: "refused, for the case") + assert bundle_mod.report(bundle)["pid"] is None + monkeypatch.setattr(bundle_mod, "refusal_before_connecting", lambda b: None) + assert bundle_mod.report(bundle)["pid"] == 4242 + + def test_an_initdb_that_dies_midway_leaves_no_data_directory( monkeypatch, tmp_path: Path, short_state: Path) -> None: """The half-built cluster: `PG_VERSION` written, then the run fails. The From 058d96efb8739ae3d3900a88959514d1ada33413 Mon Sep 17 00:00:00 2001 From: Brett Heap <1513478+brettheap@users.noreply.github.com> Date: Sat, 3 Oct 2026 00:29:50 +0000 Subject: [PATCH 49/88] Fix round 15: on Linux the parent-death signal is armed, or the server is not started (Copilot review) ctypes reports a failed prctl() by returning -1, never by raising (a seccomp filter that denies it, say). _die_with_parent's child ignored that result, so the launch succeeded without PDEATHSIG, and a killed entry point could orphan the server despite R1Q16 (iv). - The child now raises when prctl(PR_SET_PDEATHSIG) returns nonzero. subprocess re-raises that in the parent as SubprocessError, which _launch turns into a named refusal ("could not be given its parent-death signal"). No server process exists after it. - A Linux C library with no prctl at all is the same refusal, raised before the launch, where it used to fall back silently to no signal. - Other platforms are unchanged. There the ordinary stop() is the whole of (iv), as before. New case (Linux): a stand-in C library whose prctl returns -1. The start is the named refusal, server.process is None, and the stand-in postgres never ran. Both mutants are killed: the result ignored, and the launch failure left unnamed. Lane: openxfactory-4 (openXfactory-4-openDox_extraction) Co-Authored-By: Claude Opus 5.5 (1M context) --- src/opendox/runtime/bundle.py | 23 +++++++++++++++++++++-- tests_runtime/test_local_lifecycle.py | 27 +++++++++++++++++++++++++++ 2 files changed, 48 insertions(+), 2 deletions(-) diff --git a/src/opendox/runtime/bundle.py b/src/opendox/runtime/bundle.py index 0dbb2e6b..b4d3d50b 100644 --- a/src/opendox/runtime/bundle.py +++ b/src/opendox/runtime/bundle.py @@ -476,23 +476,39 @@ def _die_with_parent(): `prctl` is RESOLVED HERE, in the parent, so the forked child only calls it; and the child re-checks its parent afterwards, because a parent that died between the fork and the `prctl` would never deliver the signal. + + ON LINUX THE SIGNAL IS ARMED OR THE SERVER IS NOT STARTED (Copilot review + of openDox-code#69). `ctypes` reports a failed `prctl` by its `-1` + return, never by raising (a seccomp filter that denies it, say), and + ignoring that left a server that outlives an entry point killed + outright. So a nonzero return raises in the child, which `subprocess` + raises in this process as `SubprocessError` (`_launch` names it), and a + Linux C library with no `prctl` at all is the same refusal here. """ if not sys.platform.startswith("linux"): return None try: prctl = ctypes.CDLL(None, use_errno=True).prctl except (OSError, AttributeError): # pragma: no cover - a libc without it - return None + raise BundleRefused(_UNARMED) from None parent = os.getpid() def _preexec() -> None: # pragma: no cover - runs in the child - prctl(_PR_SET_PDEATHSIG, int(signal.SIGINT)) + if prctl(_PR_SET_PDEATHSIG, int(signal.SIGINT)) != 0: + raise OSError(ctypes.get_errno(), "prctl(PR_SET_PDEATHSIG) failed") if os.getppid() != parent: os._exit(1) return _preexec +#: The refusal when the parent-death signal cannot be armed on Linux. +_UNARMED = ("the bundled PostgreSQL server could not be given its " + "parent-death signal (prctl PR_SET_PDEATHSIG failed), so it would " + "outlive an entry point killed outright (R1Q16 (iv)); it is not " + "started") + + #: The install's own two directories under its state directory, the socket's #: parent and the socket directory (`config.BUNDLE_SOCKET_DIR`). BUNDLE_TREE = BUNDLE_SOCKET_DIR.parts @@ -1026,6 +1042,9 @@ def _launch(self, binaries: Path) -> None: # in order, after the document server has closed. start_new_session=True, preexec_fn=_die_with_parent()) + except subprocess.SubprocessError: + # `_die_with_parent`'s child refused to run unarmed: nothing started + raise BundleRefused(_UNARMED) from None finally: log.close() diff --git a/tests_runtime/test_local_lifecycle.py b/tests_runtime/test_local_lifecycle.py index 134381a3..5841bd08 100644 --- a/tests_runtime/test_local_lifecycle.py +++ b/tests_runtime/test_local_lifecycle.py @@ -1180,6 +1180,33 @@ def test_a_launch_that_cannot_exec_is_the_named_refusal( assert server.process is None +@pytest.mark.skipif(not sys.platform.startswith("linux"), + reason="PR_SET_PDEATHSIG is Linux's") +def test_a_parent_death_signal_that_cannot_be_armed_starts_no_server( + monkeypatch, tmp_path: Path, short_state: Path) -> None: + """`ctypes` reports a failed `prctl` by returning `-1` (a seccomp denial, + say), never by raising, and a server started anyway would outlive an + entry point killed outright (Copilot review of #69). The stand-in C + library's `prctl` fails; the launch is the named refusal, and no server + process exists.""" + launched = tmp_path / "launched" + + class _Libc: + @staticmethod + def prctl(*args): + return -1 + + monkeypatch.setattr(bundle_mod.ctypes, "CDLL", lambda *a, **k: _Libc()) + server = _server(monkeypatch, tmp_path, short_state, + initdb=f'{_target_of_initdb()}\necho 16 > "$T/PG_VERSION"', + postgres=f'touch "{launched}"\nsleep 30') + with pytest.raises(bundle_mod.BundleRefused) as caught: + server.start() + assert "parent-death signal" in str(caught.value), caught.value + assert server.process is None + assert not launched.exists(), "the server ran without its parent-death signal" + + def test_directories_it_cannot_make_are_the_named_refusal( monkeypatch, tmp_path: Path, short_state: Path) -> None: if hasattr(os, "geteuid") and os.geteuid() == 0: # pragma: no cover From ff70ac1e57765ff182afa9b4874ef8e6c721d84f Mon Sep 17 00:00:00 2001 From: Brett Heap <1513478+brettheap@users.noreply.github.com> Date: Sat, 3 Oct 2026 00:54:54 +0000 Subject: [PATCH 50/88] T084 fix round 2: a gate verb on the default gate is refused, not a traceback (Copilot review) openDox's own gate default refuses the governed `GateConsole` at construction (`GateRecordsNotRegistered`, a `GateRefused`). `cli._commission_cli`, the shared half every contributed gate verb runs, built the console before its `try`. So a gate verb reaching the default with no host's gate registered ended in an uncaught traceback instead of " refused: ..." (review r4170914922). The console is now built inside the refusal boundary. The human gate (`_human_gate`, with its actor refusal, which `main` renders) stays outside it, as before. tests/test_column_seams.py::test_a_gate_verb_on_the_default_gate_is_refused_not_a_traceback runs `_commission_cli` over the default gate. It asserts exit status 1 and "propose refused: ...", naming `opendox.column_seams.gate.register(`. Before (3387293e): the case fails with an uncaught GateRecordsNotRegistered. The whole suite, locally: 3223 passed, 177 skipped. Lane: openxfactory-4 (openXfactory-4-openDox_extraction) Co-Authored-By: Claude Opus 5.5 (1M context) --- src/opendox/cli.py | 9 +++++++-- tests/test_column_seams.py | 19 +++++++++++++++++++ 2 files changed, 26 insertions(+), 2 deletions(-) diff --git a/src/opendox/cli.py b/src/opendox/cli.py index a6b82abe..a920d692 100644 --- a/src/opendox/cli.py +++ b/src/opendox/cli.py @@ -957,9 +957,14 @@ def _commission_cli(verb: str, args: argparse.Namespace, target: str, engine, same guards. The CLI adds nothing of its own except the printing — which is exactly what makes the two surfaces equivalent.""" repo_root = Path(args.repo_root).resolve() - console = gate_mod.GateConsole(_human_gate(repo_root, args), - records_dir=args.records_dir) + human = _human_gate(repo_root, args) try: + # INSIDE the refusal boundary (plan 034 T084): openDox's own gate + # default refuses the governed `GateConsole` at construction + # (`GateRecordsNotRegistered`, a `GateRefused`), so a contributed gate + # verb that reaches it with no host's gate registered answers + # " refused: ..." rather than a traceback. + console = gate_mod.GateConsole(human, records_dir=args.records_dir) res = getattr(console, verb.replace("-", "_"))( target, outline=args.outline, workflow=args.workflow, note=args.note, provenance=cli_provenance(), **engine_kwargs) diff --git a/tests/test_column_seams.py b/tests/test_column_seams.py index cd50a453..4c4a9659 100644 --- a/tests/test_column_seams.py +++ b/tests/test_column_seams.py @@ -166,6 +166,25 @@ class _Partial: assert "column_seams.scope.register()" in str(caught.value) +def test_a_gate_verb_on_the_default_gate_is_refused_not_a_traceback( + isolated, tmp_path, capsys) -> None: + """`cli._commission_cli`, the shared half a contributed gate verb runs, + over openDox's own gate default: the governed `GateConsole` refuses at + construction, inside the verb's refusal boundary, so the verb answers + ` refused: ...` and exit status 1 (Copilot review of + openDox-code#77, r4170914922).""" + import argparse + from opendox import cli + cs.register_defaults() + args = argparse.Namespace(repo_root=str(tmp_path), records_dir="records/", + actor="brett", outline=None, workflow=None, + note=None) + assert cli._commission_cli("propose", args, "some-topic") == 1 + err = capsys.readouterr().err + assert err.startswith("propose refused: "), err + assert "opendox.column_seams.gate.register(" in err, err + + def test_a_gate_whose_refusal_is_not_an_exception_class_is_refused( isolated) -> None: """`except gate.GateRefused` needs a class: a function would pass the name From 3ebccb3c4872b180491c88db83afd3612694860c Mon Sep 17 00:00:00 2001 From: Brett Heap <1513478+brettheap@users.noreply.github.com> Date: Sat, 3 Oct 2026 01:09:29 +0000 Subject: [PATCH 51/88] Fix round 16: a local install's served role and database are the bundle's own (Copilot review) Under local, OPENDOX_RUNTIME_PG_ROLE could override the bundle's served role in RuntimeSettings. The bundle bootstraps opendox_runtime with the default DML, and its served DSN connects as opendox_runtime. But the migration run narrows the ledger privileges of exactly the configured role. So OPENDOX_RUNTIME_PG_ROLE=pg_read_all_data let a start succeed while opendox_runtime kept INSERT, UPDATE and DELETE on opendox_schema_migrations, and the ledger's protection was broken. refuse_what_a_local_install_cannot_be, which load_settings, load_migration_settings and generate-and-open --local all ask, now refuses two settings unless they are unset or name the bundle's own: - OPENDOX_RUNTIME_PG_ROLE must be opendox_runtime; - OPENDOX_SERVED_DATABASE must be opendox (the same kind of declaration about the same identity). Anything else is refused by name. Hosted is unchanged. New case: each setting set to another name is refused, and its own name is accepted, through both loaders. Both mutants are killed: the check skipped, and the database not asked. Lane: openxfactory-4 (openXfactory-4-openDox_extraction) Co-Authored-By: Claude Opus 5.5 (1M context) --- src/opendox/runtime/config.py | 28 +++++++++++++++++++++++++++ tests_runtime/test_local_lifecycle.py | 24 +++++++++++++++++++++++ 2 files changed, 52 insertions(+) diff --git a/src/opendox/runtime/config.py b/src/opendox/runtime/config.py index c1d9ea56..b220a6d5 100644 --- a/src/opendox/runtime/config.py +++ b/src/opendox/runtime/config.py @@ -2012,11 +2012,39 @@ def refuse_what_a_local_install_cannot_be(env: Mapping[str, str]) -> None: two must not come to disagree about what a local install is. """ _refuse_hosted_only_settings(env) + _refuse_another_identity_than_the_bundles(env) refuse_a_non_loopback_local_bind( PREFIX + "BIND_HOST", _optional(env, _by_name(PREFIX + "BIND_HOST")) or "127.0.0.1") +def _refuse_another_identity_than_the_bundles(env: Mapping[str, str]) -> None: + """A LOCAL install's served role and database are the bundle's own. + + The bundled server is bootstrapped with `BUNDLE_SERVED_ROLE` as the served + identity, granted the default DML, and its served DSN connects as that + role to `BUNDLE_DATABASE`. `OPENDOX_RUNTIME_PG_ROLE` names the role the + migration run NARROWS on the ledger. So a different one, the existing + `pg_read_all_data` say, let a start succeed while `opendox_runtime` kept + INSERT, UPDATE and DELETE on `opendox_schema_migrations`, and the + ledger's protection was broken (Copilot review of openDox-code#69). + `OPENDOX_SERVED_DATABASE` is the same kind of declaration about the same + identity. Each may be set only to the bundle's own name, and anything + else is refused by name. + """ + for name, own in ((PREFIX + "RUNTIME_PG_ROLE", BUNDLE_SERVED_ROLE), + (PREFIX + "SERVED_DATABASE", BUNDLE_DATABASE)): + given = env.get(name, "").strip() + if given and given != own: + raise ConfigurationError( + f"{name} is {given!r}, and a LOCAL install's is {own!r}: the " + "bundled server is bootstrapped with that served identity, its " + "served DSN connects as it, and the migration run narrows " + "exactly the one this names on the migration ledger. Another " + "would leave the bundle's own served role able to rewrite the " + f"ledger. Unset {name}, or set it to {own!r}") + + def _named(given: list[str]) -> str: return f"{' and '.join(given)} {'are' if len(given) > 1 else 'is'} set" diff --git a/tests_runtime/test_local_lifecycle.py b/tests_runtime/test_local_lifecycle.py index 5841bd08..1d103148 100644 --- a/tests_runtime/test_local_lifecycle.py +++ b/tests_runtime/test_local_lifecycle.py @@ -711,6 +711,30 @@ def test_parent_traversal_in_the_state_path_is_refused(variable: str) -> None: # -- the two refusal classes, each with its own reason -------------------------- +@pytest.mark.parametrize("setting, own, other", [ + ("RUNTIME_PG_ROLE", config.BUNDLE_SERVED_ROLE, "pg_read_all_data"), + ("SERVED_DATABASE", config.BUNDLE_DATABASE, "postgres"), +]) +@pytest.mark.parametrize("loader", ["load_settings", "load_migration_settings"]) +def test_a_served_identity_other_than_the_bundles_is_refused( + setting: str, own: str, other: str, loader: str) -> None: + """`OPENDOX_RUNTIME_PG_ROLE` names the role the migration run narrows on + the ledger, and a local install's served DSN connects as the bundle's own + (Copilot review of #69). Another role, `pg_read_all_data` say, left + `opendox_runtime` able to rewrite `opendox_schema_migrations`. So under + `local` each of the two identity settings may name only the bundle's + own, and the bundle's own name is accepted, as before.""" + name = config.PREFIX + setting + load = getattr(config, loader) + base = {MODE: "local", STATE: "/tmp/odx-identity"} + with pytest.raises(config.ConfigurationError) as caught: + load({**base, name: other}) + assert name in str(caught.value) and repr(own) in str(caught.value), caught.value + settings = load({**base, name: own}) + assert (settings.runtime_pg_role if setting == "RUNTIME_PG_ROLE" + else settings.served_database) == own + + def test_a_dsn_beside_local_is_refused_for_the_database_not_a_broker() -> None: with pytest.raises(config.ConfigurationError) as caught: config.load_settings({MODE: "local", From 56aae2133bab558fcae05869e5e4f639eba1d1bd Mon Sep 17 00:00:00 2001 From: Brett Heap <1513478+brettheap@users.noreply.github.com> Date: Sat, 3 Oct 2026 01:28:03 +0000 Subject: [PATCH 52/88] T103: every loopback route checks the Host (DNS rebinding) (plan 034) Adversarial review 2 (2026-10-03), at openDox-code#77 b1db1965: - M4 (pre-existing): DNS rebinding could read the whole corpus. On a loopback serve, `_serve_source`, `/snapshot.json` and the static bundle applied no Host check, so `Host: evil.example:` got `/source/.md`, the snapshot and every bundle file with 200. - L2: the `/capabilities` Host check ran only when a console token had been minted. With no git identity a local serve mints none, so the install block (data_dir, socket_dir, pid, which name the OS user) went to any Host. The gate is one check at one place, `DashboardHandler.parse_request`, which `handle_one_request` runs before it looks for a `do_`. On a loopback plane every request, whatever its route or method (the static bundle, `/source/*`, `/snapshot.json`, `/capabilities` whatever the token state, `/workbench/*`, every `/actions/*` route, a host's contributed routes, HEAD, OPTIONS and any other method), is refused unless it carries exactly ONE `Host` line naming one of the plane's own loopback authorities at the BOUND port: `127.0.0.1:`, `localhost:` in any case, and `[::1]:` only where the socket is bound to `::1`. Exact match after trimming the optional whitespace around a field value; no suffix or prefix match, no other port, no missing, empty or second Host line. The refusal is one fixed answer that never echoes the Host: 403 with `{"ok": false, "error": "invalid_host", ...}` (`FOREIGN_HOST_BODY`), the code `/capabilities` already used, after draining any declared body (bounded), and with the connection closed. A hosted (non-loopback) plane is untouched. `loopback_authorities` gains an optional `bound_host`; the one-argument form answers what it always did. `_trusted_console_host` and the console's Origin test now read the bound socket's authorities, and the console test refuses a duplicated Host line too. The token-conditional Host test in the `/capabilities` arm is removed: the gate refuses every such request first, and on a hosted plane no token is ever minted. tests/test_loopback_host_gate.py (58 cases): the table of crafted Hosts on the predicate; across every route class of an in-process server with and without a console token, and of a real `generate-and-open --local` serve with and without an identity; the browser path (every bundle file, the JSON routes, a console request with the token and its own Origin, under both spellings); and the mutants the table kills against a live server (each route class exempted, the port ignored, suffix and prefix matches, the first Host line only, a missing Host trusted). Arc: neutral-product-standalone-operability Lane: openxfactory-4 (openXfactory-4-openDox_extraction) Co-Authored-By: Claude Opus 5.5 (1M context) --- src/opendox/serve.py | 184 ++++++++- tests/test_loopback_host_gate.py | 689 +++++++++++++++++++++++++++++++ 2 files changed, 853 insertions(+), 20 deletions(-) create mode 100644 tests/test_loopback_host_gate.py diff --git a/src/opendox/serve.py b/src/opendox/serve.py index cd054f79..f054fa02 100644 --- a/src/opendox/serve.py +++ b/src/opendox/serve.py @@ -80,8 +80,12 @@ `X-Snapshot-Origin`, `X-Snapshot-Generated-At`, `X-Snapshot-Stale`), the transport half of the freshness header the renderer displays (design D11). -Security posture: binds LOOPBACK only by default (127.0.0.1); every write route -is loopback-gated; the source route is read-only and confined PER REGISTRY ENTRY +Security posture: binds LOOPBACK only by default (127.0.0.1); on a loopback +plane EVERY request, whatever its route or method, is refused unless its one +`Host` names this serve's own loopback authority at the bound port, so a +DNS-rebinding page cannot read the corpus through the engineer's browser (plan +034 T103; see `host_names_this_loopback_serve`); every write route is +loopback-gated; the source route is read-only and confined PER REGISTRY ENTRY (an entry with no declared root serves no documents at all); the actions validate their request body and confine every path through the read-side guard before touching the filesystem. @@ -703,20 +707,99 @@ def mint_console_token() -> str: return secrets.token_urlsafe(32) -def loopback_authorities(port: int) -> frozenset[str]: - """Normalized ``Host``/origin authorities for this loopback serve.""" +def loopback_authorities(port: int, bound_host: str | None = None) -> frozenset[str]: + """Normalized ``Host``/origin authorities for this loopback serve. + + `bound_host` is the address the socket is actually bound to + (`server_address[0]`). Given one, the IPv6 literal ``[::1]`` is an + authority only where the socket is bound to ``::1`` (plan 034 T103): a + browser that names ``[::1]`` connects to ``::1``, which an IPv4 bind never + answers, so on that bind no legitimate page names it. ``127.0.0.1`` and + ``localhost`` stay authorities on every loopback bind. With no + `bound_host` the set is every loopback spelling, as it always was.""" + hosts = (LOOPBACK_HOSTS if bound_host is None else + frozenset(host for host in LOOPBACK_HOSTS + if ":" not in host or host == bound_host)) authorities = { f"[{host}]:{port}" if ":" in host else f"{host}:{port}" - for host in LOOPBACK_HOSTS + for host in hosts } if port == 80: authorities |= { f"[{host}]" if ":" in host else host - for host in LOOPBACK_HOSTS + for host in hosts } return frozenset(authorities) +# --------------------------- the loopback Host gate (plan 034 T103) --------------------------- +# +# DNS REBINDING READ THE WHOLE CORPUS (adversarial review 2, 2026-10-03, M4, +# measured at openDox-code#77 `b1db1965`). A page on `evil.example` whose name +# is re-pointed at 127.0.0.1 makes the engineer's own browser send +# `GET /source/.md` with `Host: evil.example:` to this loopback +# socket, and the page may read the answer, because to the browser it is +# same-origin. Only `/capabilities` (and only when a console token had been +# minted) and the console routes asked whether the `Host` named this serve, so +# `/source/*`, `/snapshot.json` and the static bundle answered 200 to any +# `Host`. And L2: with no git identity a local serve mints no token, so +# `/capabilities` skipped its own check and handed the install block (the data +# directory, the socket directory and the bundled server's pid, which name +# the OS user) to any `Host` as well. +# +# THE GATE IS ONE CHECK AT ONE PLACE, ahead of every route: +# `DashboardHandler.parse_request`, which the standard library runs on every +# request before it looks for a `do_` at all. So the static bundle +# (`index.html`, `app.js`, `views/*`), `/source/*`, `/snapshot.json`, +# `/capabilities` whatever the token state, every `/workbench` and +# `/actions` route, a route a host contributes, HEAD, OPTIONS and any other +# method are refused alike, and a route added tomorrow is behind it without +# its author doing anything. +# +# WHAT IT ACCEPTS: exactly one `Host` line naming one of the plane's own +# loopback authorities at the BOUND port (`loopback_authorities`, the same +# set the console's Origin test reads): `127.0.0.1:`, +# `localhost:` in any case, and `[::1]:` only where the socket is +# bound to `::1`. Exact match after trimming the optional whitespace RFC 9110 +# allows around a field value, and lower-casing, which is all `localhost` +# needs. No suffix, no prefix, no other port, no missing or empty `Host`, no +# second `Host` line (RFC 9112 § 3.2 refuses that too). +# +# WHAT IT ANSWERS: one fixed status and one fixed body, built from constants +# below, so a refusal never echoes the `Host` it refused and never says which +# test failed. 403 and `invalid_host`, the code `/capabilities` already +# answered a rebinding `Host` with, so a client that read that refusal reads +# this one. A declared request body is drained, bounded, first, as every other +# refusal here drains one, so the refusal survives its own transport. +# +# A HOSTED PLANE IS UNTOUCHED. Off loopback the bind is `0.0.0.0` behind an +# ingress, whose `Host` is the public name, and the hosted plane's rules are +# its own (`hosted_ref_refused`, the gateway's identity): `parse_request` asks +# `self.loopback` first and does nothing else when it is false. +FOREIGN_HOST_STATUS = 403 +FOREIGN_HOST_ERROR = "invalid_host" +FOREIGN_HOST_MESSAGE = "the request Host does not name this loopback server" +FOREIGN_HOST_BODY = json.dumps({"ok": False, "error": FOREIGN_HOST_ERROR, + "message": FOREIGN_HOST_MESSAGE}).encode("utf-8") + + +def host_names_this_loopback_serve(host_lines, port: int, + bound_host: str | None = None) -> bool: + """Whether a request's `Host` header lines name this loopback serve. + + `host_lines` is every `Host` line the request carried, in order + (`headers.get_all("Host")`, which is None when there is none). True only + for exactly one line whose value, with the surrounding spaces and tabs + trimmed and lower-cased, is one of `loopback_authorities(port, + bound_host)`. Pure, so the table of crafted `Host`s is asserted on it + directly as well as through a live server.""" + lines = list(host_lines or ()) + if len(lines) != 1: + return False + value = str(lines[0]).strip(" \t").lower() + return value in loopback_authorities(port, bound_host) + + # --------------------------- source-path containment (pure) --------------------------- def resolve_source_path(checkout_root: Path, url_tail: str) -> Path | None: @@ -1153,14 +1236,15 @@ def _route(self, head_only: bool) -> bool: self._serve_project_register(head_only) return True if path == CAPABILITIES_ROUTE: - # The loopback console token is process-launch authority. Never - # disclose it to a DNS-rebinding Host, even though the connection - # itself arrived on the loopback socket. - if self.console_token and not self._trusted_console_host(): - self._send_json(403, {"ok": False, "error": "invalid_host", - "message": "the request Host is not this " - "loopback console"}) - return True + # NO HOST TEST OF ITS OWN HERE ANY MORE (plan 034 T103). One stood + # here, run only when a console token had been minted, so a local + # serve with no git identity handed its install block to a + # DNS-rebinding `Host` (adversarial review 2, L2). On a loopback + # plane `parse_request` has refused every such request before any + # route is reached, whatever the token state; on a hosted plane + # no token is ever minted (the `session` capability needs + # loopback), so the old test could not fire there either. + # # THE ONE REPOSITORY THIS SERVE CAN WRITE TO, reported from the # SAME authority a create is refused against (`_session_repository` # — a session lives in one tree, the served checkout), so the @@ -1485,17 +1569,77 @@ def do_POST(self): # noqa: N802 return self._send_error_code(action_errors.ERR_UNKNOWN_ACTION) + # ---- the loopback Host gate (plan 034 T103), ahead of every route ---- + def parse_request(self) -> bool: + """The standard library's request parse, then THE LOOPBACK HOST GATE. + + `BaseHTTPRequestHandler.handle_one_request` calls this for every + request and dispatches to `do_` only when it answers True, so + a refusal here is ahead of every route and every method, the ones a + host contributes included (see the gate's banner beside + `host_names_this_loopback_serve`). `DashboardHandler` is the first + base of every bound class (`route_extension.compose_handler`), so no + contributed mixin comes ahead of it. + + Off loopback it is the standard parse and nothing else: the hosted + plane keeps its own rules.""" + if not super().parse_request(): + return False + if self.loopback and not self._trusted_console_host(): + self._refuse_foreign_host() + return False + return True + + def _refuse_foreign_host(self) -> None: + """The gate's ONE answer: a fixed status and body, never the `Host`. + + The declared body, if any, is drained (bounded) first, as every other + refusal on this surface drains one: answering with bytes unread can + reset the connection under the refusal before the client reads it. The + connection closes after it, so nothing left on the socket is read as + a next request. The server log gets one fixed line, so the refusal is + reported as well as made, and that line names no request-derived + value either.""" + try: + declared = int(self.headers.get("Content-Length") or 0) + except (TypeError, ValueError): + declared = 0 + if declared > 0: + _drain_refused_body(self.rfile, declared) + sys.stderr.write("[loopback] refused a request whose Host does not " + "name this loopback server\n") + self.close_connection = True + self.send_response(FOREIGN_HOST_STATUS) + self.send_header("Content-Type", JSON_CTYPE) + self.send_header("Content-Length", str(len(FOREIGN_HOST_BODY))) + self.send_header("Connection", "close") + self.end_headers() + if self.command != "HEAD": + self.wfile.write(FOREIGN_HOST_BODY) + + def _bound_authorities(self) -> frozenset[str]: + """This serve's own loopback authorities, from the BOUND socket: its + port and its address, never anything the request said.""" + bound = self.server.server_address + return loopback_authorities(int(bound[1]), str(bound[0])) + # ---- the human-console test (FR-019's third clause) ---- def _trusted_console_host(self) -> bool: - """Whether ``Host`` names this server's bound loopback port.""" - raw = str(self.headers.get("Host") or "").strip().lower() - return raw in loopback_authorities(int(self.server.server_address[1])) + """Whether the request's ONE ``Host`` line names this server's bound + loopback port (`host_names_this_loopback_serve`). The gate in + `parse_request` asks it of every request on a loopback plane, and the + console test asks it again, so the console's answer never depends on + the gate having run (a hand-built handler).""" + bound = self.server.server_address + return host_names_this_loopback_serve( + self.headers.get_all("Host"), int(bound[1]), str(bound[0])) def _own_origin_authorities(self) -> set[str]: """The `host:port` spellings a request from THIS serve's own page can - legitimately name. Derived from the bound port, never from the request's - untrusted ``Host``, so DNS rebinding cannot define its own origin.""" - return set(loopback_authorities(int(self.server.server_address[1]))) + legitimately name. Derived from the bound socket, never from the + request's untrusted ``Host``, so DNS rebinding cannot define its own + origin.""" + return set(self._bound_authorities()) def _foreign_origin(self) -> bool: """Whether the caller declares an origin that is not this serve's own. diff --git a/tests/test_loopback_host_gate.py b/tests/test_loopback_host_gate.py new file mode 100644 index 00000000..0192eeb1 --- /dev/null +++ b/tests/test_loopback_host_gate.py @@ -0,0 +1,689 @@ +"""Every loopback route checks the `Host` (plan 034 T103; adversarial review 2, +M4 and L2). + +THE DEFECT, as measured at openDox-code#77 `b1db1965`. DNS rebinding could read +the whole corpus: on a loopback serve, `GET /source/.md` with +`Host: evil.example:` answered 200 with the document's text, and so did +`/snapshot.json` and the static bundle. Only `/capabilities` and the console +routes asked whether the `Host` named this serve, and `/capabilities` asked +only when a console token had been minted (L2): with no git identity a local +serve mints none, so its install block (the data directory, the socket +directory and the bundled server's pid, which name the OS user) went to any +`Host`. + +THE RULE these cases hold (`serve.host_names_this_loopback_serve`, applied in +`DashboardHandler.parse_request`): on a loopback plane every request, whatever +its route or method, is refused unless it carries exactly ONE `Host` line +naming one of the plane's own loopback authorities at the BOUND port: +`127.0.0.1:`, `localhost:` in any case, and `[::1]:` only +where the socket is bound to `::1`. The refusal is one fixed status and body +(`serve.FOREIGN_HOST_STATUS`, `serve.FOREIGN_HOST_BODY`) that never echoes the +`Host`. A hosted plane keeps its own rules. + +1. The table of crafted `Host`s, on the pure predicate: every refused row is + refused and every accepted row accepted, on an IPv4 bind, an IPv6 bind and + port 80. +2. The same table across every route class of an IN-PROCESS server (the static + bundle, `/snapshot.json`, `/source/*`, `/capabilities`, `/workbench/*`, + every `/actions/*` route, a route a host contributes, HEAD, OPTIONS and + other methods), with no console token minted (L2's configuration) and with + one. A large refused body is drained, so the refusal arrives whole. +3. The table across every route class of a REAL standalone + `python -m opendox.cli generate-and-open --local` serve, a child with + neither sibling importable, with no identity and with one. +4. The browser path still works: every file the bundle ships, the JSON routes + and a console request carrying the token and the page's own `Origin`, under + both `Host` spellings a browser sends. +5. The mutants the table must kill, run against a LIVE server: a route class + exempted from the gate (one per class), the port ignored, a suffix match, a + prefix match, only the first `Host` line read, and a missing `Host` + trusted. Each must produce at least one table violation, so a table that + passed would have caught it. +6. A hosted plane (`--host 0.0.0.0`) is unchanged: a `Host` the loopback gate + refuses is served there as before. + +A CREATED FILE: no carve-manifest row (RULED OQ-C). +""" + +from __future__ import annotations + +import http.client +import http.server +import json +import os +import re +import socket +import threading +from pathlib import Path + +import pytest + +from opendox import serve +from route_extension import RouteBinding +from standalone_child import Child, fresh_repository, git, run_module + +ROOT = Path(__file__).resolve().parent.parent +PLAIN = ROOT / "tests" / "fixtures" / "plain-documents" +WEB = ROOT / "src" / "opendox" / "web" +DOCUMENT = "notes-rain-barrel-leak.md" + +_URL = re.compile(r"^(http://([0-9.]+):([0-9]+))/index\.html$") + + +# --------------------------------------------------------------------------- +# the table of crafted Hosts +# --------------------------------------------------------------------------- + +def _other_port(port: int) -> int: + return port + 1 if port < 65535 else port - 1 + + +def accepted_hosts(port: int) -> list[tuple[str, tuple[str, ...]]]: + """`(id, Host lines)` a loopback serve on an IPv4 bind ACCEPTS.""" + return [ + ("ipv4-literal", (f"127.0.0.1:{port}",)), + ("localhost", (f"localhost:{port}",)), + ("LOCALHOST", (f"LOCALHOST:{port}",)), + ("LocalHost", (f"LocalHost:{port}",)), + # optional whitespace around a field value (RFC 9110 § 5.5) + ("surrounding-whitespace", (f" 127.0.0.1:{port}\t",)), + ] + + +def refused_hosts(port: int) -> list[tuple[str, tuple[str, ...]]]: + """`(id, Host lines)` a loopback serve on an IPv4 bind REFUSES. `()` is a + request with no `Host` line at all.""" + good = f"127.0.0.1:{port}" + evil = f"evil.example:{port}" + other = _other_port(port) + return [ + ("evil.example", (evil,)), + ("loopback-literal-prefix", (f"127.0.0.1.evil.example:{port}",)), + ("localhost-prefix", (f"localhost.evil:{port}",)), + ("localhost-suffix", (f"evil.localhost:{port}",)), + ("loopback-literal-suffix", (f"evil127.0.0.1:{port}",)), + ("userinfo-suffix", (f"evil.example@127.0.0.1:{port}",)), + ("other-port", (f"127.0.0.1:{other}",)), + ("localhost-other-port", (f"localhost:{other}",)), + ("port-extended", (f"127.0.0.1:{port}0",)), + ("port-zero-padded", (f"127.0.0.1:0{port}",)), + ("no-port", ("127.0.0.1",)), + ("localhost-no-port", ("localhost",)), + ("empty", ("",)), + ("missing", ()), + ("duplicate-good-then-evil", (good, evil)), + ("duplicate-evil-then-good", (evil, good)), + ("duplicate-good-twice", (good, good)), + ("ipv6-loopback-on-an-ipv4-bind", (f"[::1]:{port}",)), + ("ipv6-mapped-ipv4", (f"[::ffff:127.0.0.1]:{port}",)), + ("ipv6-long-form", (f"[0:0:0:0:0:0:0:1]:{port}",)), + ("ipv6-unbracketed", (f"::1:{port}",)), + ("trailing-dot", (f"localhost.:{port}",)), + ("other-loopback-address", (f"127.0.0.2:{port}",)), + ("short-ipv4", (f"127.1:{port}",)), + ("any-address", (f"0.0.0.0:{port}",)), + ] + + +# --------------------------------------------------------------------------- +# 1 — the predicate +# --------------------------------------------------------------------------- + +PORT = 18772 + + +@pytest.mark.parametrize("label,lines", accepted_hosts(PORT), + ids=[row[0] for row in accepted_hosts(PORT)]) +def test_the_predicate_accepts_the_bound_authorities(label, lines) -> None: + assert serve.host_names_this_loopback_serve(lines, PORT, "127.0.0.1"), label + + +@pytest.mark.parametrize("label,lines", refused_hosts(PORT), + ids=[row[0] for row in refused_hosts(PORT)]) +def test_the_predicate_refuses_every_crafted_host(label, lines) -> None: + assert not serve.host_names_this_loopback_serve( + lines, PORT, "127.0.0.1"), label + + +def test_a_missing_host_line_is_refused_whichever_way_it_is_spelled() -> None: + """`headers.get_all("Host")` answers None for a request with no `Host`.""" + assert not serve.host_names_this_loopback_serve(None, PORT, "127.0.0.1") + assert not serve.host_names_this_loopback_serve((), PORT, "127.0.0.1") + + +def test_ipv6_loopback_is_an_authority_only_where_it_is_bound() -> None: + v6 = (f"[::1]:{PORT}",) + assert serve.host_names_this_loopback_serve(v6, PORT, "::1") + assert not serve.host_names_this_loopback_serve(v6, PORT, "127.0.0.1") + for _label, lines in accepted_hosts(PORT): + assert serve.host_names_this_loopback_serve(lines, PORT, "::1") + assert not serve.host_names_this_loopback_serve( + (f"[::1]:{_other_port(PORT)}",), PORT, "::1") + assert not serve.host_names_this_loopback_serve( + (f"[::ffff:127.0.0.1]:{PORT}",), PORT, "::1") + + +def test_port_80_also_accepts_the_bare_names() -> None: + """A browser omits the default port, so on port 80 the bare names are the + authorities too (unchanged from before T103).""" + for value in ("127.0.0.1", "localhost", "LOCALHOST", "127.0.0.1:80"): + assert serve.host_names_this_loopback_serve((value,), 80, "127.0.0.1") + assert not serve.host_names_this_loopback_serve(("[::1]",), 80, "127.0.0.1") + assert serve.host_names_this_loopback_serve(("[::1]",), 80, "::1") + assert not serve.host_names_this_loopback_serve(("evil.example",), 80, + "127.0.0.1") + + +def test_loopback_authorities_without_a_bound_host_is_unchanged() -> None: + """The one-argument form every existing caller uses still answers every + loopback spelling.""" + assert serve.loopback_authorities(PORT) == frozenset( + {f"127.0.0.1:{PORT}", f"localhost:{PORT}", f"[::1]:{PORT}"}) + assert serve.loopback_authorities(PORT, "127.0.0.1") == frozenset( + {f"127.0.0.1:{PORT}", f"localhost:{PORT}"}) + + +def test_the_refusal_is_fixed_and_names_no_host() -> None: + body = json.loads(serve.FOREIGN_HOST_BODY) + assert serve.FOREIGN_HOST_STATUS == 403 + assert body == {"ok": False, "error": "invalid_host", + "message": serve.FOREIGN_HOST_MESSAGE} + + +# --------------------------------------------------------------------------- +# the route classes, and one raw request +# --------------------------------------------------------------------------- + +def _a_view_module() -> str: + return "/views/" + sorted(p.name for p in (WEB / "views").glob("*.js"))[0] + + +def route_classes(*, contributed: bool) -> list[tuple[str, str, str]]: + """`(id, method, target)`: one request per route class a loopback serve + answers. The contributed rows exist only where the server was built with + `_ProbeRoutes`.""" + rows = [ + ("static-root", "GET", "/"), + ("static-index", "GET", "/index.html"), + ("static-module", "GET", "/app.js"), + ("static-view", "GET", _a_view_module()), + ("static-stylesheet", "GET", "/styles.css"), + ("static-missing", "GET", "/no-such-file.txt"), + ("snapshot", "GET", "/snapshot.json"), + ("snapshot-keyed", "GET", "/snapshot.json?repository=fixture&ref=main"), + ("source", "GET", f"/source/{DOCUMENT}"), + ("source-keyed", "GET", f"/source/fixture@main/{DOCUMENT}"), + ("source-bare", "GET", "/source"), + ("source-traversal", "GET", "/source/../../etc/passwd"), + ("capabilities", "GET", "/capabilities"), + ("project-register", "GET", "/project-register.json"), + ("workbench-model-catalog", "GET", "/workbench/model-catalog"), + ("workbench-model-intake", "GET", "/workbench/model-intake"), + ("workbench-thread", "GET", "/workbench/thread"), + ("head-static", "HEAD", "/index.html"), + ("head-snapshot", "HEAD", "/snapshot.json"), + ("head-source", "HEAD", f"/source/{DOCUMENT}"), + ("head-capabilities", "HEAD", "/capabilities"), + ("options", "OPTIONS", f"/source/{DOCUMENT}"), + ("options-asterisk", "OPTIONS", "*"), + ("put", "PUT", "/snapshot.json"), + ("delete", "DELETE", "/snapshot.json"), + ("post-notebook", "POST", serve.ACTIONS_NOTEBOOK_ROUTE), + ("post-edit", "POST", serve.ACTIONS_EDIT_ROUTE), + ("post-chat-turn", "POST", serve.ACTIONS_WORKBENCH_CHAT_TURN_ROUTE), + ("post-document-abstract", "POST", + serve.ACTIONS_WORKBENCH_DOCUMENT_ABSTRACT_ROUTE), + ("post-model-intake", "POST", serve.ACTIONS_WORKBENCH_MODEL_INTAKE_ROUTE), + ("post-model-approval", "POST", + serve.ACTIONS_WORKBENCH_MODEL_APPROVAL_ROUTE), + ("post-unknown-action", "POST", "/actions/no-such-action"), + ("post-gate-verb", "POST", serve.ACTIONS_GATE_PREFIX + "promote"), + ("post-refresh", "POST", serve.ACTIONS_REFRESH_ROUTE), + ] + if contributed: + rows += [ + ("contributed-get", "GET", "/t103-probe.json"), + ("contributed-get-prefix", "GET", "/t103-prefix/anything"), + ("contributed-head", "HEAD", "/t103-probe.json"), + ("contributed-post", "POST", "/actions/t103-probe"), + ] + return rows + + +#: The route classes whose answer to an ACCEPTED `Host` is a 200 with a body +#: from this serve, so the browser path is asserted to be intact on each. +READABLE = {"static-root", "static-index", "static-module", "static-view", + "static-stylesheet", "snapshot", "snapshot-keyed", "source", + "capabilities", "contributed-get", "contributed-get-prefix"} + + +def _raw(base: tuple[str, int], method: str, target: str, + host_lines: tuple[str, ...], *, body: bytes | None = None, + version: str = "HTTP/1.1") -> tuple[int, dict, bytes, bytes]: + """One request written byte for byte, so a missing, empty or duplicated + `Host` is exactly what the server receives. Answers `(status, headers, + body, raw response)`; the connection is read to its close.""" + if body is None: + body = b"{}" if method == "POST" else b"" + lines = [f"{method} {target} {version}"] + lines += [f"Host: {value}" for value in host_lines] + if body: + lines += ["Content-Type: application/json", + f"Content-Length: {len(body)}"] + lines.append("Connection: close") + data = ("\r\n".join(lines) + "\r\n\r\n").encode("latin-1") + body + with socket.create_connection(base, timeout=30) as connection: + connection.sendall(data) + chunks = [] + while True: + chunk = connection.recv(65536) + if not chunk: + break + chunks.append(chunk) + response = b"".join(chunks) + head, _sep, payload = response.partition(b"\r\n\r\n") + status_line, *header_lines = head.decode("latin-1").split("\r\n") + headers = {} + for line in header_lines: + name, _colon, value = line.partition(":") + headers[name.strip().lower()] = value.strip() + return int(status_line.split(" ")[1]), headers, payload, response + + +def table_violations(base: tuple[str, int], routes, *, accepted=None, + refused=None) -> list[str]: + """Every way the served answers break the rule, over `routes` and the + crafted `Host`s (all of them unless narrowed). Empty means the table + holds.""" + port = base[1] + accepted = accepted_hosts(port) if accepted is None else accepted + refused = refused_hosts(port) if refused is None else refused + found = [] + for route, method, target in routes: + for label, lines in refused: + status, headers, payload, response = _raw(base, method, target, lines) + expected = b"" if method == "HEAD" else serve.FOREIGN_HOST_BODY + if status != serve.FOREIGN_HOST_STATUS or payload != expected: + found.append(f"{route} with Host {label}: {status} " + f"{payload[:80]!r}") + continue + if headers.get("connection", "").lower() != "close": + found.append(f"{route} with Host {label}: the refusal kept " + "the connection open") + text = response.decode("latin-1") + for value in lines: + if value.strip() and value.strip() in text: + found.append(f"{route} with Host {label}: the refusal " + f"echoes {value!r}") + for label, lines in accepted: + status, _headers, payload, _response = _raw(base, method, target, + lines) + if payload == serve.FOREIGN_HOST_BODY: + found.append(f"{route} with Host {label}: refused as foreign") + elif route in READABLE and (status != 200 or not payload): + found.append(f"{route} with Host {label}: {status}, " + f"{len(payload)} bytes") + return found + + +# --------------------------------------------------------------------------- +# an in-process server, with a contributed route beside the core ones +# --------------------------------------------------------------------------- + +class _ProbeColumn: + """A host's contributed column: one exact GET, one prefix GET, one POST.""" + + def _t103_probe(self, head_only): + self._serve_bytes(b'{"probe": "contributed"}', serve.JSON_CTYPE, + head_only) + + def _t103_prefix(self, remainder, head_only): + self._serve_bytes(json.dumps({"rest": remainder}).encode("utf-8"), + serve.JSON_CTYPE, head_only) + + def _t103_post(self): + self._send_json(200, {"probe": "posted"}) + + +class _ProbeRoutes: + HANDLER_CONTRIBUTIONS = (_ProbeColumn,) + + def routes(self): + return (RouteBinding("GET", "/t103-probe.json", False, "_t103_probe"), + RouteBinding("GET", "/t103-prefix/", True, "_t103_prefix"), + RouteBinding("POST", "/actions/t103-probe", False, "_t103_post")) + + +def _clean_environment(monkeypatch) -> None: + """No `GIT_*` and no `XF_*` reaches the server or a child, and no user or + system git configuration: the only identity is the repository's own.""" + for name in list(os.environ): + if name.startswith(("GIT_", "XF_")): + monkeypatch.delenv(name) + monkeypatch.setenv("GIT_CONFIG_GLOBAL", os.devnull) + monkeypatch.setenv("GIT_CONFIG_SYSTEM", os.devnull) + + +@pytest.fixture(scope="module") +def corpora(tmp_path_factory): + """Two fresh repositories over T050's fixture, one carrying a git identity + and one carrying none, each with the snapshot `generate` writes.""" + made = {} + for identity in (False, True): + where = tmp_path_factory.mktemp("identity" if identity else "no-identity") + repo = fresh_repository(PLAIN, where) + if identity: + git(repo, "config", "user.name", "fixture") + git(repo, "config", "user.email", "fixture@example.invalid") + out = where / "out" / "snapshot.json" + child, status = run_module( + where, "opendox.cli", "generate", "--repo-root", str(repo), + "--repository", "fixture", "--output", str(out), "--no-validate") + assert status == 0, child.stderr_text() + made[identity] = (repo, out) + return made + + +@pytest.fixture() +def in_process(corpora, monkeypatch): + """`build(identity=..., host=...)`: a server built in process over one of + the two repositories, with `_ProbeRoutes` contributed, served on a + thread. Answers `(base, capabilities)`.""" + _clean_environment(monkeypatch) + servers = [] + + def build(*, identity: bool, host: str = serve.DEFAULT_HOST): + repo, out = corpora[identity] + httpd = serve.build_server(WEB, out, repo, host=host, port=0, + route_extensions=(_ProbeRoutes(),)) + worker = threading.Thread(target=httpd.serve_forever, daemon=True) + worker.start() + servers.append((httpd, worker)) + base = ("127.0.0.1", httpd.server_address[1]) + status, _headers, payload, _raw_response = _raw( + base, "GET", "/capabilities", (f"127.0.0.1:{base[1]}",)) + assert status == 200, (status, payload[:200]) + return base, json.loads(payload) + + try: + yield build + finally: + for httpd, worker in servers: + httpd.shutdown() + httpd.server_close() + worker.join(timeout=10) + + +# --------------------------------------------------------------------------- +# 2 — every route class of an in-process server +# --------------------------------------------------------------------------- + +@pytest.mark.parametrize("identity", [False, True], + ids=["no-token", "console-token"]) +def test_every_route_class_refuses_a_foreign_host_in_process( + in_process, identity) -> None: + """With no identity no console token is minted, which is the + configuration in which `/capabilities` used to skip its own check (L2).""" + base, caps = in_process(identity=identity) + assert ("console_token" in caps) is identity, sorted(caps) + violations = table_violations(base, route_classes(contributed=True)) + assert violations == [], "\n".join(violations) + + +def test_a_refused_body_is_drained_so_the_refusal_arrives_whole( + in_process) -> None: + """A large declared body behind a foreign `Host`: the refusal is read in + full, never lost to a reset of a socket closed with bytes unread.""" + base, _caps = in_process(identity=False) + body = b'{"pad": "' + b"x" * (1024 * 1024) + b'"}' + for _attempt in range(5): + status, _headers, payload, _response = _raw( + base, "POST", serve.ACTIONS_WORKBENCH_CHAT_TURN_ROUTE, + (f"evil.example:{base[1]}",), body=body) + assert (status, payload) == (serve.FOREIGN_HOST_STATUS, + serve.FOREIGN_HOST_BODY) + + +def test_an_http_1_0_request_with_no_host_is_refused(in_process) -> None: + """HTTP/1.0 lets a client omit `Host`. A loopback serve still answers + only a request that names it.""" + base, _caps = in_process(identity=False) + status, _headers, payload, _response = _raw( + base, "GET", f"/source/{DOCUMENT}", (), version="HTTP/1.0") + assert (status, payload) == (serve.FOREIGN_HOST_STATUS, + serve.FOREIGN_HOST_BODY) + status, _headers, payload, _response = _raw( + base, "GET", f"/source/{DOCUMENT}", (f"127.0.0.1:{base[1]}",), + version="HTTP/1.0") + assert status == 200 and payload == (PLAIN / DOCUMENT).read_bytes() + + +# --------------------------------------------------------------------------- +# 3 — a real standalone `generate-and-open --local` serve +# --------------------------------------------------------------------------- + +@pytest.mark.parametrize("identity", [False, True], + ids=["no-identity", "identity"]) +def test_every_route_class_refuses_a_foreign_host_on_a_real_local_serve( + tmp_path, monkeypatch, identity) -> None: + """`python -m opendox.cli generate-and-open --local --no-open`, a child + with neither sibling importable (`standalone_child.Child`), its own state + directory and its own bundled PostgreSQL server, which stops with it.""" + _clean_environment(monkeypatch) + repo = fresh_repository(PLAIN, tmp_path) + if identity: + git(repo, "config", "user.name", "fixture") + git(repo, "config", "user.email", "fixture@example.invalid") + child = Child(tmp_path, "opendox.cli", "generate-and-open", "--local", + "--repo-root", str(repo), "--repository", "fixture", + "--no-open", "--port", "0", "--run-dir", str(tmp_path / "run")) + try: + match = child.wait_for_line(_URL) + base = (match.group(2), int(match.group(3))) + good = (f"127.0.0.1:{base[1]}",) + status, _headers, payload, _response = _raw(base, "GET", + "/capabilities", good) + assert status == 200, payload[:200] + caps = json.loads(payload) + assert ("console_token" in caps) is identity, sorted(caps) + # L2's block, served to its own Host and to no other + assert caps["install"]["mode"] == "local", caps.get("install") + violations = table_violations(base, route_classes(contributed=False)) + assert violations == [], "\n".join(violations) + assert child.interrupt() == 0, child.stderr_text() + finally: + child.kill() + assert child.refused() == [], child.refused() + assert not child.state_dir.exists(), "the child's state dir outlived it" + + +# --------------------------------------------------------------------------- +# 4 — the browser path still works +# --------------------------------------------------------------------------- + +@pytest.mark.parametrize("spelling", ["127.0.0.1", "localhost"]) +def test_the_browser_path_still_works(in_process, spelling) -> None: + """What a browser opened at `http://:/` sends: `http.client` + writes `Host` from the address it connects to, exactly as a browser writes + it from the URL. Every file the bundle ships answers 200, the JSON routes + answer, and a console request with the token and the page's own `Origin` + reaches its route rather than the gate.""" + base, _caps = in_process(identity=True) + connect = (spelling, base[1]) + + def get(path, headers=None): + connection = http.client.HTTPConnection(*connect, timeout=30) + try: + connection.request("GET", path, headers=headers or {}) + response = connection.getresponse() + return response.status, response.read() + finally: + connection.close() + + shipped = sorted(p.relative_to(WEB).as_posix() + for p in WEB.rglob("*") if p.is_file()) + assert len(shipped) > 30, shipped + for name in shipped: + status, payload = get("/" + name) + assert status == 200 and payload == (WEB / name).read_bytes(), name + status, payload = get("/") + assert status == 200 and b" tuple[str, str]: + host, _colon, port = value.rpartition(":") + return (host, port) if _colon else (value, "") + + +def _port_ignored(lines, port, bound_host=None): + lines = list(lines or ()) + if len(lines) != 1: + return False + host, _port = _authority_split(lines[0].strip(" \t").lower()) + return host in {"127.0.0.1", "localhost"} + + +def _suffix_match(lines, port, bound_host=None): + lines = list(lines or ()) + if len(lines) != 1: + return False + value = lines[0].strip(" \t").lower() + return any(value.endswith(authority) for authority in + serve.loopback_authorities(port, bound_host)) + + +def _prefix_match(lines, port, bound_host=None): + lines = list(lines or ()) + if len(lines) != 1: + return False + host, value_port = _authority_split(lines[0].strip(" \t").lower()) + return (value_port == str(port) + and host.startswith(("127.0.0.1", "localhost"))) + + +def _first_line_only(lines, port, bound_host=None): + lines = list(lines or ()) + return bool(lines) and lines[0].strip(" \t").lower() in \ + serve.loopback_authorities(port, bound_host) + + +def _missing_trusted(lines, port, bound_host=None): + lines = list(lines or ()) + if not lines: + return True + return len(lines) == 1 and lines[0].strip(" \t").lower() in \ + serve.loopback_authorities(port, bound_host) + + +PREDICATE_MUTANTS = { + "port-ignored": _port_ignored, + "suffix-match": _suffix_match, + "prefix-match": _prefix_match, + "first-host-line-only": _first_line_only, + "missing-host-trusted": _missing_trusted, +} + + +@pytest.mark.parametrize("mutant", sorted(ROUTE_CLASS_MUTANTS)) +def test_the_table_kills_a_route_class_exempted_from_the_gate( + in_process, monkeypatch, mutant) -> None: + monkeypatch.setattr(serve.DashboardHandler, "parse_request", + _gate_exempting(ROUTE_CLASS_MUTANTS[mutant])) + base, _caps = in_process(identity=False) + port = base[1] + narrowed = [row for row in refused_hosts(port) + if row[0] in ("evil.example", "missing")] + violations = table_violations(base, route_classes(contributed=True), + accepted=[], refused=narrowed) + assert violations, f"the table did not notice {mutant}" + + +@pytest.mark.parametrize("mutant", sorted(PREDICATE_MUTANTS)) +def test_the_table_kills_a_loosened_host_test(in_process, monkeypatch, + mutant) -> None: + """Each mutant still accepts every accepted row, so it is a plausible + loosening and not a broken server; the refused rows must catch it, on the + predicate and on a live server alike.""" + loosened = PREDICATE_MUTANTS[mutant] + for _label, lines in accepted_hosts(PORT): + assert loosened(lines, PORT, "127.0.0.1"), (mutant, lines) + caught = [label for label, lines in refused_hosts(PORT) + if loosened(lines, PORT, "127.0.0.1")] + assert caught, f"no refused row tells {mutant} apart from the rule" + monkeypatch.setattr(serve, "host_names_this_loopback_serve", loosened) + base, _caps = in_process(identity=False) + violations = table_violations( + base, [("source", "GET", f"/source/{DOCUMENT}")], accepted=[]) + assert violations, f"a live server did not show {mutant}" + + +# --------------------------------------------------------------------------- +# 6 — a hosted plane keeps its own rules +# --------------------------------------------------------------------------- + +def test_a_hosted_plane_is_unchanged(in_process) -> None: + """Off loopback the bind is `0.0.0.0` behind an ingress whose `Host` is + the public name, so the loopback gate does not apply: a `Host` it would + refuse is answered as before.""" + base, caps = in_process(identity=True, host="0.0.0.0") + assert "console_token" not in caps and caps["actions"]["intent"] is True + for host in (f"dashboard.example:{base[1]}", "dashboard.example"): + for target in ("/index.html", "/snapshot.json", f"/source/{DOCUMENT}", + "/capabilities"): + status, _headers, payload, _response = _raw(base, "GET", target, + (host,)) + assert status == 200 and payload != serve.FOREIGN_HOST_BODY, ( + host, target, status) From 5a3b51ee198a452cb56889a2732037ddc7ec5fba Mon Sep 17 00:00:00 2001 From: Brett Heap <1513478+brettheap@users.noreply.github.com> Date: Sat, 3 Oct 2026 01:32:07 +0000 Subject: [PATCH 53/88] T084: openDox's own settings documents are never a tile's editable material (adversarial review 2, M1) The model-provider bindings (`doxbench_binding.DEFAULT_BINDINGS_RELPATH`) and the model declarations (`doxbench_intake.DEFAULT_DECLARATIONS_RELPATH`) live in the checkout, so a tile can name them. Under the tile's-own-documents ruling (5961651355), a group listing one made it editable. A turn's proposal could then rewrite which provider a chat talks to, or approve a pending declaration. - `default_columns.SETTINGS_DOCUMENTS` holds both, imported from their owning modules (stdlib-only at import). - `resolve_scope` moves them out of every owned section into a trailing `settings` section that nothing owns. They stay readable. - `editable_paths`, the one named function, refuses them whatever section carries them. - The corpus scan's own exclusion (openDox-code#76) is a second layer, not the only one. tests/test_column_seams.py: - for each document, a group carrying it keeps it readable, not editable, not a candidate and in no owned section; - `editable_paths` refuses one in an owned section. Before (1d6a4f19): 3 failed. Mutants killed: - the function admits settings: 1 failed; - settings left in owned sections: 2 failed. Lane: openxfactory-4 (openXfactory-4-openDox_extraction) Co-Authored-By: Claude Opus 5.5 (1M context) --- src/opendox/default_columns.py | 53 ++++++++++++++++++++++++++++++++-- tests/test_column_seams.py | 40 +++++++++++++++++++++++++ 2 files changed, 90 insertions(+), 3 deletions(-) diff --git a/src/opendox/default_columns.py b/src/opendox/default_columns.py index 2654c31c..f9ff1e62 100644 --- a/src/opendox/default_columns.py +++ b/src/opendox/default_columns.py @@ -66,6 +66,10 @@ from opendox import defaults from opendox.column_seams import GATE_RECORDS_REFUSAL +# openDox's OWN SETTINGS DOCUMENTS, never a tile's editable material (plan 034 +# T084, adversarial review 2, M1). Both modules are stdlib-only at import. +from opendox.doxbench_binding import DEFAULT_BINDINGS_RELPATH +from opendox.doxbench_intake import DEFAULT_DECLARATIONS_RELPATH from opendox.boundary import GATE_SIDE_EFFECT, BoundaryViolation, HumanGate, Refusal from opendox.doxbench_scope_types import ( ScopeConfinementError, @@ -305,6 +309,46 @@ def _section(key: str, label: str, note: str, paths: Iterable[Any], *, owned=owned, documents=tuple(rows)) +#: The documents an install's own settings live in: the model-provider +#: bindings and the model declarations. They sit in the checkout, so a corpus +#: scan can list them and a tile can name them. They are never a tile's OWN +#: material: a turn that could edit one could rewrite which provider a chat +#: talks to, or approve a pending declaration, through a proposal. So +#: `resolve_scope` keeps them out of every owned section, into a section of +#: their own that is readable and owned by nothing, and `editable_paths` +#: refuses them whatever section carries them. The corpus scan's own exclusion +#: (openDox-code#76) is a second layer, not this one. +SETTINGS_DOCUMENTS: frozenset[str] = frozenset({ + DEFAULT_BINDINGS_RELPATH, DEFAULT_DECLARATIONS_RELPATH}) + +_SETTINGS_SECTION = ("settings", "openDox's own settings documents", + "the install's settings: readable here, and never " + "editable through a tile") + + +def _without_settings(sections: Sequence[ScopeSection]) -> list[ScopeSection]: + """`sections` with every settings document moved out of an OWNED section + into one trailing section that nothing owns, in the order they appeared.""" + kept: list[ScopeSection] = [] + moved: list[ScopeDocument] = [] + for section in sections: + if not section.owned: + kept.append(section) + continue + rows = [row for row in section.documents if row.path not in SETTINGS_DOCUMENTS] + moved.extend(row for row in section.documents + if row.path in SETTINGS_DOCUMENTS) + kept.append(ScopeSection(key=section.key, label=section.label, + note=section.note, inherited=section.inherited, + owned=True, documents=tuple(rows))) + if moved: + key, label, note = _SETTINGS_SECTION + kept.append(ScopeSection(key=key, label=label, note=note, + inherited=False, owned=False, + documents=tuple(moved))) + return kept + + def editable_paths(sections: Sequence[ScopeSection]) -> tuple[str, ...]: """The paths a projected tile lets a turn edit: THE TILE'S OWN DOCUMENTS. @@ -319,14 +363,16 @@ def editable_paths(sections: Sequence[ScopeSection]) -> tuple[str, ...]: (this default records none). openDox's turn guard requires a turn's paths to be in scope AND editable (`doxbench_turns._require_in_scope_and_editable`), so a turn over a tile's own document passes it, and one over any other - document is still refused. ONE named function, so the set is decided in - one place.""" + document is still refused. openDox's own settings documents + (`SETTINGS_DOCUMENTS`) are never editable, whatever section carries them. + ONE named function, so the set is decided in one place.""" editable: list[str] = [] for section in sections: if not section.owned: continue for row in section.documents: - if row.resolved and row.path not in editable: + if (row.resolved and row.path not in editable + and row.path not in SETTINGS_DOCUMENTS): editable.append(row.path) return tuple(editable) @@ -403,6 +449,7 @@ def members(group: Mapping[str, Any]) -> list[Any]: owned=True)) else: return None + sections = _without_settings(sections) context = [row.path for section in sections for row in section.documents if row.resolved] for raw in created: diff --git a/tests/test_column_seams.py b/tests/test_column_seams.py index 4c4a9659..abbb0c0d 100644 --- a/tests/test_column_seams.py +++ b/tests/test_column_seams.py @@ -353,6 +353,46 @@ def test_a_created_path_is_readable_and_never_editable(corpus) -> None: assert projection.editable_paths == ("a.md", "b.md") +@pytest.mark.parametrize("settings", [ + "ideation/dashboard/model-provider-bindings.yaml", + "ideation/dashboard/model-declarations.yaml", +]) +def test_opendoxs_own_settings_documents_are_never_editable( + corpus, settings) -> None: + """Adversarial review 2, M1: a group whose members include one of + openDox's own settings documents does not make it editable or owned. It + stays readable, in a section nothing owns.""" + from opendox import doxbench_binding, doxbench_intake + assert dc.SETTINGS_DOCUMENTS == {doxbench_binding.DEFAULT_BINDINGS_RELPATH, + doxbench_intake.DEFAULT_DECLARATIONS_RELPATH} + target = corpus / settings + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text("schema_version: 1\n", encoding="utf-8") + snapshot = _snapshot() + snapshot["documents"].append({"id": settings, "path": settings}) + snapshot["clusters"][0]["document_edges"].append({"document": settings}) + projection = dc.resolve_scope(snapshot, _key("cluster", "g1"), source_root=corpus) + assert projection.editable_paths == ("a.md", "b.md") + assert settings not in projection.active_document_candidates + assert settings in projection.context_paths, "readable, never editable" + owned = {row.path for section in projection.sections if section.owned + for row in section.documents} + assert settings not in owned + (holder,) = [section for section in projection.sections + if settings in {row.path for row in section.documents}] + assert holder.key == "settings" and holder.owned is False + + +def test_the_editable_set_refuses_a_settings_document_in_any_section() -> None: + """`editable_paths` itself, over an owned section that carries one.""" + from opendox.doxbench_scope_types import ScopeDocument, ScopeSection + section = ScopeSection( + key="k", label="l", note="n", inherited=False, owned=True, + documents=tuple(ScopeDocument(id=p, path=p, resolved=True) for p in + ("a.md", *sorted(dc.SETTINGS_DOCUMENTS)))) + assert dc.editable_paths([section]) == ("a.md",) + + def test_the_editable_set_is_the_owned_sections_resolved_rows() -> None: """`editable_paths` itself: owned sections only, resolved rows only, once each and in order.""" From 3e2d5ac736c1aa20170a583af8440ee52e0e2e61 Mon Sep 17 00:00:00 2001 From: Brett Heap <1513478+brettheap@users.noreply.github.com> Date: Sat, 3 Oct 2026 01:32:17 +0000 Subject: [PATCH 54/88] T104: the console token travels in the opened URL, not /capabilities (plan 034) RULED openxFactory#656 comment 5963851934 ("Token via the opened URL (Recommended)"), from adversarial review 2's M5: a standalone openDox served its per-serve console token from /capabilities to any loopback caller, other OS users of the machine included. - serve.build_server: on a STANDALONE plane (the profile it builds from is openDox's own default_profile) the token is minted as before but is NOT put on /capabilities. A host's plane (openxFactory's) keeps the /capabilities delivery its page and suites read. The server object carries console_token and console_token_delivery for the entry point. - console_access (new): the private copy, /console/.html, mode 0600 in a 0700 tree, created by descriptor and checked as openDox-code#69's bundle checks its tree (no links, not group- or world-writable, this user's; copied, not imported). It is an HTML page that forwards to #console_token=, the FRAGMENT only, and embeds the record as escaped JSON. A planted, linked, hard-linked or loosened copy is refused, on write and on read. - generate-and-open and python -m opendox.serve write the copy, hand the browser its file:// path (a URL given to webbrowser.open sits on a command line every user can read), print the path and never the token, with or without --no-open, refuse when no safe copy can be written, and remove the copy when the server stops. Re-opening the page is opening that file again. - web/views/notebook.js: takes #console_token at import, keeps it in sessionStorage (memory where blocked), strips it with history.replaceState, and probeCapabilities fills it into a payload that carries none, so every view keeps reading caps.console_token and a host's published token wins. - Every route that requires the token still requires it. Tests: tests/test_console_token_delivery.py and tests/test_console_token_view.py. The standalone child tests that took the token from /capabilities now read the private copy (standalone_child.Child.console_token); the web census re-measures views/notebook.js. Arc: neutral-product-standalone-operability Lane: openxfactory-4 (openXfactory-4-openDox_extraction) Co-Authored-By: Claude Opus 5.5 (1M context) --- src/opendox/cli.py | 82 ++- src/opendox/console_access.py | 536 ++++++++++++++++++ src/opendox/serve.py | 85 ++- src/opendox/web/views/notebook.js | 101 +++- tests/fixtures/web_boundary_census.yaml | 6 +- tests/standalone_child.py | 13 + tests/test_capability_honesty.py | 20 +- tests/test_console_token_delivery.py | 711 ++++++++++++++++++++++++ tests/test_console_token_view.py | 194 +++++++ tests/test_doxbench_defaults.py | 9 +- tests/test_neutral_turn_scope.py | 5 +- 11 files changed, 1703 insertions(+), 59 deletions(-) create mode 100644 src/opendox/console_access.py create mode 100644 tests/test_console_token_delivery.py create mode 100644 tests/test_console_token_view.py diff --git a/src/opendox/cli.py b/src/opendox/cli.py index a6b82abe..9f825620 100644 --- a/src/opendox/cli.py +++ b/src/opendox/cli.py @@ -102,6 +102,10 @@ # the driver is imported when the server is started, never here. from opendox.runtime import bundle as bundle_mod # noqa: E402 from opendox.runtime import migrations as migrations_mod # noqa: E402 +# THE CONSOLE TOKEN'S PRIVATE COPY (plan 034 T104): on a standalone plane the +# token is not on `/capabilities`, and `generate-and-open` opens the page +# through a 0600 copy in the state directory instead. Stdlib-only. +from opendox import console_access # noqa: E402 from opendox.boundary import ( # noqa: E402 BoundaryViolation, HumanGate, OutputBoundary, ) @@ -868,34 +872,62 @@ def _generate_and_open(args: argparse.Namespace, *, opener) -> int: # bundled server it started as its child. install_report=_install_report(args)) url = serve_mod.server_url(httpd, "/index.html") - print(f" serving {url}") - print(f" snapshot {serve_mod.server_url(httpd, '/snapshot.json')}") - # The URL is ALWAYS printed on its own line, AND FLUSHED (plan 034 T056). - # Where standard output is a pipe or a file, Python buffers it by block, - # and the process is about to block in `serve_forever()`. So without the - # flush, a wrapper reading this line never sees it while the server runs, - # and it cannot learn an ephemeral port or tell that the server started. - # Measured at openDox-code#59 e3ef506a: zero lines in 20 s on a pipe. - print(url, flush=True) - - if not args.no_open: - try: - opener(url) - except Exception as exc: # a headless box has no browser — never fatal - print(f" (could not open a browser: {exc}; open the URL above manually)") - - if args.no_serve: + # THE CONSOLE TOKEN, ON A STANDALONE PLANE (plan 034 T104; RULED + # openxFactory#656 `5963851934`). `/capabilities` no longer carries it, so + # this entry point writes it into a 0600 private copy in the install's + # state directory, an HTML page that forwards to `url` with the token in + # the FRAGMENT. The browser is handed the copy's PATH, because a URL given + # to `webbrowser.open` sits on a command line every user can read + # (`/proc//cmdline`). The path is printed with or without + # `--no-open`, and the token never is: opening that file again re-opens + # the page. `None` on a host's plane and where no token was minted, and + # then nothing changes. A copy that cannot be written safely refuses the + # run before it serves. + try: + console = console_access.publish(httpd, page_url=url) + except console_access.ConsoleAccessRefused as exc: httpd.server_close() - return 0 - - print(" serving until interrupted (Ctrl-C to stop)", flush=True) + print(f"generate-and-open refused: {exc}", file=sys.stderr) + return 1 try: - httpd.serve_forever() - except KeyboardInterrupt: - pass + print(f" serving {url}") + print(f" snapshot {serve_mod.server_url(httpd, '/snapshot.json')}") + if console is not None: + print(f" console {console.file_url} (this user's private copy, " + "mode 0600: open it to open the console page again)") + # The URL is ALWAYS printed on its own line, AND FLUSHED (plan 034 + # T056). Where standard output is a pipe or a file, Python buffers it + # by block, and the process is about to block in `serve_forever()`. + # So without the flush, a wrapper reading this line never sees it + # while the server runs, and it cannot learn an ephemeral port or tell + # that the server started. Measured at openDox-code#59 e3ef506a: zero + # lines in 20 s on a pipe. It carries no token. + print(url, flush=True) + + if not args.no_open: + try: + opener(console.file_url if console is not None else url) + except Exception as exc: # a headless box has no browser — never fatal + print(f" (could not open a browser: {exc}; open the " + f"{'console file' if console is not None else 'URL'} " + "above manually)") + + if args.no_serve: + httpd.server_close() + return 0 + + print(" serving until interrupted (Ctrl-C to stop)", flush=True) + try: + httpd.serve_forever() + except KeyboardInterrupt: + pass + finally: + httpd.server_close() + return 0 finally: - httpd.server_close() - return 0 + # The copy goes with the server: its token is this serve's, and dies + # with it. + console_access.remove_private_copy(console) # ---- gate console (US9): human-only executable gate actions ---------------- diff --git a/src/opendox/console_access.py b/src/opendox/console_access.py new file mode 100644 index 00000000..eb3d291b --- /dev/null +++ b/src/opendox/console_access.py @@ -0,0 +1,536 @@ +"""How the console token reaches the page on a STANDALONE plane (plan 034 T104). + +THE RULING. openxFactory#656 comment `5963851934` (Brett, 2026-10-03, "Token +via the opened URL (Recommended)"): adversarial review 2's M5 found that a +standalone openDox served its per-serve console token from `/capabilities` to +any loopback caller, other OS users on the same machine included, and nothing +checks which local user connects. The token lets a page edit documents and run +chat turns that spend the operator's model credential. Release 1 changes the +DELIVERY, as Jupyter does, and nothing else: + + * the server stops handing the token out from `/capabilities` on a + standalone plane (`delivery_for`, read by `serve.build_server`); + * the entry point writes a PRIVATE COPY, mode 0600, in openDox's state + directory (`write_private_copy`), and the page is opened through it, so the + token travels only in the opened URL's FRAGMENT (`opened_url`). A fragment + is never sent to a server, so it never reaches a request line, a server + log or a `Referer`; + * the page reads it from `location.hash`, keeps it in `sessionStorage`, and + strips it from the address bar (`web/views/notebook.js`); + * every route that requires the token still requires it. + +WHO IS "STANDALONE". A plane built from openDox's OWN default profile +(`opendox.default_profile`), which an entry point registers where no host has. +A HOST's plane, openxFactory's, keeps the `/capabilities` delivery its page and +its suites read today: the ruling changes the standalone plane, and the +governed one is unchanged by it. + +WHY THE BROWSER IS GIVEN A FILE AND NOT THE URL. `webbrowser.open(url)` runs +`xdg-open url` or the browser with the URL on its command line, and a command +line is readable by every user of the machine (`/proc//cmdline`), for as +long as that process lives. So the URL that carries the token is written into +the private copy, an HTML page that forwards to it, and the browser is handed +the copy's `file://` path. That is Jupyter's own redirect file, and for the +same reason. The start prints the copy's PATH, never the token, with or +without `--no-open`, and opening that file again is how a user re-opens the +page while the server runs. The copy is removed when the server stops. + +THE COPY IS CHECKED THE WAY openDox-code#69's BUNDLE CHECKS ITS TREE +(`opendox.runtime.bundle`: `refuse_an_unsafe_tree`, `_make_private_directories`, +`write_authentication`). The rules are copied here, not imported, because they +are that module's private helpers and its refusal names a socket: + + * the state directory and `console/` must be real directories, this user's, + writable by no one else; every directory above them must be this user's or + root's, and one that others can write must be sticky; every symbolic link + on the configured path must be this user's or root's; + * a missing directory is made relative to its parent's DESCRIPTOR, born + 0700, and opened without following a link before anything is made under + it; + * the file is created exclusively, without following a link, set to exactly + 0600 by its descriptor, and renamed into place. A name already at the + target that is not this user's own regular file (a link, a directory, a + file another user owns, a file with a second hard link) is REFUSED, never + followed or replaced; + * a READ asks all of it again of what exists, and of the file by its + descriptor: a regular file, this user's, exactly 0600, one link. + +A CREATED FILE, with no carve-manifest row (RULED OQ-C). +""" + +from __future__ import annotations + +import contextlib +import dataclasses +import html +import json +import os +import re +import stat +import urllib.parse +from collections.abc import Mapping +from pathlib import Path +from typing import Any + +from opendox.runtime import config as runtime_config + +__all__ = [ + "CONSOLE_DIRNAME", "ConsoleAccessRefused", "DELIVERY_CAPABILITIES", + "DELIVERY_OPENED_URL", "FRAGMENT_KEY", "PrivateCopy", "RECORD_ELEMENT_ID", + "RECORD_KIND", "delivery_for", "opened_url", "private_copy_path", "publish", + "read_private_copy", "remove_private_copy", "write_private_copy", +] + +#: The token rides on `/capabilities`, as a host's plane has always read it. +DELIVERY_CAPABILITIES = "capabilities" +#: The token rides only in the opened URL's fragment (the standalone plane). +DELIVERY_OPENED_URL = "opened-url" + +#: The private copies' directory, under the state directory. +CONSOLE_DIRNAME = "console" +#: The fragment's one key: `…/index.html#console_token=`. The page reads +#: the same key (`web/views/notebook.js`, `CONSOLE_TOKEN_FRAGMENT_KEY`). +FRAGMENT_KEY = "console_token" +#: The machine-readable record inside the copy, for a harness or a script. +RECORD_KIND = "opendox-console-access" +RECORD_SCHEMA_VERSION = 1 +RECORD_ELEMENT_ID = "opendox-console" +#: The one mode a private copy may have. +PRIVATE_MODE = 0o600 +#: A copy is a few hundred bytes; a read stops well past that. +_READ_LIMIT = 64 * 1024 +#: `secrets.token_urlsafe` spells a token in these characters only, so a token +#: needs no escaping in a fragment and a value outside them is not one. +_TOKEN_SHAPE = re.compile(r"[A-Za-z0-9_-]{16,512}") +_RECORD_PATTERN = re.compile( + r'') +_LOOPBACK_HOSTS = frozenset({"127.0.0.1", "::1", "localhost"}) + + +class ConsoleAccessRefused(Exception): + """The private copy cannot be written or read safely, and why.""" + + +@dataclasses.dataclass(frozen=True) +class PrivateCopy: + """One written copy: where it is, what it opens, and which file it is.""" + + path: Path + page_url: str + opened_url: str + #: `(st_dev, st_ino)` of the file this process wrote, so a removal at + #: shutdown removes that file and never one written after it. + identity: tuple[int, int] + + @property + def file_url(self) -> str: + """The `file://` URL a browser is given: a path, never the token.""" + return self.path.as_uri() + + +def delivery_for(profile: Any) -> str: + """Which delivery a plane built from `profile` uses. + + openDox's OWN default profile is the standalone plane: the token travels in + the opened URL. Any other profile is a host's, and keeps `/capabilities`.""" + from opendox import default_profile + + return (DELIVERY_OPENED_URL if profile is default_profile + else DELIVERY_CAPABILITIES) + + +def _refuse_page_url(page_url: str) -> None: + parts = urllib.parse.urlsplit(page_url) + if parts.scheme != "http" or parts.hostname not in _LOOPBACK_HOSTS: + raise ConsoleAccessRefused( + f"the console page {page_url!r} is not a loopback http URL, and a " + "console token is only ever opened on this machine's own plane") + if parts.query or parts.fragment or "#" in page_url or "?" in page_url: + raise ConsoleAccessRefused( + f"the console page {page_url!r} already carries a query or a " + "fragment, so the token's fragment cannot be the only one") + + +def _refuse_token(token: Any) -> str: + if not isinstance(token, str) or not _TOKEN_SHAPE.fullmatch(token): + raise ConsoleAccessRefused("the console token is not a token this " + "server mints") + return token + + +def opened_url(page_url: str, token: str) -> str: + """`page_url` with the token in its FRAGMENT and never in its query. + + A fragment stays in the browser: it is not part of the request line, so no + server log and no `Referer` carries it.""" + _refuse_page_url(page_url) + _refuse_token(token) + return page_url + "#" + urllib.parse.urlencode({FRAGMENT_KEY: token}) + + +def private_copy_path(state_dir: Path | str, port: int) -> Path: + """Where the copy for the plane on `port` lives, under `state_dir`.""" + return Path(state_dir) / CONSOLE_DIRNAME / f"{int(port)}.html" + + +# --------------------------- the tree's rules (bundle.py's, copied) --------------------------- + +def _unsafe_because(info: os.stat_result, *, uid: int, own: bool) -> str | None: + """Why one directory on the copy's path is unsafe, or `None`. + + `opendox.runtime.bundle._unsafe_because`, rule for rule.""" + mode = info.st_mode + if stat.S_ISLNK(mode): + return "is a symbolic link" + if not stat.S_ISDIR(mode): + return "is not a directory" + if own: + if info.st_uid != uid: + return f"is owned by uid {info.st_uid}, not by this user" + if mode & 0o022: + return (f"is writable by {'every user' if mode & 0o002 else 'its group'}" + f" (mode {stat.S_IMODE(mode):o})") + return None + if info.st_uid not in (uid, 0): + return f"is owned by uid {info.st_uid}, neither this user nor root" + if mode & 0o022 and not mode & stat.S_ISVTX: + return (f"is writable by {'every user' if mode & 0o002 else 'its group'}" + f" and is not sticky (mode {stat.S_IMODE(mode):o})") + return None + + +def _unsafe(path: Path, reason: str) -> ConsoleAccessRefused: + return ConsoleAccessRefused( + f"{path} {reason}, so another user could replace or read the console " + "token's private copy. Use a state directory only this user can change " + f"({runtime_config.PREFIX}STATE_DIR)") + + +def _refuse_an_unsafe_tree(state_dir: Path, *, existing_only: bool) -> None: + """The copy's whole path is this user's to change, or it is refused. + + `opendox.runtime.bundle.refuse_an_unsafe_tree`'s three rules, over the + state directory and `console/` instead of the socket's tree.""" + uid = os.getuid() + configured = Path(state_dir) + if not configured.is_absolute() or ".." in configured.parts: + raise ConsoleAccessRefused( + f"the state directory {str(configured)!r} is not an absolute path " + "without `..`, so the copy's path is not the one the kernel walks") + + def present(path: Path) -> bool: + return not existing_only or os.path.lexists(path) + + for component in (configured, *configured.parents): + if not present(component): + continue + info = os.lstat(component) + if stat.S_ISLNK(info.st_mode) and info.st_uid not in (uid, 0): + raise _unsafe(component, f"is a symbolic link owned by uid " + f"{info.st_uid}, neither this user nor root, who " + "could point it elsewhere") + state = configured.resolve() + tree = [state, state / CONSOLE_DIRNAME] + checks = [(path, True) for path in tree] + [ + (path, False) for path in dict.fromkeys( + [*state.parents, *configured.parents])] + for directory, mine in checks: + if not present(directory): + continue + info = os.lstat(directory) if mine else os.stat(directory) + reason = _unsafe_because(info, uid=uid, own=mine) + if reason is not None: + raise _unsafe(directory, reason) + + +def _open_private_directory(leaf: Path, *, state: Path) -> int: + """A descriptor on `leaf`, every missing directory on the way born 0700. + + `opendox.runtime.bundle._make_private_directories`, which it copies: each + missing component is made RELATIVE TO ITS PARENT'S DESCRIPTOR, under a + umask of 077, and opened with `O_NOFOLLOW` before anything is made beneath + it. The directory it starts from is judged by its descriptor first. The + caller owns the descriptor returned.""" + uid = os.getuid() + missing: list[str] = [] + base = leaf + while not os.path.lexists(base): + missing.append(base.name) + base = base.parent + flags = os.O_RDONLY | os.O_DIRECTORY + if not missing: + # It exists: open it without following a link, and judge what opened. + try: + descriptor = os.open(leaf, flags | os.O_NOFOLLOW) + except OSError: + info = os.lstat(leaf) + reason = _unsafe_because(info, uid=uid, own=True) + if reason is None: + raise + raise _unsafe(leaf, reason) from None + reason = _unsafe_because(os.fstat(descriptor), uid=uid, own=True) + if reason is not None: + os.close(descriptor) + raise _unsafe(leaf, reason) + return descriptor + descriptor = os.open(base, flags) + own = base == state or state in base.parents + reason = _unsafe_because(os.fstat(descriptor), uid=uid, own=own) + if reason is not None: + os.close(descriptor) + raise _unsafe(base, reason) + path = base + previous = os.umask(0o077) + try: + for name in reversed(missing): + path = path / name + with contextlib.suppress(FileExistsError): + os.mkdir(name, 0o700, dir_fd=descriptor) + try: + child = os.open(name, flags | os.O_NOFOLLOW, dir_fd=descriptor) + except OSError: + info = os.stat(name, dir_fd=descriptor, follow_symlinks=False) + reason = _unsafe_because(info, uid=uid, own=True) + if reason is None: + raise + raise _unsafe(path, reason) from None + os.close(descriptor) + descriptor = child + reason = _unsafe_because(os.fstat(descriptor), uid=uid, own=True) + if reason is not None: + raise _unsafe(path, reason) + except BaseException: + os.close(descriptor) + raise + finally: + os.umask(previous) + return descriptor + + +def _file_unsafe_because(info: os.stat_result, *, uid: int) -> str | None: + """Why a private copy is not one, or `None`.""" + mode = info.st_mode + if stat.S_ISLNK(mode): + return "is a symbolic link" + if not stat.S_ISREG(mode): + return "is not a regular file" + if info.st_uid != uid: + return f"is owned by uid {info.st_uid}, not by this user" + if info.st_nlink != 1: + return f"has {info.st_nlink} hard links, not one" + if stat.S_IMODE(mode) != PRIVATE_MODE: + return f"has mode {stat.S_IMODE(mode):o}, not {PRIVATE_MODE:o}" + return None + + +# --------------------------- the copy --------------------------- + +def _record_json(record: Mapping[str, Any]) -> str: + """The record as JSON that cannot end its own `` or open a tag.""" + text = json.dumps(dict(record), sort_keys=True, ensure_ascii=True) + return (text.replace("<", "\\u003c").replace(">", "\\u003e") + .replace("&", "\\u0026")) + + +def _opener_html(record: Mapping[str, Any]) -> str: + target = html.escape(str(record["opened_url"]), quote=True) + return ( + "\n" + '\n' + "\n" + '\n' + '\n' + f'\n' + "Opening openDox\n" + f'\n" + "\n" + "\n" + "

This private file opens this user's openDox console. If the page " + f'does not open, open openDox. Keep this file ' + "private: anyone who can read it can act as this console.

\n" + "\n" + "\n") + + +def write_private_copy(state_dir: Path | str, *, page_url: str, port: int, + token: str) -> PrivateCopy: + """Write the copy for the plane on `port`, mode 0600, or refuse. + + Replaces this user's own earlier copy for the same port (a server + restarted there), and refuses anything else already at that name.""" + state = Path(state_dir) + record = { + "schema_version": RECORD_SCHEMA_VERSION, + "kind": RECORD_KIND, + "page_url": page_url, + "opened_url": opened_url(page_url, token), + "port": int(port), + "pid": os.getpid(), + FRAGMENT_KEY: token, + } + _refuse_an_unsafe_tree(state, existing_only=True) + target = private_copy_path(state, port) + directory = _open_private_directory(target.parent, state=state) + uid = os.getuid() + temporary = f".{target.name}.opendox-{os.getpid()}" + try: + # Judged only after the directories exist: what `existing_only` could + # not see before they were made, it sees now. + _refuse_an_unsafe_tree(state, existing_only=False) + try: + present = os.stat(target.name, dir_fd=directory, + follow_symlinks=False) + except FileNotFoundError: + present = None + # THIS USER'S OWN regular file, with one link, is an earlier serve's + # copy for this port, and is replaced. Anything else was PLANTED or + # LINKED there, and is refused, never followed or replaced. + if present is not None and not ( + stat.S_ISREG(present.st_mode) and present.st_uid == uid + and present.st_nlink == 1): + reason = (_file_unsafe_because(present, uid=uid) + or "is not this user's own file") + raise ConsoleAccessRefused( + f"{target} {reason}: something other than this user's own " + "private copy is at that name, so it is refused, never " + "followed or replaced") + with contextlib.suppress(FileNotFoundError): + os.unlink(temporary, dir_fd=directory) # an interrupted start's; a link itself, never its target + handle = os.open(temporary, + os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, + PRIVATE_MODE, dir_fd=directory) + try: + os.fchmod(handle, PRIVATE_MODE) + data = _opener_html(record).encode("utf-8") + view = memoryview(data) + while view: + view = view[os.write(handle, view):] + os.fsync(handle) + written = os.fstat(handle) + finally: + os.close(handle) + try: + os.replace(temporary, target.name, src_dir_fd=directory, + dst_dir_fd=directory) + except BaseException: + with contextlib.suppress(FileNotFoundError): + os.unlink(temporary, dir_fd=directory) + raise + finally: + os.close(directory) + copy = PrivateCopy(path=target, page_url=page_url, + opened_url=record["opened_url"], + identity=(written.st_dev, written.st_ino)) + read_private_copy(target) # what was written is what a reader accepts + return copy + + +def read_private_copy(path: Path | str) -> dict: + """The record in the copy at `path`, or a refusal naming why. + + The tree is judged again, and the file by its own descriptor, opened + without following a link: a regular file, this user's, exactly 0600, with + one link. A planted, linked or loosened copy is refused.""" + target = Path(path) + state = target.parent.parent + if target.parent.name != CONSOLE_DIRNAME: + raise ConsoleAccessRefused(f"{target} is not in a `{CONSOLE_DIRNAME}/` " + "directory of a state directory") + try: + _refuse_an_unsafe_tree(state, existing_only=False) + except FileNotFoundError: + raise ConsoleAccessRefused(f"{target} does not exist: no plane wrote a " + "private copy there") from None + uid = os.getuid() + # NEVER MADE BY A READ: the directory is opened as it is, without + # following a link, and judged by its descriptor. + try: + directory = os.open(target.parent, + os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW) + except OSError: + info = os.lstat(target.parent) + raise _unsafe(target.parent, _unsafe_because(info, uid=uid, own=True) + or "cannot be opened") from None + try: + reason = _unsafe_because(os.fstat(directory), uid=uid, own=True) + if reason is not None: + raise _unsafe(target.parent, reason) + try: + handle = os.open(target.name, os.O_RDONLY | os.O_NOFOLLOW, + dir_fd=directory) + except FileNotFoundError: + raise ConsoleAccessRefused(f"{target} does not exist: no plane on " + "that port wrote a private copy") from None + except OSError: + info = os.stat(target.name, dir_fd=directory, follow_symlinks=False) + reason = _file_unsafe_because(info, uid=uid) or "cannot be opened" + raise ConsoleAccessRefused(f"{target} {reason}") from None + try: + reason = _file_unsafe_because(os.fstat(handle), uid=uid) + if reason is not None: + raise ConsoleAccessRefused( + f"{target} {reason}, so it is not this user's private copy") + data = os.read(handle, _READ_LIMIT) + finally: + os.close(handle) + finally: + os.close(directory) + match = _RECORD_PATTERN.search(data.decode("utf-8", "replace")) + if match is None: + raise ConsoleAccessRefused(f"{target} carries no console record") + try: + record = json.loads(match.group("record")) + except ValueError: + raise ConsoleAccessRefused(f"{target}'s console record is not JSON") from None + if (not isinstance(record, dict) or record.get("kind") != RECORD_KIND + or record.get("schema_version") != RECORD_SCHEMA_VERSION): + raise ConsoleAccessRefused(f"{target}'s console record is not a " + f"{RECORD_KIND} v{RECORD_SCHEMA_VERSION}") + token = _refuse_token(record.get(FRAGMENT_KEY)) + page_url = record.get("page_url") + if not isinstance(page_url, str) or record.get("opened_url") != opened_url( + page_url, token): + raise ConsoleAccessRefused(f"{target}'s console record does not open " + "its own page with its own token") + return record + + +def remove_private_copy(copy: PrivateCopy | None) -> None: + """Remove `copy` when the server stops, if it is still the file written. + + A later serve on the same port writes a file of its own, and that one is + left alone. Never raises: a copy already gone is the goal reached.""" + if copy is None: + return + with contextlib.suppress(OSError): + info = os.lstat(copy.path) + if (info.st_dev, info.st_ino) == copy.identity: + os.unlink(copy.path) + + +def publish(httpd: Any, *, page_url: str, + env: Mapping[str, str] | None = None) -> PrivateCopy | None: + """What an ENTRY POINT does after `serve.build_server`: on a standalone + plane that minted a token, write the private copy into the install's + state directory and return it; otherwise `None`, and nothing is written. + + A state directory that cannot be named, or a tree that is not this user's + alone, refuses (`ConsoleAccessRefused`), and the entry point refuses with + it: a console nobody can open is not served as if it could be.""" + token = getattr(httpd, "console_token", None) + if not token or getattr(httpd, "console_token_delivery", + None) != DELIVERY_OPENED_URL: + return None + try: + state = runtime_config.state_dir(env) + except runtime_config.ConfigurationError as exc: + raise ConsoleAccessRefused( + f"the console token's private copy has no state directory: {exc}" + ) from None + return write_private_copy(state, page_url=page_url, + port=int(httpd.server_address[1]), token=token) diff --git a/src/opendox/serve.py b/src/opendox/serve.py index cd054f79..12940cae 100644 --- a/src/opendox/serve.py +++ b/src/opendox/serve.py @@ -170,6 +170,9 @@ # `opendox.runtime.config` and `opendox.corpus_adapter` besides the stdlib. from opendox import corpus_adapter # noqa: E402 from opendox.runtime import local_git_adapter # noqa: E402 +# THE CONSOLE TOKEN'S DELIVERY on a standalone plane (plan 034 T104): which +# delivery a plane uses, and the private copy an entry point writes. +from opendox import console_access # noqa: E402 # THE GENERATOR SEAM'S DEFAULT (5.4; plan 034 T052): openDox's own snapshot # generator, which `build_server()` and `main()` register where no host has. # Both modules are stdlib-only and name no sibling, so this adds no reach. @@ -676,10 +679,15 @@ def hosted_ref_refused(loopback: bool, ref: str | None) -> bool: # The realization is a HUMAN CONSOLE test, applied to the session verbs before the # body is parsed, exactly where the other two clauses live: # -# 1. a per-serve TOKEN, minted at start-up and published ONLY on -# `/capabilities`. The served page reads it same-origin; a cross-origin page -# cannot read a same-origin JSON response at all, so the drive-by class is -# structurally out. +# 1. a per-serve TOKEN, minted at start-up. On a HOST's plane it is published +# ONLY on `/capabilities`: the served page reads it same-origin, and a +# cross-origin page cannot read a same-origin JSON response at all, so the +# drive-by class is structurally out. On a STANDALONE plane it is not on +# `/capabilities` at all (plan 034 T104; RULED openxFactory#656 +# `5963851934`, adversarial review 2's M5): the page is opened with it in +# the URL's FRAGMENT, through a 0600 private copy in the state directory +# (`console_access`), so another OS user of the machine cannot simply ask +# this loopback server for it. # 2. a same-origin `Origin`/`Referer` when the caller sends one, so a browser # that CAN reach the plane cannot borrow the human's session from another # site. @@ -687,7 +695,8 @@ def hosted_ref_refused(loopback: bool, ref: str | None) -> bool: # # What this HONESTLY does not do, stated so no reader over-reads it: a process # already running as the engineer, on the engineer's own machine, can `GET -# /capabilities` and present the token. Hardening THAT is the xForge host's +# /capabilities` on a host's plane, or read the private copy on a standalone +# one, and present the token. Hardening THAT is the xForge host's # concern (the pre-existing ruling recorded at `cli.py`'s `_human_gate` and D22), # not this local console's. What the check removes is every caller that cannot # demonstrate it came from the console this serve started — which is the whole of @@ -2088,12 +2097,22 @@ def build_server( route_bindings=route_bindings, ) # The human console's per-serve token (FR-019's third clause, review finding - # 2). Minted only where session verbs exist at all, and published on - # `/capabilities` — the one route the served page reads same-origin and no - # cross-origin page can read. + # 2). Minted only where session verbs exist at all. + # + # WHERE IT IS DELIVERED depends on whose plane this is (plan 034 T104; + # RULED openxFactory#656 `5963851934`). On a HOST's plane it is published on + # `/capabilities`, the one route the served page reads same-origin and no + # cross-origin page can read, as it always was. On a STANDALONE plane, + # built from openDox's own default profile, it is NOT: any loopback caller + # can read `/capabilities`, other OS users of the machine included. The + # entry point writes it into a 0600 private copy instead and opens the page + # with it in the URL's fragment (`console_access.publish`). The routes that + # require it require it exactly as before; only the delivery differs. console_token = (mint_console_token() if capabilities["actions"]["session"] else None) - if console_token: + console_delivery = (console_access.delivery_for(domain_profile.current()) + if console_token else None) + if console_token and console_delivery == console_access.DELIVERY_CAPABILITIES: capabilities[CONSOLE_TOKEN_FIELD] = console_token # THE ONE REPOSITORY THIS SERVE CAN WRITE TO. A plane reaching several # repositories serves them all for READING through per-entry source roots, @@ -2273,7 +2292,13 @@ def build_server( # trace on the first live connection. route_extension.resolve_handlers(route_bindings, bound) factory = functools.partial(bound, directory=str(web_dir)) - return http.server.ThreadingHTTPServer((host, port), factory) + httpd = http.server.ThreadingHTTPServer((host, port), factory) + # FOR THE ENTRY POINT, which delivers the token where `/capabilities` does + # not (`console_access.publish`): the token, and which delivery this plane + # uses. Both `None` where no token was minted. + httpd.console_token = console_token + httpd.console_token_delivery = console_delivery + return httpd def server_url(httpd: http.server.ThreadingHTTPServer, path: str = "/") -> str: @@ -2346,18 +2371,33 @@ def serve( checkout_root=checkout_root)) httpd = build_server(web_dir, snapshot_path, checkout_root, host=host, port=port, quiet=quiet, actor=actor, **build_kwargs) - # FLUSHED before the process blocks (plan 034 T056): where standard - # output is a pipe or a file it is block-buffered, so an unflushed line - # never reaches a wrapper while the server runs, and the wrapper cannot - # learn an ephemeral port or tell that the server started. - print(f"serving ideation dashboard at {server_url(httpd, '/index.html')}", - flush=True) + page = server_url(httpd, "/index.html") + # THE CONSOLE TOKEN'S PRIVATE COPY on a standalone plane (plan 034 T104), + # as `cli.cmd_generate_and_open` writes it: its PATH is printed, never the + # token, and it goes when the server does. A copy that cannot be written + # safely refuses the start (`console_access.ConsoleAccessRefused`). try: - httpd.serve_forever() - except KeyboardInterrupt: - pass - finally: + console = console_access.publish(httpd, page_url=page) + except BaseException: httpd.server_close() + raise + try: + if console is not None: + print(f"console {console.file_url} (this user's private copy, " + "mode 0600: open it to open the console page)") + # FLUSHED before the process blocks (plan 034 T056): where standard + # output is a pipe or a file it is block-buffered, so an unflushed line + # never reaches a wrapper while the server runs, and the wrapper cannot + # learn an ephemeral port or tell that the server started. + print(f"serving ideation dashboard at {page}", flush=True) + try: + httpd.serve_forever() + except KeyboardInterrupt: + pass + finally: + httpd.server_close() + finally: + console_access.remove_private_copy(console) def _source_roots_from_args(values) -> dict: @@ -2570,6 +2610,11 @@ def main(argv: list[str] | None = None) -> int: # `--checkout-root` refusal above is. print(f"serve refused: {exc}", file=sys.stderr) return 1 + except console_access.ConsoleAccessRefused as exc: + # NO SAFE PRIVATE COPY, NO SERVE (plan 034 T104): a standalone console + # whose token nobody can be handed is refused, before it serves. + print(f"serve refused: {exc}", file=sys.stderr) + return 1 return 0 diff --git a/src/opendox/web/views/notebook.js b/src/opendox/web/views/notebook.js index 7e8015a8..81a11d5b 100644 --- a/src/opendox/web/views/notebook.js +++ b/src/opendox/web/views/notebook.js @@ -27,16 +27,111 @@ export function notebookCapable(caps) { return caps?.actions?.notebook === true; } +// ---- the console token, DELIVERED IN THE OPENED URL (plan 034 T104) -------- +// +// RULED openxFactory#656 `5963851934` (adversarial review 2's M5): on a +// STANDALONE plane the server no longer publishes the per-serve console token +// on `/capabilities`, where any loopback caller, another OS user included, +// could read it. `opendox generate-and-open` opens this page through a private +// 0600 file instead, which forwards here with the token in the URL FRAGMENT +// (`#console_token=…`). A fragment never reaches a server, so it is in no +// request line, no server log and no `Referer`. +// +// This module takes it the moment it is imported (before the shell's first +// fetch), keeps it in `sessionStorage` (this tab only, so a reload keeps it; +// memory where storage is blocked), and STRIPS the fragment from the address +// bar with `history.replaceState`. `probeCapabilities` then fills it into the +// payload where the server published none, so every view keeps reading +// `caps.console_token` exactly as before. A HOST's plane still publishes its +// own token on `/capabilities`, and that one always wins. +export const CONSOLE_TOKEN_FRAGMENT_KEY = "console_token"; +export const CONSOLE_TOKEN_STORAGE_KEY = "opendox.console-token"; +// The `/capabilities` field every view reads the token from (the server's +// `CONSOLE_TOKEN_FIELD`; `staging-workbench-model.js` spells it too, and this +// leaf module imports nothing). +const CAPS_CONSOLE_TOKEN_FIELD = "console_token"; +// `secrets.token_urlsafe`'s alphabet: a value outside it is not a token. +const CONSOLE_TOKEN_SHAPE = /^[A-Za-z0-9_-]{16,512}$/; + +function consoleStore(scope) { + try { + return scope && scope.sessionStorage ? scope.sessionStorage : null; + } catch { + return null; // blocked storage throws on access + } +} + +// pure over `scope` (`window` in the page) — node-testable. Returns the +// delivered token or null. A fragment that names the key is stripped from the +// address bar whatever it holds, so a malformed token does not linger either. +export function takeDeliveredConsoleToken(scope) { + const loc = scope && scope.location; + if (!loc || typeof loc.hash !== "string") return null; + let fromFragment = null; + if (loc.hash.length > 1) { + let raw = null; + try { + raw = new URLSearchParams(loc.hash.slice(1)).get(CONSOLE_TOKEN_FRAGMENT_KEY); + } catch { + raw = null; + } + if (raw !== null) { + try { + const history = scope.history; + if (history && typeof history.replaceState === "function") { + history.replaceState(history.state, "", + String(loc.pathname || "") + String(loc.search || "")); + } + } catch { + // an address bar that cannot be rewritten still yields the token + } + if (CONSOLE_TOKEN_SHAPE.test(raw)) fromFragment = raw; + } + } + const store = consoleStore(scope); + if (fromFragment) { + try { + if (store) store.setItem(CONSOLE_TOKEN_STORAGE_KEY, fromFragment); + } catch { + // memory only: this load still has it + } + return fromFragment; + } + let kept = null; + try { + kept = store ? store.getItem(CONSOLE_TOKEN_STORAGE_KEY) : null; + } catch { + kept = null; + } + return typeof kept === "string" && CONSOLE_TOKEN_SHAPE.test(kept) ? kept : null; +} + +// Taken ONCE, at import. Outside a page (node) there is no `location`, and +// this is null. +const DELIVERED_CONSOLE_TOKEN = takeDeliveredConsoleToken(globalThis); + +// pure — node-testable. The payload, with the delivered token filled in where +// the server published none. Mutated in place, so the object the shell holds +// is the object the views read. +export function withDeliveredConsoleToken(payload, token = DELIVERED_CONSOLE_TOKEN) { + if (!payload || typeof payload !== "object" || Array.isArray(payload)) return payload; + const served = payload[CAPS_CONSOLE_TOKEN_FIELD]; + if (typeof served === "string" && served) return payload; + if (typeof token === "string" && token) payload[CAPS_CONSOLE_TOKEN_FIELD] = token; + return payload; +} + // Probe the local backend ONCE. Any non-OK response (the static image 404s this // route) or a network failure (file://) degrades to "not available" — never -// throws, never blocks the dashboard. `injectedFetch` is for tests. -export async function probeCapabilities(injectedFetch) { +// throws, never blocks the dashboard. `injectedFetch` is for tests; so is +// `deliveredToken`, which defaults to the one this page was opened with. +export async function probeCapabilities(injectedFetch, deliveredToken = DELIVERED_CONSOLE_TOKEN) { try { const response = injectedFetch ? await injectedFetch(CAPABILITIES_ROUTE, { cache: "no-store" }) : await fetch(CAPABILITIES_ROUTE, { cache: "no-store" }); if (!response?.ok) return { actions: { notebook: false } }; - return await response.json(); + return withDeliveredConsoleToken(await response.json(), deliveredToken); } catch { return { actions: { notebook: false } }; } diff --git a/tests/fixtures/web_boundary_census.yaml b/tests/fixtures/web_boundary_census.yaml index 85677478..00063c15 100644 --- a/tests/fixtures/web_boundary_census.yaml +++ b/tests/fixtures/web_boundary_census.yaml @@ -191,7 +191,7 @@ measured_at: "opensoft/openDox-code main a99eba03e31a0aee1cc15a061fdf718cc88a2c4 # shape and `test_the_declared_totals_are_re_derived_from_the_rows` can refuse a # drift between the two. Measured at slice S4 (see the S4 block above). totals: - A: {files: 26, loc: 18073} + A: {files: 26, loc: 18168} B: {files: 1, loc: 73} C: {files: 14, loc: 12587} "?": {files: 1, loc: 1577} @@ -350,8 +350,8 @@ files: - path: views/notebook.js class: A - loc: 109 - note: "open in NotebookLM tile action; both routes are openDox's serve.py" + loc: 204 + note: "open in NotebookLM tile action; both routes are openDox's serve.py loc 109 -> 204 on plan 034 T104 (RULED openxFactory#656 5963851934): the module also takes the console token from the opened URL's fragment at import, keeps it in sessionStorage, strips it with history.replaceState, and probeCapabilities fills it into a payload that carries none; still import-free, and still no route of another column" - path: views/outline-model.js class: C diff --git a/tests/standalone_child.py b/tests/standalone_child.py index 2956ae25..f0f78996 100644 --- a/tests/standalone_child.py +++ b/tests/standalone_child.py @@ -266,6 +266,19 @@ def refused(self) -> list[str]: return [] return self.refused_log.read_text(encoding="utf-8").split() + def console_token(self, port: int) -> str: + """The console token this child's STANDALONE plane delivers (plan 034 + T104; RULED openxFactory#656 `5963851934`), read as a user's browser + is handed it: from the 0600 private copy the entry point wrote in the + child's own state directory. `/capabilities` carries none on a + standalone plane, so this is the only place a case can take it from, + and reading it applies every check a real reader's does.""" + from opendox import console_access + + record = console_access.read_private_copy( + console_access.private_copy_path(self.state_dir, port)) + return record[console_access.FRAGMENT_KEY] + def run_module(workdir: Path, module: str, *args: str) -> tuple[Child, int]: """A child run to completion: `(child, exit status)`.""" diff --git a/tests/test_capability_honesty.py b/tests/test_capability_honesty.py index 78ec7f5e..7ef83f57 100644 --- a/tests/test_capability_honesty.py +++ b/tests/test_capability_honesty.py @@ -450,7 +450,10 @@ def test_the_three_crash_sites_answer_a_standalone_server( repo = _repository(tmp_path, identity=True) child, base, caps = _standalone(tmp_path, repo) try: - token = caps.get("console_token") + # THE TOKEN IS NOT ON `/capabilities` (plan 034 T104): a standalone + # plane delivers it in the opened URL, through its private copy. + assert "console_token" not in caps, caps + token = child.console_token(base[1]) assert caps["actions"]["session"] is True and token, caps abstract = _call(base, "POST", "/actions/workbench/document-abstract", body=_json(_ABSTRACT), token=token) @@ -493,7 +496,10 @@ def test_the_rails_thread_read_answers_a_standalone_server( repo = _repository(tmp_path, identity=True) child, base, caps = _standalone(tmp_path, repo) try: - token = caps.get("console_token") + # THE TOKEN IS NOT ON `/capabilities` (plan 034 T104): a standalone + # plane delivers it in the opened URL, through its private copy. + assert "console_token" not in caps, caps + token = child.console_token(base[1]) assert caps["actions"]["session"] is True and token, caps status, body, raw = _call( base, "GET", @@ -536,7 +542,10 @@ def test_a_chat_turn_with_a_binding_configured_is_answered_standalone( assert status == 0, added.stderr_text() child, base, caps = _standalone(tmp_path, repo) try: - token = caps.get("console_token") + # THE TOKEN IS NOT ON `/capabilities` (plan 034 T104): a standalone + # plane delivers it in the opened URL, through its private copy. + assert "console_token" not in caps, caps + token = child.console_token(base[1]) assert caps["actions"]["session"] is True and token, caps answer = _call(base, "POST", "/actions/workbench/chat-turn", body=_json(_chat_turn()), token=token) @@ -699,7 +708,10 @@ def test_a_standalone_server_does_not_offer_an_intake_it_could_not_approve( before = document.read_bytes() child, base, caps = _standalone(tmp_path, repo) try: - token = caps.get("console_token") + # THE TOKEN IS NOT ON `/capabilities` (plan 034 T104): a standalone + # plane delivers it in the opened URL, through its private copy. + assert "console_token" not in caps, caps + token = child.console_token(base[1]) assert caps["actions"]["session"] is True and token, caps status, surface, raw = _call(base, "GET", "/workbench/model-intake", token=token) diff --git a/tests/test_console_token_delivery.py b/tests/test_console_token_delivery.py new file mode 100644 index 00000000..708e86af --- /dev/null +++ b/tests/test_console_token_delivery.py @@ -0,0 +1,711 @@ +"""The console token travels in the opened URL, not `/capabilities` (plan 034 +T104; RULED openxFactory#656 comment `5963851934`, adversarial review 2's M5). + +A standalone openDox served its per-serve console token from `/capabilities` +to any loopback caller, other OS users of the machine included. The token +lets a page edit documents and run chat turns that spend the operator's model +credential. Only the DELIVERY changes, and this file holds each part of it: + +1. A STANDALONE plane (openDox's own default profile) carries no token on + `/capabilities`, and no route a second local user can ask serves it. A + HOST's plane (this suite's own `_SuiteProfile`, as openxFactory's) keeps + the `/capabilities` delivery it reads today. +2. The token travels in the opened URL's FRAGMENT and never in its query: the + URL, the private copy's forwarding targets, and what the entry point hands + the browser (a `file://` path) and prints (never the token). +3. The private copy is 0600 in a 0700 directory, this user's, and a copy that + is PLANTED, LINKED, hard-linked or loosened is refused, never followed. + Its record cannot break out of its `` or a tag is escaped, so the record ends where its element + does, every attribute is quoted, and it reads back as written.""" + from opendox import console_access + + hostile = ("http://127.0.0.1:8080/" + "/'\"&/index.html") + token = _token() + copy = _write(_state(tmp_path), token=token, page_url=hostile) + text = copy.path.read_text(encoding="utf-8") + assert text.count("") == 1, text + assert "" not in text, text + targets = _targets(copy.path) + (attrs, data), = targets.scripts + assert attrs == {"type": "application/json", "id": console_access.RECORD_ELEMENT_ID} + record = json.loads(data) + assert record["page_url"] == hostile and record["console_token"] == token + assert console_access.read_private_copy(copy.path) == record + for url in (*targets.refresh, *targets.links): + assert url == console_access.opened_url(hostile, token), url + + +def _generate_and_open_args(tmp_path: Path, repo: Path, *extra: str) -> argparse.Namespace: + from opendox import cli + return cli.build_parser().parse_args([ + "generate-and-open", "--repo-root", str(repo), "--repository", "fixture", + "--run-dir", str(tmp_path / "run"), "--port", "0", "--no-validate", + "--no-serve", *extra]) + + +def test_generate_and_open_hands_the_browser_a_file_and_prints_no_token( + tmp_path, monkeypatch, capsys, standalone_profile) -> None: + """What the entry point gives the browser is the copy's `file://` path: + a URL handed to `webbrowser.open` sits on a command line every user can + read. The copy it names opens the page with the token in the fragment. + Nothing it prints carries the token. The copy goes when the run does.""" + from opendox import cli, console_access + + _clean_git(monkeypatch) + repo = _repository(tmp_path) + state = _state(tmp_path) + monkeypatch.setenv("OPENDOX_STATE_DIR", str(state)) + opened: list[tuple[str, dict]] = [] + + def opener(url): + path = Path(urllib.parse.urlsplit(url).path) + opened.append((url, console_access.read_private_copy(path))) + + assert cli._generate_and_open(_generate_and_open_args(tmp_path, repo), + opener=opener) == 0 + out = capsys.readouterr().out + (url, record), = opened + token = record["console_token"] + assert url.startswith("file://") and token not in url, url + _assert_fragment_only(record["opened_url"], token) + assert token not in out, "the token was printed" + console_line = next(line for line in out.splitlines() + if line.startswith(" console ")) + assert url in console_line, console_line + assert record["page_url"] in out.splitlines(), out + assert not list((state / console_access.CONSOLE_DIRNAME).iterdir()), \ + "the copy outlived the run" + + +def test_no_open_prints_the_copy_path_and_opens_nothing( + tmp_path, monkeypatch, capsys, standalone_profile) -> None: + """`--no-open`: no browser, and the way back to the page is still printed, + the copy's path, while the token is still never printed.""" + from opendox import cli + + _clean_git(monkeypatch) + repo = _repository(tmp_path) + monkeypatch.setenv("OPENDOX_STATE_DIR", str(_state(tmp_path))) + opened: list[str] = [] + assert cli._generate_and_open( + _generate_and_open_args(tmp_path, repo, "--no-open"), + opener=opened.append) == 0 + out = capsys.readouterr().out + assert opened == [] + match = next(_CONSOLE.match(line) for line in out.splitlines() + if _CONSOLE.match(line)) + assert match.group(1).startswith("file://"), out + assert "console_token=" not in out, out + + +def test_generate_and_open_refuses_where_no_safe_copy_can_be_written( + tmp_path, monkeypatch, capsys, standalone_profile) -> None: + """A state directory another user could change refuses the run before it + serves, naming the directory; a console nobody can safely be handed is + not served as if it could be.""" + from opendox import cli + + _clean_git(monkeypatch) + repo = _repository(tmp_path) + state = _state(tmp_path) + state.chmod(0o770) + monkeypatch.setenv("OPENDOX_STATE_DIR", str(state)) + opened: list[str] = [] + assert cli._generate_and_open(_generate_and_open_args(tmp_path, repo), + opener=opened.append) == 1 + captured = capsys.readouterr() + assert opened == [] + assert "generate-and-open refused:" in captured.err + assert str(state) in captured.err and "writable by its group" in captured.err + + +# --------------------------------------------------------------------------- +# 3 — the private copy: planted, linked, loosened +# --------------------------------------------------------------------------- + +def test_the_copy_is_born_0600_in_a_0700_tree_whatever_the_umask(tmp_path) -> None: + from opendox import console_access + + state = tmp_path / "fresh" / "state" # nothing exists yet + previous = os.umask(0) + try: + copy = _write(state) + finally: + os.umask(previous) + assert stat.S_IMODE(os.lstat(copy.path).st_mode) == console_access.PRIVATE_MODE == 0o600 + for directory in (state, copy.path.parent): + assert stat.S_IMODE(os.lstat(directory).st_mode) == 0o700, directory + assert console_access.read_private_copy(copy.path)["port"] == 8080 + + +def test_a_link_planted_at_the_copy_is_refused_and_never_followed(tmp_path) -> None: + """Another user, or anyone, puts a link where the copy goes, pointing at a + file they can read: the write refuses, the token goes nowhere, and a read + through the link refuses too.""" + from opendox import console_access + + state = _state(tmp_path) + console = state / console_access.CONSOLE_DIRNAME + console.mkdir(mode=0o700) + leak = tmp_path / "leak.html" + leak.write_text("nothing yet\n", encoding="utf-8") + planted = console_access.private_copy_path(state, 8080) + planted.symlink_to(leak) + token = _token() + with pytest.raises(console_access.ConsoleAccessRefused, match="symbolic link"): + _write(state, token=token) + assert leak.read_text(encoding="utf-8") == "nothing yet\n" + assert planted.is_symlink(), "the planted link was replaced, not refused" + # a real copy elsewhere, linked in: a read refuses the link itself + other = _write(_state(tmp_path / "other")) + planted.unlink() + planted.symlink_to(other.path) + with pytest.raises(console_access.ConsoleAccessRefused, match="symbolic link"): + console_access.read_private_copy(planted) + + +def test_a_dangling_link_planted_at_the_copy_creates_nothing(tmp_path) -> None: + from opendox import console_access + + state = _state(tmp_path) + (state / console_access.CONSOLE_DIRNAME).mkdir(mode=0o700) + target = tmp_path / "would-be-created.html" + console_access.private_copy_path(state, 8080).symlink_to(target) + with pytest.raises(console_access.ConsoleAccessRefused): + _write(state) + assert not target.exists() + + +def test_a_hard_linked_copy_is_refused(tmp_path) -> None: + from opendox import console_access + + state = _state(tmp_path) + copy = _write(state) + os.link(copy.path, tmp_path / "second-name.html") + with pytest.raises(console_access.ConsoleAccessRefused, match="2 hard links"): + console_access.read_private_copy(copy.path) + with pytest.raises(console_access.ConsoleAccessRefused, match="2 hard links"): + _write(state) + + +def test_a_loosened_copy_is_refused(tmp_path) -> None: + from opendox import console_access + + copy = _write(_state(tmp_path)) + copy.path.chmod(0o644) + with pytest.raises(console_access.ConsoleAccessRefused, match="mode 644"): + console_access.read_private_copy(copy.path) + + +def test_a_planted_directory_or_file_at_the_copy_is_refused(tmp_path) -> None: + from opendox import console_access + + state = _state(tmp_path) + (state / console_access.CONSOLE_DIRNAME).mkdir(mode=0o700) + console_access.private_copy_path(state, 8080).mkdir() + with pytest.raises(console_access.ConsoleAccessRefused, match="not a regular file"): + _write(state) + + +def test_a_linked_console_directory_is_refused(tmp_path) -> None: + from opendox import console_access + + state = _state(tmp_path) + elsewhere = tmp_path / "elsewhere" + elsewhere.mkdir(mode=0o700) + (state / console_access.CONSOLE_DIRNAME).symlink_to(elsewhere) + with pytest.raises(console_access.ConsoleAccessRefused, match="symbolic link"): + _write(state) + assert list(elsewhere.iterdir()) == [] + + +@pytest.mark.parametrize("mode, reason", [(0o770, "writable by its group"), + (0o707, "writable by every user")]) +def test_a_state_directory_others_can_write_is_refused(tmp_path, mode, reason) -> None: + from opendox import console_access + + state = _state(tmp_path) + state.chmod(mode) + with pytest.raises(console_access.ConsoleAccessRefused, match=reason): + _write(state) + assert not (state / console_access.CONSOLE_DIRNAME).exists() + + +def test_a_state_directory_below_a_non_sticky_shared_directory_is_refused(tmp_path) -> None: + from opendox import console_access + + shared = tmp_path / "shared" + shared.mkdir() + shared.chmod(0o777) + try: + with pytest.raises(console_access.ConsoleAccessRefused, match="not sticky"): + _write(shared / "state") + shared.chmod(0o1777) # sticky, as /tmp is: accepted + assert _write(shared / "state").path.exists() + finally: + shared.chmod(0o755) + + +def test_a_later_copy_replaces_this_users_earlier_one_and_removal_is_exact( + tmp_path) -> None: + """A server restarted on the same port replaces its own earlier copy. The + first server's removal at shutdown leaves the later copy in place.""" + from opendox import console_access + + state = _state(tmp_path) + first = _write(state) + second_token = _token() + second = _write(state, token=second_token) + assert first.path == second.path + console_access.remove_private_copy(first) + assert console_access.read_private_copy(second.path)["console_token"] == second_token + console_access.remove_private_copy(second) + assert not second.path.exists() + console_access.remove_private_copy(second) # already gone: no error + console_access.remove_private_copy(None) + + +# --------------------------------------------------------------------------- +# 4 — the guarded routes still require it +# --------------------------------------------------------------------------- + +def test_every_route_that_requires_the_token_still_requires_it( + tmp_path, monkeypatch, standalone_profile) -> None: + """Only the delivery changed. Without the token, each guarded route + refuses with its console refusal; with the token read from the private + copy, the same request passes the console check.""" + from opendox import console_access, serve + from opendox.serve_wire import DOXBENCH_ERR_CONSOLE_REQUIRED + + with _serving(tmp_path, monkeypatch) as (httpd, base, _repo): + copy = console_access.publish( + httpd, page_url=serve.server_url(httpd, "/index.html"), + env={"OPENDOX_STATE_DIR": str(_state(tmp_path))}) + token = console_access.read_private_copy(copy.path)["console_token"] + assert token == httpd.console_token + catalog = "/workbench/model-catalog" + status, _h, raw = _call(base, "GET", catalog) + assert json.loads(raw).get("error") == DOXBENCH_ERR_CONSOLE_REQUIRED, raw + status, _h, raw = _call(base, "GET", catalog, token="x" * 43) + assert json.loads(raw).get("error") == DOXBENCH_ERR_CONSOLE_REQUIRED, raw + status, _h, raw = _call(base, "GET", catalog, token=token) + assert json.loads(raw).get("error") != DOXBENCH_ERR_CONSOLE_REQUIRED, raw + turn = "/actions/workbench/chat-turn" + status, _h, raw = _call(base, "POST", turn, body=b"{}") + assert json.loads(raw).get("error") == DOXBENCH_ERR_CONSOLE_REQUIRED, raw + status, _h, raw = _call(base, "POST", turn, body=b"{}", token=token) + assert json.loads(raw).get("error") != DOXBENCH_ERR_CONSOLE_REQUIRED, raw + + +# --------------------------------------------------------------------------- +# 5 — the documented command, as a user runs it +# --------------------------------------------------------------------------- + +def test_the_documented_command_delivers_the_token_only_through_its_copy( + tmp_path, monkeypatch) -> None: + """`python -m opendox.cli generate-and-open --local --no-open`, in a + process of its own (openDox's default profile, no host, no sibling): the + console line names the copy, `/capabilities` carries no token, the copy's + token opens the catalog, nothing printed carries it, and the copy goes + when the server stops.""" + from opendox import console_access + + _clean_git(monkeypatch) + repo = _repository(tmp_path) + child = Child(tmp_path, "opendox.cli", "generate-and-open", "--local", + "--repo-root", str(repo), "--repository", "fixture", + "--no-open", "--port", "0", "--run-dir", str(tmp_path / "run")) + try: + console = child.wait_for_line(_CONSOLE) + match = child.wait_for_line(_URL) + base, port = (match.group(2), int(match.group(3))), int(match.group(3)) + copy_path = console_access.private_copy_path(child.state_dir, port) + assert console.group(1) == copy_path.as_uri(), console.group(0) + token = child.console_token(port) + status, _h, raw = _call(base, "GET", "/capabilities") + assert status == 200 and "console_token" not in json.loads(raw) + assert token.encode() not in raw + status, _h, raw = _call(base, "GET", "/workbench/model-catalog", token=token) + assert status == 200, raw + assert child.interrupt() == 0, child.stderr_text() + assert not copy_path.exists(), "the copy outlived the server" + finally: + child.kill() + assert token not in child.stdout_text() + child.stderr_text() + assert child.refused() == [], child.refused() + + +_SERVE_URL = re.compile(r"^serving ideation dashboard at " + r"(http://([0-9.]+):([0-9]+))/index\.html$") +_SERVE_CONSOLE = re.compile(r"^console (file://\S+) ") + + +def test_the_servers_own_entry_point_delivers_the_token_the_same_way( + tmp_path, monkeypatch) -> None: + """`python -m opendox.serve`, the standalone secondary entry point: the + same private copy, the same printed path, no token on `/capabilities` or + on the output, and the copy gone when the server stops.""" + from opendox import console_access + + _clean_git(monkeypatch) + repo = _repository(tmp_path) + snapshot = tmp_path / "snapshot.json" + snapshot.write_text(json.dumps({"generation": {}}), encoding="utf-8") + child = Child(tmp_path, "opendox.serve", "--snapshot", str(snapshot), + "--checkout-root", str(repo), "--port", "0") + try: + console = child.wait_for_line(_SERVE_CONSOLE) + match = child.wait_for_line(_SERVE_URL) + base, port = (match.group(2), int(match.group(3))), int(match.group(3)) + copy_path = console_access.private_copy_path(child.state_dir, port) + assert console.group(1) == copy_path.as_uri() + token = child.console_token(port) + status, _h, raw = _call(base, "GET", "/capabilities") + assert status == 200 and token.encode() not in raw + status, _h, raw = _call(base, "GET", "/workbench/model-catalog", token=token) + assert json.loads(raw).get("error") != "console_required", raw + assert child.interrupt() == 0, child.stderr_text() + assert not copy_path.exists(), "the copy outlived the server" + finally: + child.kill() + assert token not in child.stdout_text() + child.stderr_text() + + +def test_the_servers_own_entry_point_refuses_where_no_safe_copy_can_be_written( + tmp_path, monkeypatch, capsys, standalone_profile) -> None: + from opendox import serve + + _clean_git(monkeypatch) + repo = _repository(tmp_path) + snapshot = tmp_path / "snapshot.json" + snapshot.write_text(json.dumps({"generation": {}}), encoding="utf-8") + state = _state(tmp_path) + state.chmod(0o707) + monkeypatch.setenv("OPENDOX_STATE_DIR", str(state)) + monkeypatch.setattr(serve, "real_notebook_adapter", lambda *a, **k: None) + assert serve.main(["--snapshot", str(snapshot), "--checkout-root", str(repo), + "--port", "0"]) == 1 + err = capsys.readouterr().err + assert "serve refused:" in err and "writable by every user" in err, err diff --git a/tests/test_console_token_view.py b/tests/test_console_token_view.py new file mode 100644 index 00000000..47cfdf34 --- /dev/null +++ b/tests/test_console_token_view.py @@ -0,0 +1,194 @@ +"""The page's half of plan 034 T104: `web/views/notebook.js` takes the +console token from the opened URL's FRAGMENT, keeps it, and strips it +(RULED openxFactory#656 comment `5963851934`). + +On a standalone plane `/capabilities` carries no token. The page is opened at +`…/index.html#console_token=` (`opendox.console_access.opened_url`), and +the module takes it the moment it is imported: it keeps it in +`sessionStorage`, or in memory where storage is blocked, and rewrites the +address bar with `history.replaceState` so the token does not stay in it. +`probeCapabilities` then fills it into a payload that carries none, so every +view keeps reading `caps.console_token`, and a host's own published token +always wins. Each case runs in node, against the module itself, with the +browser globals stood in; a fresh module instance per case (`?case=`). + +A CREATED FILE, with no carve-manifest row (RULED OQ-C). +""" + +from __future__ import annotations + +import json +import shutil +import subprocess +from pathlib import Path + +import pytest + +ROOT = Path(__file__).resolve().parent.parent +NOTEBOOK_JS = ROOT / "src" / "opendox" / "web" / "views" / "notebook.js" +NODE = shutil.which("node") + +TOKEN = "Tk_" + "a1B2-c3D4" * 4 # `secrets.token_urlsafe`'s alphabet +SERVED = "Served_" + "z9Y8-x7W6" * 4 + +_HARNESS = r""" +const TOKEN = %(token)s, SERVED = %(served)s; +const out = {}; + +function storage({ throwing = false, seed = null } = {}) { + const map = new Map(seed ? [["opendox.console-token", seed]] : []); + const api = { + getItem: (k) => (map.has(k) ? map.get(k) : null), + setItem: (k, v) => { map.set(k, String(v)); }, + removeItem: (k) => { map.delete(k); }, + dump: () => Object.fromEntries(map), + }; + return throwing ? null : api; +} + +function scope({ hash = "", search = "", store = storage(), throwing = false } = {}) { + const calls = []; + const s = { + location: { hash, search, pathname: "/index.html" }, + history: { state: { kept: 1 }, replaceState: (st, title, url) => { calls.push([st, title, url]); } }, + calls, + store, + }; + if (throwing) { + Object.defineProperty(s, "sessionStorage", { get() { throw new Error("blocked"); } }); + } else { + s.sessionStorage = store; + } + return s; +} + +function install(s) { + for (const name of ["location", "history"]) globalThis[name] = s[name]; + try { delete globalThis.sessionStorage; } catch {} + Object.defineProperty(globalThis, "sessionStorage", { + configurable: true, + get() { return s.sessionStorage; }, + }); +} + +const ok = (payload) => async () => ({ ok: true, json: async () => payload }); + +// 1. THE MODULE ITSELF, opened with the token in the fragment. +let s = scope({ hash: "#console_token=" + TOKEN }); +install(s); +let m = await import("./notebook.js?case=fragment"); +out.fragment = { + replaced: s.calls, + stored: s.store.dump(), + probed: await m.probeCapabilities(ok({ actions: { session: true } })), + hostWins: await m.probeCapabilities(ok({ actions: {}, console_token: SERVED })), + absentRoute: await m.probeCapabilities(async () => ({ ok: false })), + offline: await m.probeCapabilities(async () => { throw new Error("file://"); }), + notAnObject: await m.probeCapabilities(ok([1, 2])), +}; + +// 2. A RELOAD: no fragment, the tab's storage holds the token. +s = scope({ store: storage({ seed: TOKEN }) }); +install(s); +m = await import("./notebook.js?case=reload"); +out.reload = { replaced: s.calls, + probed: await m.probeCapabilities(ok({ actions: {} })) }; + +// 3. THE QUERY STRING IS NEVER READ: a token there is not taken. +s = scope({ search: "?console_token=" + TOKEN }); +install(s); +m = await import("./notebook.js?case=query"); +out.query = { replaced: s.calls, stored: s.store.dump(), + probed: await m.probeCapabilities(ok({ actions: {} })) }; + +// 4. A MALFORMED TOKEN: stripped from the address bar, not kept. +s = scope({ hash: "#console_token=%%3Cscript%%3E" }); +install(s); +m = await import("./notebook.js?case=malformed"); +out.malformed = { replaced: s.calls, stored: s.store.dump(), + probed: await m.probeCapabilities(ok({ actions: {} })) }; + +// 5. BLOCKED STORAGE: the token is still taken, in memory. +s = scope({ hash: "#console_token=" + TOKEN, throwing: true }); +install(s); +m = await import("./notebook.js?case=blocked"); +out.blocked = { replaced: s.calls, + probed: await m.probeCapabilities(ok({ actions: {} })) }; + +// 6. THE PURE FUNCTIONS, with no page at all. +out.pure = { + noScope: m.takeDeliveredConsoleToken(undefined), + noHash: m.takeDeliveredConsoleToken({ location: {} }), + otherFragment: m.takeDeliveredConsoleToken(scope({ hash: "#section-2" })), + explicitNull: m.withDeliveredConsoleToken({ actions: {} }, null), + keys: [m.CONSOLE_TOKEN_FRAGMENT_KEY, m.CONSOLE_TOKEN_STORAGE_KEY], +}; + +process.stdout.write(JSON.stringify(out)); +""" + + +@pytest.fixture(scope="module") +def page(tmp_path_factory): + if NODE is None: + pytest.skip("node not available for the console-token page probe") + root = tmp_path_factory.mktemp("console-token-view") + shutil.copy(NOTEBOOK_JS, root / "notebook.js") + (root / "package.json").write_text('{"type": "module"}', encoding="utf-8") + harness = root / "harness.mjs" + harness.write_text(_HARNESS % {"token": json.dumps(TOKEN), + "served": json.dumps(SERVED)}, + encoding="utf-8") + proc = subprocess.run([NODE, str(harness)], capture_output=True, text=True, + timeout=60) + assert proc.returncode == 0, proc.stderr + return json.loads(proc.stdout) + + +def test_the_fragment_token_is_taken_kept_and_stripped(page) -> None: + case = page["fragment"] + # stripped: the address bar is rewritten to the path, the state kept + assert case["replaced"] == [[{"kept": 1}, "", "/index.html"]] + assert case["stored"] == {"opendox.console-token": TOKEN} + assert case["probed"] == {"actions": {"session": True}, "console_token": TOKEN} + + +def test_a_hosts_published_token_wins_and_a_degraded_probe_gets_none(page) -> None: + case = page["fragment"] + assert case["hostWins"]["console_token"] == SERVED + assert case["absentRoute"] == {"actions": {"notebook": False}} + assert case["offline"] == {"actions": {"notebook": False}} + assert case["notAnObject"] == [1, 2] + + +def test_a_reload_keeps_the_token_from_the_tabs_storage(page) -> None: + assert page["reload"]["replaced"] == [] + assert page["reload"]["probed"]["console_token"] == TOKEN + + +def test_a_token_in_the_query_string_is_never_read(page) -> None: + case = page["query"] + assert case["replaced"] == [] and case["stored"] == {} + assert "console_token" not in case["probed"] + + +def test_a_malformed_token_is_stripped_and_not_kept(page) -> None: + case = page["malformed"] + assert case["replaced"] == [[{"kept": 1}, "", "/index.html"]] + assert case["stored"] == {} + assert "console_token" not in case["probed"] + + +def test_blocked_storage_still_yields_the_token_in_memory(page) -> None: + assert page["blocked"]["replaced"] == [[{"kept": 1}, "", "/index.html"]] + assert page["blocked"]["probed"]["console_token"] == TOKEN + + +def test_the_pure_readers_take_nothing_from_nothing(page) -> None: + case = page["pure"] + assert case["noScope"] is None and case["noHash"] is None + assert case["otherFragment"] is None + assert case["explicitNull"] == {"actions": {}} + # the page reads the key the server writes (`console_access.FRAGMENT_KEY`) + from opendox import console_access + assert case["keys"] == [console_access.FRAGMENT_KEY, "opendox.console-token"] diff --git a/tests/test_doxbench_defaults.py b/tests/test_doxbench_defaults.py index 8adaa9ce..76bf3c39 100644 --- a/tests/test_doxbench_defaults.py +++ b/tests/test_doxbench_defaults.py @@ -438,8 +438,10 @@ def _get(base: tuple[str, int], path: str, headers: dict | None = None): def served_catalog(child: Child) -> dict: """The served model catalog, read as the chat rail reads it: the console - token from `/capabilities`, then `GET /workbench/model-catalog`. Asserts - the route ANSWERS, with an envelope openDox's own validator accepts.""" + token the page was opened with (on a standalone plane, its private copy, + plan 034 T104; never `/capabilities`), then `GET + /workbench/model-catalog`. Asserts the route ANSWERS, with an envelope + openDox's own validator accepts.""" assert ACTOR in GATE_TEST_PRINCIPALS match = child.wait_for_line(_URL) base = (match.group(2), int(match.group(3))) @@ -447,8 +449,9 @@ def served_catalog(child: Child) -> dict: assert status == 200, status capabilities = json.loads(body) assert capabilities["actions"]["session"] is True, capabilities + assert "console_token" not in capabilities, capabilities status, body = _get(base, "/workbench/model-catalog", - {"X-XF-Console-Token": capabilities["console_token"]}) + {"X-XF-Console-Token": child.console_token(base[1])}) assert status == 200, (status, body) envelope = json.loads(body) assert envelope["kind"] == serve_wire.DOXBENCH_MODEL_CATALOG_KIND diff --git a/tests/test_neutral_turn_scope.py b/tests/test_neutral_turn_scope.py index df6a88ba..51dcf91e 100644 --- a/tests/test_neutral_turn_scope.py +++ b/tests/test_neutral_turn_scope.py @@ -261,9 +261,12 @@ def test_a_standalone_turn_over_the_tiles_own_document_is_answered( base = (match.group(2), int(match.group(3))) status, caps, raw = _call(base, "GET", "/capabilities") assert status == 200 and caps["actions"]["session"] is True, raw + # a standalone plane delivers its token in the opened URL, through + # its private copy, and never on `/capabilities` (plan 034 T104) + assert "console_token" not in caps, caps status, body, raw = _call(base, "POST", "/actions/workbench/chat-turn", body=_turn(repo, OWN), - token=caps["console_token"]) + token=child.console_token(base[1])) # past the validators, the guard and identity, answered by the model # step: never `turn_scope_refused`, never a dropped connection assert body.get("error") == DOXBENCH_ERR_MODEL_UNAVAILABLE, (status, raw) From 278855d41cb93e90de983829a675dded2a0192c9 Mon Sep 17 00:00:00 2001 From: Brett Heap <1513478+brettheap@users.noreply.github.com> Date: Sat, 3 Oct 2026 01:33:01 +0000 Subject: [PATCH 55/88] T084: an unknown tile kind is a malformed request, never a dropped connection (adversarial review 2, L1) `ScopeKey` refuses a `tile_kind` outside its closed vocabulary with a `ValueError`. Three workbench routes built one unguarded, so the error escaped the handler and the connection dropped: - the thread read (`serve_workbench.py:558`); - the document abstract (`:2725`); - the chat turn. Its released schema refuses most of these first, but validators that admit the shape carried it through. Each now answers its fixed malformed-request code: - the thread read: `invalid_turn_request`, as for a missing field; - the abstract: `invalid_abstract_request`; - the chat turn: `invalid_turn_request` in the released failure envelope. tests/test_capability_honesty.py covers all three: a standalone child for the thread read, and composed hosts for the abstract (past step 1) and the turn (past the validators). Before (5a3b51ee): 3 failed, each with RemoteDisconnected. Lane: openxfactory-4 (openXfactory-4-openDox_extraction) Co-Authored-By: Claude Opus 5.5 (1M context) --- src/opendox/serve_workbench.py | 46 ++++++++++++++++++++++++------ tests/test_capability_honesty.py | 49 ++++++++++++++++++++++++++++++++ 2 files changed, 86 insertions(+), 9 deletions(-) diff --git a/src/opendox/serve_workbench.py b/src/opendox/serve_workbench.py index e3b7c8ab..03edebb3 100644 --- a/src/opendox/serve_workbench.py +++ b/src/opendox/serve_workbench.py @@ -554,10 +554,20 @@ def _one(name): doxbench_error_status(DOXBENCH_ERR_INVALID_TURN_REQUEST), doxbench_error_body(DOXBENCH_ERR_INVALID_TURN_REQUEST)) return - # openDox's OWN scope type (`doxbench_scope_types`), no seam (T084) - key = ScopeKey( - repository=fields["repository"], ref=fields["ref"], - tile_kind=fields["tile_kind"], tile_id=fields["tile_id"]) + # openDox's OWN scope type (`doxbench_scope_types`), no seam (T084). + # It refuses a `tile_kind` outside its closed vocabulary with a + # `ValueError`, which used to escape and drop the connection + # (adversarial review 2, L1). An unknown kind is a malformed query, + # answered as a missing field is. + try: + key = ScopeKey( + repository=fields["repository"], ref=fields["ref"], + tile_kind=fields["tile_kind"], tile_id=fields["tile_id"]) + except ValueError: + self._send_json( + doxbench_error_status(DOXBENCH_ERR_INVALID_TURN_REQUEST), + doxbench_error_body(DOXBENCH_ERR_INVALID_TURN_REQUEST)) + return worktree = self._session_worktree_for(key) if worktree is None: # No live session on this scope. A DISTINCT cause (adversarial @@ -1729,10 +1739,19 @@ def _handle_workbench_chat_turn(self) -> None: scope_authority = column_seams.scope.current() from opendox import doxbench_turns - key = ScopeKey(repository=scope_fields["repository"], - ref=scope_fields["ref"], - tile_kind=scope_fields["tile_kind"], - tile_id=scope_fields["tile_id"]) + try: + key = ScopeKey(repository=scope_fields["repository"], + ref=scope_fields["ref"], + tile_kind=scope_fields["tile_kind"], + tile_id=scope_fields["tile_id"]) + except ValueError: + # A scope outside `ScopeKey`'s closed vocabulary (an unknown + # `tile_kind`, an empty field) is a malformed request, refused in + # the released envelope, never a dropped connection (adversarial + # review 2, L1). The released schema refuses most of these first. + self._refuse_turn(validators, DOXBENCH_ERR_INVALID_TURN_REQUEST, + turn_id, failure_kind=failure_kind) + return # ---- step 5: scope, all from SERVER truth ---- projection = None session_base = None @@ -2722,7 +2741,16 @@ def _handle_workbench_document_abstract(self) -> None: subject_path = fields["subject_path"] model_id = fields["model_id"] refresh = fields["refresh"] - key = ScopeKey(**fields["scope"]) + try: + key = ScopeKey(**fields["scope"]) + except ValueError: + # An unknown `tile_kind` is outside `ScopeKey`'s closed vocabulary: + # the request is malformed, and is answered so rather than with a + # dropped connection (adversarial review 2, L1). + self._send_json( + doxbench_error_status(DOXBENCH_ERR_INVALID_ABSTRACT_REQUEST), + doxbench_error_body(DOXBENCH_ERR_INVALID_ABSTRACT_REQUEST)) + return # ---- step 4: scope, all from SERVER truth ---- # THE SCOPE AUTHORITY IS READ HERE, BELOW STEP 1 (plan 034 T084; #1144 diff --git a/tests/test_capability_honesty.py b/tests/test_capability_honesty.py index 78ec7f5e..30aa6ad1 100644 --- a/tests/test_capability_honesty.py +++ b/tests/test_capability_honesty.py @@ -509,6 +509,55 @@ def test_the_rails_thread_read_answers_a_standalone_server( assert child.refused() == [], child.refused() +def test_an_unknown_tile_kind_on_the_thread_read_is_refused_not_dropped( + tmp_path, monkeypatch) -> None: + """Adversarial review 2, L1: `ScopeKey` refuses a `tile_kind` outside + its closed vocabulary with a `ValueError`, which escaped the thread read + and dropped the connection. It is a malformed query, answered so.""" + from opendox.serve_wire import DOXBENCH_ERR_INVALID_TURN_REQUEST + _clean_environment(monkeypatch) + repo = _repository(tmp_path, identity=True) + child, base, caps = _standalone(tmp_path, repo) + try: + status, body, raw = _call( + base, "GET", + "/workbench/thread?repository=fixture&ref=main&tile_kind=bogus" + "&tile_id=barrel-rain&document=notes-rain-barrel-leak.md", + token=caps["console_token"]) + assert body.get("error") == DOXBENCH_ERR_INVALID_TURN_REQUEST, (status, raw) + assert child.interrupt() == 0, child.stderr_text() + finally: + child.kill() + assert child.refused() == [], child.refused() + + +def test_an_unknown_tile_kind_on_the_abstract_is_refused_not_dropped( + composed_turns) -> None: + """L1 on the document abstract, past step 1 (a host contributing the gate + routes, so `gate` reads true).""" + from opendox.serve_wire import DOXBENCH_ERR_INVALID_ABSTRACT_REQUEST + base, caps = composed_turns(_gate()) + status, body, raw = _call( + base, "POST", "/actions/workbench/document-abstract", + body=_json({**_ABSTRACT, "scope": {**_SCOPE, "tile_kind": "bogus"}}), + token=caps["console_token"]) + assert body.get("error") == DOXBENCH_ERR_INVALID_ABSTRACT_REQUEST, (status, raw) + + +def test_an_unknown_tile_kind_on_the_chat_turn_is_refused_not_dropped( + composed_turns) -> None: + """L1 on the chat turn, where validators that admit the shape carry it to + the scope step: refused in the released envelope.""" + from opendox.serve_wire import DOXBENCH_ERR_INVALID_TURN_REQUEST + base, caps = composed_turns() + turn = _chat_turn() + turn["scope"] = {**_SCOPE, "tile_kind": "bogus"} + status, body, raw = _call(base, "POST", "/actions/workbench/chat-turn", + body=_json(turn), token=caps["console_token"]) + assert body.get("error") == DOXBENCH_ERR_INVALID_TURN_REQUEST, (status, raw) + assert body.get("client_turn_id") == "honesty-turn-1", body + + #: The binding `opendox model-binding add` declares for the cases below, as #: `tests/test_model_provider_broker.py` declares its own. Nothing is spawned #: and nothing is contacted: no case dispatches a turn. From 5a9cb0d8852a769b73736621e944f420bc33baa4 Mon Sep 17 00:00:00 2001 From: Brett Heap <1513478+brettheap@users.noreply.github.com> Date: Sat, 3 Oct 2026 01:35:55 +0000 Subject: [PATCH 56/88] T084: generate-and-open removes the run directory it minted, named opendox- (adversarial review 2, G7) With no `--run-dir`, `generate-and-open` minted a temporary `ideation-dashboard-*` directory, which is openxFactory's pre-carve name, and never removed it. Every run left one under the system's temporary directory. - `cli.RUN_DIR_PREFIX = "opendox-"`. - `_generate_and_open` removes the directory it minted when the run is done with it: a served run stopped by an interrupt (and, in a local install, by SIGTERM, which `_run_the_local_lifecycle` already reads as one), a `--no-serve` run, a refusal, a failure. - A `--run-dir` the caller names is the caller's, and is kept. - The serve path moves unchanged into `_generate_and_serve`. tests/test_run_dir_lifetime.py (new) runs hosted children with their own TMPDIR: - a served run mints exactly one `opendox-*` directory holding its snapshot, and none is left after the interrupt; - a `--no-serve` run leaves none; - a named `--run-dir` is kept. Before (278855d4): 2 failed. Mutant killed: the minted directory kept, 2 failed. Lane: openxfactory-4 (openXfactory-4-openDox_extraction) Co-Authored-By: Claude Opus 5.5 (1M context) --- src/opendox/cli.py | 30 +++++++++- tests/test_run_dir_lifetime.py | 101 +++++++++++++++++++++++++++++++++ 2 files changed, 128 insertions(+), 3 deletions(-) create mode 100644 tests/test_run_dir_lifetime.py diff --git a/src/opendox/cli.py b/src/opendox/cli.py index a920d692..6f1a0f97 100644 --- a/src/opendox/cli.py +++ b/src/opendox/cli.py @@ -20,6 +20,7 @@ import json import os import re +import shutil import signal import sys import tempfile @@ -787,16 +788,39 @@ def report() -> dict: return report +#: The prefix of the temporary run directory `generate-and-open` mints when +#: no `--run-dir` is given: the installed command's own name (plan 034 T084, +#: adversarial review 2, G7), not openxFactory's pre-carve one. +RUN_DIR_PREFIX = "opendox-" + + def _generate_and_open(args: argparse.Namespace, *, opener) -> int: - """`generate-and-open`'s generate-then-serve half, once the install is known.""" + """`generate-and-open`'s generate-then-serve half, once the install is known. + + A RUN DIRECTORY THIS PROCESS MINTED IS REMOVED WHEN IT IS DONE WITH IT + (plan 034 T084, adversarial review 2, G7): when the server stops, on a + `--no-serve` run, on a refusal and on a failure. It used to be left under + the system's temporary directory on every run. A `--run-dir` the caller + names is the caller's, and is left exactly as this run wrote it.""" # Ahead of minting the run dir, so a refused root leaves not even an empty # temp directory behind. `_generate_and_write` is still the guard that MATTERS # (it is the one no caller can skip); these are the same checks, earlier. _refuse_non_corpus_repo_root(args) _refuse_malformed_generated_at(args) _refuse_empty_source_options(args) - run_dir = Path(args.run_dir).resolve() if args.run_dir else Path( - tempfile.mkdtemp(prefix="ideation-dashboard-")) + if args.run_dir: + return _generate_and_serve(args, Path(args.run_dir).resolve(), + opener=opener) + minted = Path(tempfile.mkdtemp(prefix=RUN_DIR_PREFIX)) + try: + return _generate_and_serve(args, minted, opener=opener) + finally: + shutil.rmtree(minted, ignore_errors=True) + + +def _generate_and_serve(args: argparse.Namespace, run_dir: Path, *, + opener) -> int: + """Generate into `run_dir`, serve it, and stop, for `_generate_and_open`.""" run_dir.mkdir(parents=True, exist_ok=True) output = run_dir / "snapshot.json" diff --git a/tests/test_run_dir_lifetime.py b/tests/test_run_dir_lifetime.py new file mode 100644 index 00000000..5baff93b --- /dev/null +++ b/tests/test_run_dir_lifetime.py @@ -0,0 +1,101 @@ +"""`generate-and-open`'s minted run directory is removed when the run is done +with it, and is named for openDox (plan 034 T084, adversarial review 2, G7). + +With no `--run-dir`, the verb mints a temporary directory, writes the snapshot +there and serves it. It used to be minted as `ideation-dashboard-*`, which is +openxFactory's pre-carve name, and it was never removed: every run left one +under the system's temporary directory. Now it is minted as +`cli.RUN_DIR_PREFIX` (`opendox-`), and it is removed however the run ends: a +served run stopped by an interrupt, a `--no-serve` run, a refusal. A +`--run-dir` the caller names is the caller's, and is kept. + +Each case is a child process, `python -m opendox.cli generate-and-open` over +the plain fixture, with its own TMPDIR so its temporary directories can be +counted. It is a HOSTED install, with a hosted install's settings given on +purpose (`Child(extra_env=)`), so no database is started: the run directory's +lifetime is the same in both install shapes. + +A CREATED FILE: no carve-manifest row (RULED OQ-C). +""" + +from __future__ import annotations + +import re +from pathlib import Path + +from opendox.runtime import config as runtime_config +from standalone_child import Child, fresh_repository + +ROOT = Path(__file__).resolve().parent.parent +PLAIN = ROOT / "tests" / "fixtures" / "plain-documents" +PREFIX = runtime_config.PREFIX + +#: A hosted install's settings, as `tests/test_served_install_block.py` +#: gives them. Nothing in these cases connects to either database. +HOSTED = { + PREFIX + "INSTALL_MODE": runtime_config.INSTALL_MODE_HOSTED, + PREFIX + "DATABASE_URL": "postgresql://serve@127.0.0.1:1/opendox", + PREFIX + "MIGRATION_DATABASE_URL": "postgresql://migrate@127.0.0.1:1/opendox", + PREFIX + "OIDC_AUDIENCE": "fixture", + PREFIX + "OIDC_ISSUER": "https://issuer.example.invalid/realms/fixture", +} + +_URL = re.compile(r"^(http://([0-9.]+):([0-9]+))/index\.html$") + + +def _setup(tmp_path: Path) -> tuple[Path, Path, dict]: + repo = fresh_repository(PLAIN, tmp_path) + scratch = tmp_path / "tmp" + scratch.mkdir() + return repo, scratch, {**HOSTED, "TMPDIR": str(scratch)} + + +def _minted(scratch: Path) -> list[Path]: + return sorted(p for p in scratch.iterdir() if p.is_dir()) + + +def test_a_served_run_names_its_run_dir_for_opendox_and_removes_it_at_stop( + tmp_path) -> None: + from opendox import cli + repo, scratch, env = _setup(tmp_path) + child = Child(tmp_path, "opendox.cli", "generate-and-open", + "--repo-root", str(repo), "--repository", "fixture", + "--no-open", "--port", "0", extra_env=env) + try: + child.wait_for_line(_URL) + (minted,) = _minted(scratch) + assert cli.RUN_DIR_PREFIX == "opendox-" + assert minted.name.startswith("opendox-"), minted.name + assert (minted / "snapshot.json").is_file(), "it is the run's own" + assert child.interrupt() == 0, child.stderr_text() + finally: + child.kill() + assert _minted(scratch) == [], "the minted run directory outlived the run" + assert child.refused() == [], child.refused() + + +def test_a_no_serve_run_removes_its_minted_run_dir(tmp_path) -> None: + repo, scratch, env = _setup(tmp_path) + child = Child(tmp_path, "opendox.cli", "generate-and-open", + "--repo-root", str(repo), "--repository", "fixture", + "--no-open", "--no-serve", "--port", "0", extra_env=env) + try: + assert child.wait() == 0, child.stderr_text() + finally: + child.kill() + assert _minted(scratch) == [] + + +def test_a_run_dir_the_caller_names_is_kept(tmp_path) -> None: + repo, scratch, env = _setup(tmp_path) + run_dir = tmp_path / "kept" + child = Child(tmp_path, "opendox.cli", "generate-and-open", + "--repo-root", str(repo), "--repository", "fixture", + "--no-open", "--no-serve", "--port", "0", + "--run-dir", str(run_dir), extra_env=env) + try: + assert child.wait() == 0, child.stderr_text() + finally: + child.kill() + assert (run_dir / "snapshot.json").is_file() + assert _minted(scratch) == [] From 6519660e3407fb8bdb5c3ffde68baa559e256355 Mon Sep 17 00:00:00 2001 From: Brett Heap <1513478+brettheap@users.noreply.github.com> Date: Sat, 3 Oct 2026 01:35:55 +0000 Subject: [PATCH 57/88] T084: `python -m opendox.serve --help` names openDox only, as `opendox --help` does The holder carried this over from adversarial review 2. `serve.main`'s parser had `prog="ideation-dashboard-serve"` and printed the module's pre-carve docstring as its description ("Local backend for the ideation dashboard", `python3 scripts/ideation_dashboard/serve.py`, the openxFactory snapshot-index locator). - `serve.SERVE_PROG = "python -m opendox.serve"`, which is how the entry point is run. - `serve.SERVE_DESCRIPTION` is neutral and names `opendox generate-and-open`. tests/test_installed_help.py: the server's help starts `usage: python -m opendox.serve` and carries none of the foreign wording. Before (278855d4): failed. Mutant killed: the prog reverted, 1 failed. Lane: openxfactory-4 (openXfactory-4-openDox_extraction) Co-Authored-By: Claude Opus 5.5 (1M context) --- src/opendox/serve.py | 16 ++++++++++++++-- tests/test_installed_help.py | 15 +++++++++++++++ 2 files changed, 29 insertions(+), 2 deletions(-) diff --git a/src/opendox/serve.py b/src/opendox/serve.py index cd054f79..c18cf3bd 100644 --- a/src/opendox/serve.py +++ b/src/opendox/serve.py @@ -2424,6 +2424,18 @@ def _refuse_impossible_checkout_root(value: Path | str) -> int: return 0 +#: THE SERVER ENTRY POINT'S OWN NAME AND WORDS (plan 034 T084, with +#: `cli.PROG`; adversarial review 2). `python -m opendox.serve --help` printed +#: `usage: ideation-dashboard-serve` and this module's docstring, which is +#: openxFactory's pre-carve history. It names how it is run and openDox only. +SERVE_PROG = "python -m opendox.serve" +SERVE_DESCRIPTION = ( + "Serve an openDox snapshot locally: the browser bundle, the snapshot and " + "the read-only source of the checkout it was generated from, on a " + "loopback address. `opendox generate-and-open` generates a snapshot and " + "serves it in one command.") + + def main(argv: list[str] | None = None) -> int: # The process entry point registers openDox's own default where no host has # (R1Q3 (a)), exactly where a host would register its own. Nothing is BUILT @@ -2449,8 +2461,8 @@ def main(argv: list[str] | None = None) -> int: # R1Q10 (a)): the gate primitives, the doxBench scope, kickoff and # the cross-reference register, the same way. column_seams.register_defaults() - parser = argparse.ArgumentParser(prog="ideation-dashboard-serve", description=__doc__, - formatter_class=argparse.RawDescriptionHelpFormatter) + parser = argparse.ArgumentParser(prog=SERVE_PROG, + description=SERVE_DESCRIPTION) parser.add_argument("--web-dir", default=str(Path(__file__).resolve().parent / "web"), help="static bundle directory (default: the packaged web/)") parser.add_argument("--snapshot", required=True, diff --git a/tests/test_installed_help.py b/tests/test_installed_help.py index 6578f296..2e9bad61 100644 --- a/tests/test_installed_help.py +++ b/tests/test_installed_help.py @@ -82,3 +82,18 @@ def test_the_parser_carries_the_neutral_name_and_words() -> None: assert parser.epilog == cli.PARSER_EPILOG for text in (cli.PARSER_DESCRIPTION, cli.PARSER_EPILOG): assert not FOREIGN.search(text), text + + +def test_the_servers_own_help_names_openDox_only() -> None: + """`python -m opendox.serve --help`, the server entry point's help, under + the same rule (adversarial review 2): it printed + `usage: ideation-dashboard-serve` and the module's pre-carve docstring.""" + from opendox import serve + done = subprocess.run([sys.executable, "-m", "opendox.serve", "--help"], + capture_output=True, text=True, timeout=120, + env={**os.environ, "COLUMNS": "100"}) + assert done.returncode == 0, done.stderr + assert done.stdout.startswith(f"usage: {serve.SERVE_PROG} "), \ + done.stdout.splitlines()[0] + found = sorted({m.group(0) for m in FOREIGN.finditer(done.stdout)}) + assert found == [], f"the server's help names {found}:\n{done.stdout}" From 4a9dbe951955b97ab4996df1c2edd120eaae7032 Mon Sep 17 00:00:00 2001 From: Brett Heap <1513478+brettheap@users.noreply.github.com> Date: Sat, 3 Oct 2026 01:38:21 +0000 Subject: [PATCH 58/88] Fix round 17: the /proc falsifier is gated, and a backslash in the map is pinned literal (Copilot review) Two threads from Copilot's review at 52a2b8dc. - test_the_entry_point_owns_a_migrated_server_with_no_tcp_listener reads Linux's /proc for the server's parent, its TCP listeners and status's pid. The bundle runs on any POSIX platform, so it is now skipped where /proc is absent, as the other /proc and PDEATHSIG cases already are. On Linux, and in CI, nothing changes. - The second thread suggested escaping backslashes in pg_ident.conf's quoted user field. Measured against the bundled PostgreSQL 16.14, that would be wrong. pg_ident_file_mappings reads `"DOMAIN\alice"`, `"alice\"` and `"a\\b"` back as exactly those names, with no error. PostgreSQL's tokenizer treats a backslash specially only at the end of a line, as a continuation, never inside quotes. So escaping would map a different name. The code is unchanged, and the behaviour is now pinned: - test_bundled_postgres.py asks the server's own reading of the map for the three shapes; - the hermetic authentication-files case asserts that DOMAIN\alice is written verbatim. The mutant that escapes backslashes is killed by all four. Lane: openxfactory-4 (openXfactory-4-openDox_extraction) Co-Authored-By: Claude Opus 5.5 (1M context) --- tests_runtime/test_bundled_postgres.py | 33 ++++++++++++++++++++++++++ tests_runtime/test_local_lifecycle.py | 6 +++++ 2 files changed, 39 insertions(+) diff --git a/tests_runtime/test_bundled_postgres.py b/tests_runtime/test_bundled_postgres.py index 1b70aa81..3157c37d 100644 --- a/tests_runtime/test_bundled_postgres.py +++ b/tests_runtime/test_bundled_postgres.py @@ -347,6 +347,12 @@ def test_the_default_state_dir_is_the_users_own(monkeypatch) -> None: # -- 13.1 and R1Q16 (i), (ii), (iv): F13.1's two blocks, on the real entry point +# LINUX'S `/proc`, FOR THE KERNEL'S OWN ANSWERS (Copilot review of #69): the +# server's parent, its TCP listeners and `status`'s pid are read from it, so +# a POSIX platform without it skips this case rather than failing it. +# F13.1 names Linux's socket table; the lifecycle it shares with every POSIX +# platform is held by the cases around it. +@pytest.mark.skipif(not Path("/proc/self").exists(), reason="asks Linux's /proc") def test_the_entry_point_owns_a_migrated_server_with_no_tcp_listener( corpus: Path, state_dir: Path, tmp_path: Path) -> None: """F13.1's `runtime status` block and its TCP-listener block, against the @@ -546,6 +552,33 @@ def test_the_bundle_authenticates_by_peer_through_the_one_map( (bundle_mod.IDENT_MAP, user, config.BUNDLE_SERVED_ROLE, None)], mappings +@pytest.mark.parametrize("user", ["DOMAIN\\alice", "alice\\", "a\\\\b"]) +def test_the_server_reads_a_backslash_in_the_map_literally( + state_dir: Path, user: str) -> None: + """A user name holding a backslash, the shape an NSS or AD account takes + (`DOMAIN\\alice`), is written into `pg_ident.conf` as it is. PostgreSQL + 16 reads a quoted field's backslash LITERALLY: its tokenizer treats a + backslash specially only at the end of a line, as a continuation, and + never inside quotes. So the name is NOT escaped, and escaping it would + map a different name (Copilot review of #69, which suggested escaping, + answered by measurement). The server's own reading of the file is + asked, for each shape.""" + settings = config.load_settings({MODE: "local", STATE: str(state_dir)}) + with bundle_mod.BundledServer(settings) as server: + data = server.bundle.data_dir + (data / "pg_ident.conf").write_text( + bundle_mod.authentication_files(user)["pg_ident.conf"], encoding="utf-8") + try: + with _owner(server) as conn: + mappings = conn.execute( + "select sys_name, pg_username, error from pg_ident_file_mappings " + "order by map_number").fetchall() + finally: + bundle_mod.write_authentication(data, bundle_mod.os_user()) + assert mappings == [(user, config.BUNDLE_OWNER_ROLE, None), + (user, config.BUNDLE_SERVED_ROLE, None)], mappings + + def test_a_role_outside_the_map_is_refused_even_for_this_os_user( state_dir: Path) -> None: """The MAP decides, not the socket. The same OS user, over the same diff --git a/tests_runtime/test_local_lifecycle.py b/tests_runtime/test_local_lifecycle.py index 1d103148..b4692375 100644 --- a/tests_runtime/test_local_lifecycle.py +++ b/tests_runtime/test_local_lifecycle.py @@ -866,6 +866,12 @@ def test_the_authentication_files_admit_one_os_user_as_the_two_roles() -> None: [bundle_mod.IDENT_MAP, '"alice"', config.BUNDLE_OWNER_ROLE], [bundle_mod.IDENT_MAP, '"alice"', config.BUNDLE_SERVED_ROLE]] assert "trust" not in " ".join(" ".join(r) for rows in active.values() for r in rows) + # A BACKSLASH IS WRITTEN AS IT IS, never escaped: PostgreSQL 16 reads a + # quoted field's backslash literally (`test_bundled_postgres.py` asks + # the server's own reading; Copilot review of #69). + domain = bundle_mod.authentication_files("DOMAIN\\alice")["pg_ident.conf"] + assert f'{bundle_mod.IDENT_MAP} "DOMAIN\\alice" {config.BUNDLE_OWNER_ROLE}' \ + in domain.splitlines(), domain def test_the_files_are_written_0600_and_replace_what_was_there( From ff321f3c781eb3f8c599583041bbf50929feda63 Mon Sep 17 00:00:00 2001 From: Brett Heap <1513478+brettheap@users.noreply.github.com> Date: Sat, 3 Oct 2026 01:38:56 +0000 Subject: [PATCH 59/88] T104: T103's real local serve reads its token from the private copy tests/test_loopback_host_gate.py (T103, openDox-code#80) asserted that a real `generate-and-open --local` serve with an identity carries `console_token` on /capabilities. Under T104 a standalone plane never does: with an identity the token is minted and delivered through the 0600 private copy, and with none there is no token and no copy. The case now asserts exactly that; its Host-gate table is unchanged. Arc: neutral-product-standalone-operability Lane: openxfactory-4 (openXfactory-4-openDox_extraction) Co-Authored-By: Claude Opus 5.5 (1M context) --- tests/test_loopback_host_gate.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/tests/test_loopback_host_gate.py b/tests/test_loopback_host_gate.py index 0192eeb1..309f7af6 100644 --- a/tests/test_loopback_host_gate.py +++ b/tests/test_loopback_host_gate.py @@ -485,7 +485,16 @@ def test_every_route_class_refuses_a_foreign_host_on_a_real_local_serve( "/capabilities", good) assert status == 200, payload[:200] caps = json.loads(payload) - assert ("console_token" in caps) is identity, sorted(caps) + # A STANDALONE plane never publishes its token on `/capabilities` + # (plan 034 T104): with an identity one is minted and delivered + # through the private copy the entry point wrote, and with none + # there is no token and no copy. + assert "console_token" not in caps, sorted(caps) + from opendox import console_access + copy = console_access.private_copy_path(child.state_dir, base[1]) + assert copy.exists() is identity, copy + if identity: + assert child.console_token(base[1]) # L2's block, served to its own Host and to no other assert caps["install"]["mode"] == "local", caps.get("install") violations = table_violations(base, route_classes(contributed=False)) From 978f26462a07ef53f58b93b321a96dced7edf46b Mon Sep 17 00:00:00 2001 From: Brett Heap <1513478+brettheap@users.noreply.github.com> Date: Sat, 3 Oct 2026 01:48:32 +0000 Subject: [PATCH 60/88] T103 fix round 1: an IPv6 loopback bind binds, and its URL is bracketed (Copilot review) Copilot at openDox-code#80 badf2479, r4171161548: the gate accepts `[::1]:` only where the socket is bound to `::1`, but no socket ever was. `build_server` instantiated `http.server.ThreadingHTTPServer`, which is `AF_INET` only, so `host="::1"` (named by `LOOPBACK_HOSTS` and by the local install's `LOCAL_BIND_HOSTS`) failed at the bind with `gaierror`, and `generate-and-open --local --host ::1` ended in a traceback (reproduced at the base `3387293e`). And `server_url` would have printed `http://::1:/`. - `build_server` binds an IPv6 literal with `_IPv6ThreadingHTTPServer` (`AF_INET6`) and everything else with the standard class, as before (`_server_class_for`). - `server_url` brackets an IPv6 literal: `http://[::1]:/`. - Three cases, live over `::1`: the table across every route class in process (with `[::1]:` accepted and `[::1]:` refused), a real `generate-and-open --local --host ::1` child doing the same, and the browser path over `::1`. Arc: neutral-product-standalone-operability Lane: openxfactory-4 (openXfactory-4-openDox_extraction) Co-Authored-By: Claude Opus 5.5 (1M context) --- src/opendox/serve.py | 30 +++++++++++- tests/test_loopback_host_gate.py | 84 ++++++++++++++++++++++++++++---- 2 files changed, 103 insertions(+), 11 deletions(-) diff --git a/src/opendox/serve.py b/src/opendox/serve.py index f054fa02..f611e146 100644 --- a/src/opendox/serve.py +++ b/src/opendox/serve.py @@ -106,6 +106,7 @@ import http.server import json import secrets +import socket import subprocess import sys import urllib.parse @@ -2417,13 +2418,40 @@ def build_server( # trace on the first live connection. route_extension.resolve_handlers(route_bindings, bound) factory = functools.partial(bound, directory=str(web_dir)) - return http.server.ThreadingHTTPServer((host, port), factory) + return _server_class_for(host)((host, port), factory) + + +class _IPv6ThreadingHTTPServer(http.server.ThreadingHTTPServer): + """`ThreadingHTTPServer` over `AF_INET6`, for a bind to an IPv6 literal + (plan 034 T103; Copilot at openDox-code#80, r4171161548). + + The standard class is `AF_INET` only, so `host="::1"`, which + `LOOPBACK_HOSTS` and the local install's `LOCAL_BIND_HOSTS` both name, + failed at the bind with `gaierror`, and `generate-and-open --local --host + ::1` ended in a traceback. The loopback Host gate accepts `[::1]:` + exactly where the socket is bound to `::1`, so the bind it names has to + be one this module can make.""" + + address_family = socket.AF_INET6 + + +def _server_class_for(host: str) -> type[http.server.ThreadingHTTPServer]: + """The server class a bind to `host` takes: `AF_INET6` for an IPv6 + literal (the one spelling of a host that contains a colon), and the + standard `AF_INET` class for everything else, exactly as before.""" + return (_IPv6ThreadingHTTPServer if ":" in str(host) + else http.server.ThreadingHTTPServer) def server_url(httpd: http.server.ThreadingHTTPServer, path: str = "/") -> str: host, port = httpd.server_address[:2] if host in ("0.0.0.0", "", "::"): host = "127.0.0.1" + # AN IPv6 LITERAL IS BRACKETED in a URL (RFC 3986 § 3.2.2): `::1` is + # `http://[::1]:/`, and `[::1]:` is also the `Host` a browser + # then sends, which the loopback gate accepts on that bind (T103). + if ":" in str(host): + host = f"[{host}]" # plain-HTTP by design (S5332): a loopback-only local dev server — TLS adds # nothing on 127.0.0.1; the scheme is composed so no insecure-URL literal # exists for a copy-paste into non-loopback code. diff --git a/tests/test_loopback_host_gate.py b/tests/test_loopback_host_gate.py index 0192eeb1..c92176df 100644 --- a/tests/test_loopback_host_gate.py +++ b/tests/test_loopback_host_gate.py @@ -30,7 +30,9 @@ one. A large refused body is drained, so the refusal arrives whole. 3. The table across every route class of a REAL standalone `python -m opendox.cli generate-and-open --local` serve, a child with - neither sibling importable, with no identity and with one. + neither sibling importable, with no identity and with one. And an IPv6 + loopback bind (`--host ::1`), in process and as that real child: it binds, + prints a bracketed URL, and accepts `[::1]:` as its own authority. 4. The browser path still works: every file the bundle ships, the JSON routes and a console request carrying the token and the page's own `Origin`, under both `Host` spellings a browser sends. @@ -399,12 +401,17 @@ def build(*, identity: bool, host: str = serve.DEFAULT_HOST): worker = threading.Thread(target=httpd.serve_forever, daemon=True) worker.start() servers.append((httpd, worker)) - base = ("127.0.0.1", httpd.server_address[1]) + ipv6 = ":" in host + base = ("::1" if ipv6 else "127.0.0.1", httpd.server_address[1]) + own = f"[::1]:{base[1]}" if ipv6 else f"127.0.0.1:{base[1]}" status, _headers, payload, _raw_response = _raw( - base, "GET", "/capabilities", (f"127.0.0.1:{base[1]}",)) + base, "GET", "/capabilities", (own,)) assert status == 200, (status, payload[:200]) + build.urls.append(serve.server_url(httpd, "/index.html")) return base, json.loads(payload) + build.urls = [] + try: yield build finally: @@ -497,19 +504,76 @@ def test_every_route_class_refuses_a_foreign_host_on_a_real_local_serve( assert not child.state_dir.exists(), "the child's state dir outlived it" +# --------------------------------------------------------------------------- +# 3a — an IPv6 loopback bind: `[::1]` is its authority, and only there +# --------------------------------------------------------------------------- + +def ipv6_bind_tables(port: int): + """The table as an `::1` bind reads it: `[::1]:` joins the accepted + rows, and the one refused row it answers is replaced by its other-port + form.""" + accepted = accepted_hosts(port) + [("ipv6-literal", (f"[::1]:{port}",))] + refused = [row for row in refused_hosts(port) + if row[0] != "ipv6-loopback-on-an-ipv4-bind"] + refused.append(("ipv6-other-port", (f"[::1]:{_other_port(port)}",))) + return accepted, refused + + +def test_an_ipv6_loopback_bind_serves_and_gates_in_process(in_process) -> None: + """`host="::1"` binds (it used to fail with `gaierror`: the standard + server class is `AF_INET` only), announces a bracketed URL, and answers + the table on every route class (Copilot at openDox-code#80, + r4171161548).""" + base, caps = in_process(identity=False, host="::1") + assert "console_token" not in caps + assert in_process.urls[-1] == f"http://[::1]:{base[1]}/index.html" + accepted, refused = ipv6_bind_tables(base[1]) + violations = table_violations(base, route_classes(contributed=True), + accepted=accepted, refused=refused) + assert violations == [], "\n".join(violations) + + +def test_a_real_local_serve_binds_ipv6_loopback(tmp_path, monkeypatch) -> None: + """`generate-and-open --local --host ::1`, which the local install's + `LOCAL_BIND_HOSTS` admits, starts and prints `http://[::1]:/` + instead of ending in a traceback, and its every route class answers the + `::1` table.""" + _clean_environment(monkeypatch) + repo = fresh_repository(PLAIN, tmp_path) + child = Child(tmp_path, "opendox.cli", "generate-and-open", "--local", + "--repo-root", str(repo), "--repository", "fixture", + "--no-open", "--port", "0", "--host", "::1", + "--run-dir", str(tmp_path / "run")) + try: + match = child.wait_for_line( + re.compile(r"^http://\[::1\]:([0-9]+)/index\.html$")) + base = ("::1", int(match.group(1))) + accepted, refused = ipv6_bind_tables(base[1]) + violations = table_violations(base, route_classes(contributed=False), + accepted=accepted, refused=refused) + assert violations == [], "\n".join(violations) + assert child.interrupt() == 0, child.stderr_text() + finally: + child.kill() + assert child.refused() == [], child.refused() + assert not child.state_dir.exists(), "the child's state dir outlived it" + + # --------------------------------------------------------------------------- # 4 — the browser path still works # --------------------------------------------------------------------------- -@pytest.mark.parametrize("spelling", ["127.0.0.1", "localhost"]) -def test_the_browser_path_still_works(in_process, spelling) -> None: +@pytest.mark.parametrize("bind,spelling", [ + ("127.0.0.1", "127.0.0.1"), ("127.0.0.1", "localhost"), ("::1", "::1")]) +def test_the_browser_path_still_works(in_process, bind, spelling) -> None: """What a browser opened at `http://:/` sends: `http.client` writes `Host` from the address it connects to, exactly as a browser writes - it from the URL. Every file the bundle ships answers 200, the JSON routes - answer, and a console request with the token and the page's own `Origin` - reaches its route rather than the gate.""" - base, _caps = in_process(identity=True) + it from the URL (bracketing an IPv6 literal). Every file the bundle ships + answers 200, the JSON routes answer, and a console request with the token + and the page's own `Origin` reaches its route rather than the gate.""" + base, _caps = in_process(identity=True, host=bind) connect = (spelling, base[1]) + authority = f"[{spelling}]" if ":" in spelling else spelling def get(path, headers=None): connection = http.client.HTTPConnection(*connect, timeout=30) @@ -538,7 +602,7 @@ def get(path, headers=None): assert status == 200 and payload == (PLAIN / DOCUMENT).read_bytes() status, payload = get(serve.WORKBENCH_MODEL_CATALOG_ROUTE, { serve.CONSOLE_TOKEN_HEADER: token, - "Origin": f"http://{spelling}:{base[1]}"}) + "Origin": f"http://{authority}:{base[1]}"}) assert payload != serve.FOREIGN_HOST_BODY assert status == 200, payload[:300] From 8986158e4fbc88bec034add8738bb0835183670c Mon Sep 17 00:00:00 2001 From: Brett Heap <1513478+brettheap@users.noreply.github.com> Date: Sat, 3 Oct 2026 14:00:44 +0000 Subject: [PATCH 61/88] Fix round 18a: both bundle DSNs name their schema, public (Copilot review) The bundle's DSNs left search_path implicit, so PostgreSQL used its default, "$user", public. The owner (opendox) and the served role (opendox_runtime) are different users. So in a reused cluster holding a schema named after either role, the migration's ledger and the served workload's reads landed in two different schemas. That is the split-schema fault _refuse_two_dsns_that_select_different_schemas refuses for an operator's DSNs. DatabaseBundle.dsn now appends options=-c search_path=public (BUNDLE_SEARCH_PATH_OPTION, over the new BUNDLE_SCHEMA). public is the schema _bootstrap grants in, and schema_selected_by reads it back from both DSNs. New real-server case: a running bundle gains schemas `opendox` (the owner's) and `opendox_runtime` (authorization opendox_runtime). After a restart: - both roles' current_schema() is public; - both read the same ledger; - the restart applies nothing; - status is reachable with nothing pending. Without the pin, the restart migrates into `opendox` and fails with RuntimeAccessMissingError. The two-DSN layout case also asserts that both select public. Both mutants are killed: the option dropped, and the path left at "$user", public. Lane: openxfactory-4 (openXfactory-4-openDox_extraction) Co-Authored-By: Claude Opus 5.5 (1M context) --- src/opendox/runtime/config.py | 16 ++++++++++- tests_runtime/test_bundled_postgres.py | 39 ++++++++++++++++++++++++++ 2 files changed, 54 insertions(+), 1 deletion(-) diff --git a/src/opendox/runtime/config.py b/src/opendox/runtime/config.py index b220a6d5..852bd2c9 100644 --- a/src/opendox/runtime/config.py +++ b/src/opendox/runtime/config.py @@ -1647,6 +1647,10 @@ def _refuse_two_dsns_that_select_different_schemas( BUNDLE_OWNER_ROLE = "opendox" BUNDLE_SERVED_ROLE = "opendox_runtime" BUNDLE_DATABASE = "opendox" +#: The schema both bundle DSNs select, and the libpq `options` that select +#: it, percent-encoded for a URI's query (`DatabaseBundle.dsn`). +BUNDLE_SCHEMA = "public" +BUNDLE_SEARCH_PATH_OPTION = f"-c%20search_path%3D{BUNDLE_SCHEMA}" #: The longest socket path the kernel takes, in bytes: `sizeof(sun_path)` less #: its terminating NUL — 108 on Linux, 104 on macOS and the BSDs. PostgreSQL @@ -1694,10 +1698,20 @@ def dsn(self, role: str) -> str: port, and a host connection is rejected outright. SonarCloud's S2115 ("add password protection") is ACCEPTED on this line for that reason, with the same ruling as its authority. + + THE SCHEMA IS NAMED, `public`, in both DSNs (Copilot review of + openDox-code#69). Left implicit, `search_path` is PostgreSQL's + default, `"$user", public`, and the two roles are different users: + a reused cluster holding a schema named `opendox` or + `opendox_runtime` would put the migration's ledger in one schema + and the served workload's reads in another, the split-schema fault + `_refuse_two_dsns_that_select_different_schemas` refuses for an + operator's DSNs. `public` is the schema `_bootstrap` grants in. """ host = urllib.parse.quote(str(self.socket_dir), safe="/") return (f"postgresql://{role}@/{BUNDLE_DATABASE}" - f"?host={host}&port={BUNDLE_PORT}") + f"?host={host}&port={BUNDLE_PORT}" + f"&options={BUNDLE_SEARCH_PATH_OPTION}") @property def served_dsn(self) -> str: diff --git a/tests_runtime/test_bundled_postgres.py b/tests_runtime/test_bundled_postgres.py index 3157c37d..a6aceebf 100644 --- a/tests_runtime/test_bundled_postgres.py +++ b/tests_runtime/test_bundled_postgres.py @@ -294,11 +294,50 @@ def test_the_two_dsns_are_two_users_over_the_one_socket(state_dir: Path) -> None assert config.user_named_by(settings.database_url) == config.BUNDLE_SERVED_ROLE assert config.user_named_by(settings.migration_database_url) == \ config.BUNDLE_OWNER_ROLE + # both name their schema, the same one (Copilot review of #69) + assert config.schema_selected_by(settings.database_url) == \ + config.schema_selected_by(settings.migration_database_url) == "public" assert bundle.data_dir.parent == bundle.socket_dir.parent assert bundle.data_dir.is_relative_to(state_dir) assert bundle.socket_dir.is_relative_to(state_dir) +def test_schemas_named_for_the_roles_never_split_the_two_dsns( + state_dir: Path) -> None: + """PostgreSQL's default `search_path` is `"$user", public`, and the two + roles are different users. In a reused cluster holding a schema named + `opendox` (the owner's) and one named `opendox_runtime` (the served + role's), an implicit path put the migration's ledger and the served + workload's reads in two different schemas (Copilot review of #69). Both + DSNs name `public`, so both land there, a restart migrates nothing new, + and `status` reads the one ledger.""" + import psycopg + from psycopg import sql + + settings = config.load_settings({MODE: "local", STATE: str(state_dir)}) + with bundle_mod.BundledServer(settings) as server: + with _owner(server) as conn: + conn.execute("create schema opendox") + conn.execute(sql.SQL("create schema opendox_runtime authorization {}") + .format(sql.Identifier(config.BUNDLE_SERVED_ROLE))) + with bundle_mod.BundledServer(settings) as server: + applied_on_restart = list(server.applied) + current = {} + for role, dsn in ((config.BUNDLE_OWNER_ROLE, server.bundle.migration_dsn), + (config.BUNDLE_SERVED_ROLE, server.bundle.served_dsn)): + with psycopg.connect(dsn) as conn: + current[role] = conn.execute( + "select current_schema(), (select count(*) from " + "opendox_schema_migrations)").fetchone() + code, status = _status(state_dir) + assert current[config.BUNDLE_OWNER_ROLE][0] == "public", current + assert current[config.BUNDLE_SERVED_ROLE][0] == "public", current + assert current[config.BUNDLE_OWNER_ROLE][1] == current[config.BUNDLE_SERVED_ROLE][1] > 0 + assert applied_on_restart == [], applied_on_restart + assert status["database"] == "reachable" and status["pending_migrations"] == [], status + assert code == 0, status + + def test_a_state_dir_too_long_for_a_unix_socket_is_refused_naming_it() -> None: long = "/tmp/" + "x" * 120 with pytest.raises(config.ConfigurationError) as caught: From c04690f8270e019def40c74ceed359096f569ccb Mon Sep 17 00:00:00 2001 From: Brett Heap <1513478+brettheap@users.noreply.github.com> Date: Sat, 3 Oct 2026 14:00:44 +0000 Subject: [PATCH 62/88] Fix round 18b: a live pid /proc will not describe fails closed (Copilot review) _identity read every OSError from /proc/ as "gone", and _serves turned that into "not this server". But a procfs mount option or a security module can withhold a LIVE process of this very user, a PostgreSQL server included. So _remove_a_proven_stale_lock could unlink the postmaster.pid of a live server, and a second server was launched beside it. - _identity now returns None only when the entry has vanished (FileNotFoundError or ProcessLookupError). Any other OSError raises _Withheld, a LookupError, so _serves answers None (unknown), not False. - _remove_a_proven_stale_lock fails closed on a withheld live pid. The lock is kept, and the start is refused by name: it names the pid and says "will not describe", and tells the user to stop that process or remove the lock once they know it is not a server on this data directory. - With no /proc at all the lock is still left for PostgreSQL to judge, as before. - A gone pid is still "not ours", and a lock /proc proves stale is still removed. New case (Linux): an existing cluster whose lock names this process's own live pid, with that pid's /proc links withheld. _serves is None, the start is refused by name, the lock is kept, and the stand-in server never runs. Afterwards a gone pid reads False, and the now-describable stale lock is removed. The case fails against 4a9dbe95's bundle.py. Both mutants are killed: withheld read as gone, and withheld left to PostgreSQL. Lane: openxfactory-4 (openXfactory-4-openDox_extraction) Co-Authored-By: Claude Opus 5.5 (1M context) --- src/opendox/runtime/bundle.py | 55 ++++++++++++++++++++++----- tests_runtime/test_local_lifecycle.py | 52 +++++++++++++++++++++++++ 2 files changed, 97 insertions(+), 10 deletions(-) diff --git a/src/opendox/runtime/bundle.py b/src/opendox/runtime/bundle.py index b4d3d50b..da727389 100644 --- a/src/opendox/runtime/bundle.py +++ b/src/opendox/runtime/bundle.py @@ -248,23 +248,37 @@ def _lock_file_pid(bundle: DatabaseBundle) -> int | None: PROC = Path("/proc") +class _Withheld(LookupError): + """`/proc` exists, and it will not describe this live pid.""" + + def _identity(pid: int) -> tuple[str, str] | None: """`(executable, working directory)` of `pid`, from the kernel's `/proc`. - `None` where the process is gone or is not this user's to inspect. A - process that exits between the lock file's read and this one is GONE, + `None` where the process is GONE: its entry has vanished, as it does for + a process that exits between the lock file's read and this one, which is never proof of anything (Copilot review of openDox-code#69). Raises `LookupError` where there is no `/proc` at all (macOS, the BSDs): the standard library has no portable way to ask, and `running_pid` then believes nothing it cannot prove. + + AND RAISES `_Withheld`, a `LookupError` too, where the entry exists and + the kernel refuses to describe it (Copilot review of openDox-code#69). A + procfs mount option or a security module can hide a LIVE process of this + very user, a PostgreSQL server included. That is not knowledge that the + pid is something else. It used to read as `None`, which `_serves` turned + into "not this server", so a start removed a lock that a live server + still held. """ if not PROC.joinpath("self").exists(): raise LookupError("no /proc to ask") try: return (os.readlink(PROC / str(pid) / "exe"), os.readlink(PROC / str(pid) / "cwd")) - except OSError: # gone, or another user's: not inspectable - return None + except (FileNotFoundError, ProcessLookupError): + return None # gone + except OSError as exc: + raise _Withheld(f"pid {pid}: {type(exc).__name__}") from None def _serves(pid: int, bundle: DatabaseBundle) -> bool | None: @@ -277,10 +291,12 @@ def _serves(pid: int, bundle: DatabaseBundle) -> bool | None: anything else is not it, even another `postgres` serving another directory (Copilot review of openDox-code#69). - `False` also for a process that is gone, or that the platform will not - describe: another user's process cannot be this bundle's server, because - the server runs as the owner of a 0700 data directory, which is the user - this runs as. `None` only where nothing can be asked at all. + `False` also for a process that is gone. `None` where nothing can be + asked at all (no `/proc`), and where the platform will not describe this + live pid (`_Withheld`): unknown is not "not ours". Another user's + process never reaches this, because every caller asks `os.kill(pid, 0)` + first, and the server runs as the owner of a 0700 data directory, which + is the user this runs as. """ try: identity = _identity(pid) @@ -332,7 +348,13 @@ def _remove_a_proven_stale_lock(bundle: DatabaseBundle) -> None: of the same user, and that refusal would last as long as the unrelated process does. Where `/proc` shows that process is not this data directory's postmaster, the lock is stale by proof and is removed. Where - nothing can be proven, it is left for PostgreSQL to judge. + there is no `/proc` at all, it is left for PostgreSQL to judge. + + A LIVE PID `/proc` WILL NOT DESCRIBE FAILS CLOSED (Copilot review of + openDox-code#69). It may be this data directory's own server, hidden + by a procfs or security-module policy, so the lock is KEPT and the start + is refused by name, rather than the lock removed and a second server + launched beside the first. """ pid = _lock_file_pid(bundle) if pid is None: @@ -341,8 +363,21 @@ def _remove_a_proven_stale_lock(bundle: DatabaseBundle) -> None: os.kill(pid, 0) except (ProcessLookupError, PermissionError): return # PostgreSQL's own rule covers both + lock = bundle.data_dir / "postmaster.pid" + try: + _identity(pid) + except _Withheld: + raise BundleRefused( + f"{lock} names pid {pid}, a live process of this user that the " + "platform will not describe (its /proc entry is withheld), so it " + "may be this data directory's own server. The lock is kept and no " + "second server is started: stop that process, or remove the lock " + "yourself once you know it is not a PostgreSQL server on " + f"{bundle.data_dir}") from None + except LookupError: + return # no /proc: PostgreSQL judges if _serves(pid, bundle) is False: - (bundle.data_dir / "postmaster.pid").unlink(missing_ok=True) + lock.unlink(missing_ok=True) def refusal_before_connecting(bundle: DatabaseBundle) -> str | None: diff --git a/tests_runtime/test_local_lifecycle.py b/tests_runtime/test_local_lifecycle.py index b4692375..acfdb577 100644 --- a/tests_runtime/test_local_lifecycle.py +++ b/tests_runtime/test_local_lifecycle.py @@ -1237,6 +1237,58 @@ def prctl(*args): assert not launched.exists(), "the server ran without its parent-death signal" +def _withhold(monkeypatch, pid: int) -> None: + """`/proc/` exists and the kernel refuses to describe it, as a + procfs `hidepid` mount or a security module can for this user's own + live process. Only that pid's links are withheld.""" + real_readlink = os.readlink + entry = str(bundle_mod.PROC / str(pid)) + os.sep + + def _readlink(path, *args, **kwargs): + if str(path).startswith(entry): + raise PermissionError(13, "Permission denied", str(path)) + return real_readlink(path, *args, **kwargs) + + monkeypatch.setattr(bundle_mod.os, "readlink", _readlink) + + +@pytest.mark.skipif(not Path("/proc/self").exists(), reason="asks /proc") +def test_a_live_pid_the_platform_will_not_describe_keeps_its_lock( + monkeypatch, tmp_path: Path, short_state: Path) -> None: + """The lock file names a LIVE process of this user, and `/proc` withholds + what it is (Copilot review of #69). That is not proof it is something + other than this data directory's server, so the start FAILS CLOSED: the + lock is kept, the start is refused by name, and no second server is + launched. A pid that is gone is still "not ours", and a pid `/proc` + describes as something else still has its stale lock removed.""" + launched = tmp_path / "launched" + data = short_state / "postgres" / "data" + data.mkdir(parents=True, mode=0o700) + (short_state / "postgres").chmod(0o700) + (data / "PG_VERSION").write_text("16\n", encoding="utf-8") + lock = data / "postmaster.pid" + live = os.getpid() # alive, this user's + lock.write_text(f"{live}\n{data}\n", encoding="utf-8") + bundle = config.DatabaseBundle(short_state) + server = _server( + monkeypatch, tmp_path, short_state, initdb="exit 1", + postgres=('if [ "$1" = --version ]; then echo "postgres (PostgreSQL) 16.14"; ' + f'exit 0; fi\ntouch "{launched}"\nexit 3')) + _withhold(monkeypatch, live) + assert bundle_mod._serves(live, bundle) is None # unknown, not "not ours" + with pytest.raises(bundle_mod.BundleRefused) as caught: + server.start() + message = str(caught.value) + assert f"names pid {live}" in message and "will not describe" in message, message + assert lock.exists(), "the lock of a possibly live server was removed" + assert not launched.exists(), "a second server was launched beside it" + monkeypatch.undo() + # a pid that is gone reads as gone, and one /proc describes is still judged + assert bundle_mod._serves(2 ** 22 + 17, bundle) is False + bundle_mod._remove_a_proven_stale_lock(bundle) + assert not lock.exists(), "a lock /proc proves stale was kept" + + def test_directories_it_cannot_make_are_the_named_refusal( monkeypatch, tmp_path: Path, short_state: Path) -> None: if hasattr(os, "geteuid") and os.geteuid() == 0: # pragma: no cover From 7d69dc33bdb01b1290c9b88bfcde2d2b2259295b Mon Sep 17 00:00:00 2001 From: Brett Heap <1513478+brettheap@users.noreply.github.com> Date: Sat, 3 Oct 2026 14:01:30 +0000 Subject: [PATCH 63/88] T104: the private copy refuses a state directory in a served root The holder's ruling on openxFactory#1220's review (Copilot r4171166321), mirroring T100's served-repository boundary: the console token's copy must never sit inside what /source can serve, nor where a clone or an accidental commit could carry it. write_private_copy now takes the plane's served roots (build_server reports them on the server object: the checkout and each declared source root, resolved), and an OPENDOX_STATE_DIR that IS one of them, or lies inside one, is refused by name before anything is created or written. The check is on resolved paths, so a link from outside into a served root is the served root. Falsifiers: the state directory equal to the served root, under it, under a declared source root, reached through a link, and through generate-and-open (refused, naming OPENDOX_STATE_DIR, the repository unchanged). Arc: neutral-product-standalone-operability Lane: openxfactory-4 (openXfactory-4-openDox_extraction) Co-Authored-By: Claude Opus 5.5 (1M context) --- src/opendox/console_access.py | 47 ++++++++++++-- src/opendox/serve.py | 3 + tests/test_console_token_delivery.py | 95 +++++++++++++++++++++++++++- 3 files changed, 138 insertions(+), 7 deletions(-) diff --git a/src/opendox/console_access.py b/src/opendox/console_access.py index eb3d291b..be7caf03 100644 --- a/src/opendox/console_access.py +++ b/src/opendox/console_access.py @@ -53,7 +53,12 @@ file another user owns, a file with a second hard link) is REFUSED, never followed or replaced; * a READ asks all of it again of what exists, and of the file by its - descriptor: a regular file, this user's, exactly 0600, one link. + descriptor: a regular file, this user's, exactly 0600, one link; + * and the state directory may not BE, or lie inside, a root the plane + serves (its checkout and any declared source root), by name and before + any write, as T100's served-repository boundary refuses its own: the + token must never sit inside what `/source` can serve (holder's ruling on + openxFactory#1220's review, Copilot `r4171166321`). A CREATED FILE, with no carve-manifest row (RULED OQ-C). """ @@ -68,7 +73,7 @@ import re import stat import urllib.parse -from collections.abc import Mapping +from collections.abc import Iterable, Mapping from pathlib import Path from typing import Any @@ -357,13 +362,42 @@ def _opener_html(record: Mapping[str, Any]) -> str: "\n") +def _refuse_a_served_state_dir(state_dir: Path, + served_roots: Iterable[Path | str]) -> None: + """The state directory may not BE, or lie inside, a root this plane serves. + + RULED by the holder on openxFactory#1220's review (Copilot + `r4171166321`), mirroring T100's served-repository boundary + (`doxbench_trust`'s state directory): the token's copy must never sit + inside what `/source` can serve, nor where a clone or an accidental commit + could carry it. Asked of the RESOLVED paths, before anything is written.""" + resolved = Path(state_dir).resolve() + for root in served_roots: + served = Path(root).resolve() + if resolved == served or served in resolved.parents: + where = ("is the served repository" if resolved == served + else "lies inside the served repository") + raise ConsoleAccessRefused( + f"{runtime_config.PREFIX}STATE_DIR ({state_dir}) {where} " + f"({served}), which this plane serves through `/source` and a " + "clone or a commit could carry, so the console token's private " + "copy is refused there and nothing is written. Set " + f"{runtime_config.PREFIX}STATE_DIR to a directory outside the " + "repositories this machine serves") + + def write_private_copy(state_dir: Path | str, *, page_url: str, port: int, - token: str) -> PrivateCopy: + token: str, + served_roots: Iterable[Path | str]) -> PrivateCopy: """Write the copy for the plane on `port`, mode 0600, or refuse. - Replaces this user's own earlier copy for the same port (a server - restarted there), and refuses anything else already at that name.""" + `served_roots` are the roots this plane serves (`/source`'s checkout and + any declared source root): a state directory that is one of them, or lies + inside one, is refused before anything is written. Replaces this user's + own earlier copy for the same port (a server restarted there), and refuses + anything else already at that name.""" state = Path(state_dir) + _refuse_a_served_state_dir(state, tuple(served_roots)) record = { "schema_version": RECORD_SCHEMA_VERSION, "kind": RECORD_KIND, @@ -533,4 +567,5 @@ def publish(httpd: Any, *, page_url: str, f"the console token's private copy has no state directory: {exc}" ) from None return write_private_copy(state, page_url=page_url, - port=int(httpd.server_address[1]), token=token) + port=int(httpd.server_address[1]), token=token, + served_roots=getattr(httpd, "served_roots", ())) diff --git a/src/opendox/serve.py b/src/opendox/serve.py index c652ed73..ab8f4c8a 100644 --- a/src/opendox/serve.py +++ b/src/opendox/serve.py @@ -2442,6 +2442,9 @@ def build_server( # uses. Both `None` where no token was minted. httpd.console_token = console_token httpd.console_token_delivery = console_delivery + # ...and the roots `/source` serves, which the token's copy may not sit in. + httpd.served_roots = (checkout_root, *( + Path(path).resolve() for path in (source_roots or {}).values())) return httpd diff --git a/tests/test_console_token_delivery.py b/tests/test_console_token_delivery.py index 708e86af..8adb60e0 100644 --- a/tests/test_console_token_delivery.py +++ b/tests/test_console_token_delivery.py @@ -144,7 +144,7 @@ def _write(state: Path, port: int = 8080, token: str | None = None, from opendox import console_access return console_access.write_private_copy( state, page_url=page_url or f"http://127.0.0.1:{port}/index.html", - port=port, token=token or _token()) + port=port, token=token or _token(), served_roots=()) class _Targets(html.parser.HTMLParser): @@ -709,3 +709,96 @@ def test_the_servers_own_entry_point_refuses_where_no_safe_copy_can_be_written( "--port", "0"]) == 1 err = capsys.readouterr().err assert "serve refused:" in err and "writable by every user" in err, err + + +# --------------------------------------------------------------------------- +# 6 — never inside what the plane serves +# --------------------------------------------------------------------------- + +def _tree(path: Path) -> list[str]: + return sorted(str(p.relative_to(path)) for p in path.rglob("*")) + + +@pytest.mark.parametrize("where", ["the served root", "under the served root"]) +def test_a_state_directory_in_the_served_root_is_refused_before_any_write( + tmp_path, where) -> None: + """The holder's ruling on openxFactory#1220's review (Copilot + `r4171166321`), mirroring T100's served-repository boundary: the token's + copy must never sit inside what `/source` can serve. A state directory + that IS a served root, or lies inside one (a declared source root + included), is refused by name, and nothing is created or written.""" + from opendox import console_access + + served = tmp_path / "served" + served.mkdir(mode=0o700) + other = tmp_path / "other-source-root" + other.mkdir(mode=0o700) + before = (_tree(served), _tree(other)) + for root in (served, other): + state = root if where == "the served root" else root / "nested" / "state" + with pytest.raises(console_access.ConsoleAccessRefused) as refused: + console_access.write_private_copy( + state, page_url="http://127.0.0.1:8080/index.html", port=8080, + token=_token(), served_roots=(served, other)) + message = str(refused.value) + assert "OPENDOX_STATE_DIR" in message and str(root.resolve()) in message + assert (("is the served repository" if where == "the served root" + else "lies inside the served repository") in message), message + assert (_tree(served), _tree(other)) == before, "something was written" + + +def test_a_state_directory_reached_through_a_link_into_the_served_root_is_refused( + tmp_path) -> None: + """Judged on the RESOLVED path, so a link from outside that lands inside + the served root is the served root.""" + from opendox import console_access + + served = tmp_path / "served" + (served / "inside").mkdir(parents=True, mode=0o700) + link = tmp_path / "looks-outside" + link.symlink_to(served / "inside") + with pytest.raises(console_access.ConsoleAccessRefused, match="inside the served"): + console_access.write_private_copy( + link / "state", page_url="http://127.0.0.1:8080/index.html", + port=8080, token=_token(), served_roots=(served,)) + assert _tree(served / "inside") == [] + + +def test_generate_and_open_refuses_a_state_directory_inside_the_served_repository( + tmp_path, monkeypatch, capsys, standalone_profile) -> None: + """Through the entry point: `OPENDOX_STATE_DIR` inside the repository it + serves refuses the run, naming the setting, and the repository gains no + file.""" + from opendox import cli + + _clean_git(monkeypatch) + repo = _repository(tmp_path) + monkeypatch.setenv("OPENDOX_STATE_DIR", str(repo / ".opendox-state")) + before = _tree(repo) + opened: list[str] = [] + assert cli._generate_and_open(_generate_and_open_args(tmp_path, repo), + opener=opened.append) == 1 + err = capsys.readouterr().err + assert opened == [] + assert "generate-and-open refused:" in err and "OPENDOX_STATE_DIR" in err + assert "lies inside the served repository" in err, err + assert _tree(repo) == before + + +def test_the_plane_reports_every_root_it_serves(tmp_path, monkeypatch, + standalone_profile) -> None: + """`publish` reads the plane's own served roots: its checkout, and each + declared source root, resolved.""" + from opendox import serve + + with _serving(tmp_path, monkeypatch) as (httpd, _base, repo): + assert httpd.served_roots == (repo.resolve(),) + other = tmp_path / "other-source-root" + other.mkdir() + snapshot = tmp_path / "snapshot.json" + declared = serve.build_server(WEB, snapshot, repo, port=0, quiet=True, + source_roots={"other": str(other)}) + try: + assert declared.served_roots == (repo.resolve(), other.resolve()) + finally: + declared.server_close() From ebe0a35f2d2ae972d9653979f809a2d3b07e242d Mon Sep 17 00:00:00 2001 From: Brett Heap <1513478+brettheap@users.noreply.github.com> Date: Sat, 3 Oct 2026 14:07:48 +0000 Subject: [PATCH 64/88] T084 fix round 3: a group's edges name documents by id, and the scope projects them by path (Copilot review) A group's `document_edges[].document` names a document by ID, and a selection's `files` by PATH (the snapshot schema's `$defs/id` and `$defs/path`). openDox's default scope looked every reference up as a path. The neutral generator writes ids equal to paths, but a valid snapshot with id `notes/soil-test` and path `notes/soil-test.md` left every group and candidate member unresolved, with empty editable and context sets, so valid turns were refused (review r4171136778). `default_columns._document_index` maps each listed document's id, then its path, to the document's path, as openXdox's authority does. `_section` looks each reference up there, carries the document's path (confined to the root as before) and keeps the reference as the row's id. A reference no listed document answers stays unresolved under its own spelling, which must still be a safe path, so the confinement cases are unchanged. tests/test_column_seams.py::test_a_group_edge_names_its_document_by_id covers a snapshot whose ids differ from its paths. The group's and the candidate's members resolve and are editable, and a selection's files resolve as before. Before (b333bf16): failed, with editable_paths () where ('a.md', 'b.md') was expected. Mutant killed: look references up as paths only, 1 failed. tests/: 2689 passed, 11 skipped. Lane: openxfactory-4 (openXfactory-4-openDox_extraction) Co-Authored-By: Claude Opus 5.5 (1M context) --- src/opendox/default_columns.py | 48 +++++++++++++++++++++++++++------- tests/test_column_seams.py | 25 ++++++++++++++++++ 2 files changed, 63 insertions(+), 10 deletions(-) diff --git a/src/opendox/default_columns.py b/src/opendox/default_columns.py index f9ff1e62..082a68b7 100644 --- a/src/opendox/default_columns.py +++ b/src/opendox/default_columns.py @@ -291,20 +291,48 @@ def _canonical(path: Any) -> str: return path -def _section(key: str, label: str, note: str, paths: Iterable[Any], *, - known: set[str], seen: set[str], root: Path, +def _document_index(snapshot: Mapping[str, Any]) -> dict[str, str]: + """Each listed document's PATH, by its id and by its path. + + A group's `document_edges[].document` names a document by ID, and a + selection's `files` by PATH (the snapshot schema's `$defs/id` and + `$defs/path`). The neutral generator happens to write ids equal to paths, + but the contract does not promise it: a snapshot with id `notes/soil-test` + and path `notes/soil-test.md` is valid (Copilot review of + openDox-code#77, r4171136778). So every reference is looked up here and + the section carries the document's path, as openXdox's authority does + (`_document_index`). An id is indexed first, so a path that equals some + other document's id cannot take that id's place.""" + index: dict[str, str] = {} + documents = [_mapping(d) for d in _sequence(snapshot.get("documents"))] + for field in ("id", "path"): + for document in documents: + key, path = _text(document.get(field)), _text(document.get("path")) + if key and path and key not in index: + index[key] = path + return index + + +def _section(key: str, label: str, note: str, references: Iterable[Any], *, + index: Mapping[str, str], seen: set[str], root: Path, inherited: bool, owned: bool) -> ScopeSection: + """One section of a tile: each reference (a document id or path) as the + document it names, by that document's path, confined to `root`. A + reference no listed document answers is kept, unresolved, under its own + spelling, which must still be a safe path.""" from opendox import projection_seams resolve_within = projection_seams.registry.current().resolve_within rows: list[ScopeDocument] = [] - for raw in paths: - path = _canonical(raw) + for raw in references: + listed = index.get(_text(raw)) if isinstance(raw, str) else None + path = _canonical(listed if listed is not None else raw) if path in seen: continue seen.add(path) - resolved = path in known and resolve_within(root, path) is not None - rows.append(ScopeDocument(id=path, path=path, resolved=resolved)) + resolved = listed is not None and resolve_within(root, path) is not None + rows.append(ScopeDocument(id=_text(raw) or path, path=path, + resolved=resolved)) return ScopeSection(key=key, label=label, note=note, inherited=inherited, owned=owned, documents=tuple(rows)) @@ -403,7 +431,7 @@ def resolve_scope(snapshot: Mapping[str, Any], key: ScopeKey, *, "created_paths must be a collection of repository-relative paths" ) from error root = Path(source_root) - known = {_text(_mapping(d).get("path")) for d in _sequence(snapshot.get("documents"))} + index = _document_index(snapshot) groups = {_text(_mapping(g).get("id")): _mapping(g) for g in _sequence(snapshot.get("clusters"))} @@ -422,7 +450,7 @@ def members(group: Mapping[str, Any]) -> list[Any]: keywords = tuple(_text(t) for t in _sequence(group.get("topics")) if _text(t)) sections.append(_section( "members", "group documents", "the group's own document edges", - members(group), known=known, seen=seen, root=root, inherited=False, + members(group), index=index, seen=seen, root=root, inherited=False, owned=True)) elif key.tile_kind == "staged": selection = next((_mapping(s) for s in _sequence(snapshot.get("staged_topics")) @@ -432,7 +460,7 @@ def members(group: Mapping[str, Any]) -> list[Any]: title = key.tile_id sections.append(_section( "files", "selection files", "the documents this selection names", - _sequence(selection.get("files")), known=known, seen=seen, root=root, + _sequence(selection.get("files")), index=index, seen=seen, root=root, inherited=False, owned=True)) elif key.tile_kind == "possible": candidate = next((_mapping(p) for p in _sequence(snapshot.get("possibles")) @@ -445,7 +473,7 @@ def members(group: Mapping[str, Any]) -> list[Any]: sections.append(_section( "claiming", "documents of the claiming groups", "membership inferred from the groups that claim this candidate", - claimed, known=known, seen=seen, root=root, inherited=True, + claimed, index=index, seen=seen, root=root, inherited=True, owned=True)) else: return None diff --git a/tests/test_column_seams.py b/tests/test_column_seams.py index abbb0c0d..7a6dc1fa 100644 --- a/tests/test_column_seams.py +++ b/tests/test_column_seams.py @@ -338,6 +338,31 @@ def test_nothing_outside_the_tile_is_editable(corpus) -> None: assert other not in projection.context_paths +def test_a_group_edge_names_its_document_by_id(corpus) -> None: + """A group's `document_edges[].document` is a document ID, and a valid + snapshot's ids need not equal its paths (Copilot review of + openDox-code#77, r4171136778): `notes/a` is the id of `a.md`. The tile + projects the document by its PATH, resolved and editable, keeping the + id; a candidate's claiming group the same.""" + snapshot = _snapshot() + snapshot["documents"] = [ + {"id": "notes/" + p.removesuffix(".md"), "path": p} + for p in ("a.md", "b.md", "c.md", "sel.md", "gone.md")] + for group in snapshot["clusters"]: + for edge in group["document_edges"]: + edge["document"] = "notes/" + edge["document"].removesuffix(".md") + group = dc.resolve_scope(snapshot, _key("cluster", "g1"), source_root=corpus) + assert group.context_paths == ("a.md", "b.md") + assert group.editable_paths == ("a.md", "b.md") + assert [(row.id, row.path, row.resolved) for row in group.sections[0].documents] \ + == [("notes/a", "a.md", True), ("notes/b", "b.md", True)] + candidate = dc.resolve_scope(snapshot, _key("possible", "p1"), source_root=corpus) + assert candidate.editable_paths == ("a.md", "b.md", "c.md") + # a selection's files are PATHS, and resolve as before + staged = dc.resolve_scope(snapshot, _key("staged", "s1"), source_root=corpus) + assert staged.editable_paths == ("sel.md",) + + def test_a_listed_document_missing_from_the_tree_is_not_resolved(corpus) -> None: projection = dc.resolve_scope(_snapshot(), _key("cluster", "g2"), source_root=corpus) rows = {row.path: row.resolved for row in projection.sections[0].documents} From 77020042ea70a8ea2c390a973312ee7e8a4c7e37 Mon Sep 17 00:00:00 2001 From: Brett Heap <1513478+brettheap@users.noreply.github.com> Date: Sat, 3 Oct 2026 14:20:19 +0000 Subject: [PATCH 65/88] T103 fix round 2: an IPv6 wildcard bind is announced at ::1 (Copilot review) Copilot at openDox-code#80 fb0393df, r4173481146. Since fix round 1, `::` binds an `AF_INET6` socket, but `server_url` still announced every wildcard at `127.0.0.1`. An `AF_INET6` socket is IPv6-only on some platforms, so the printed or opened URL could name nothing that answers. `server_url` now announces `::` at `::1` (bracketed: `http://[::1]:/`) and keeps `0.0.0.0` and `""` at `127.0.0.1`. `::` is still a hosted plane, not one of `LOOPBACK_HOSTS`, so the loopback gate does not apply to it, as before. Six cases: `server_url` over the five bind spellings, and a live `::` bind answering at the URL it announces. Arc: neutral-product-standalone-operability Lane: openxfactory-4 (openXfactory-4-openDox_extraction) Co-Authored-By: Claude Opus 5.5 (1M context) --- src/opendox/serve.py | 9 ++++++++- tests/test_loopback_host_gate.py | 32 ++++++++++++++++++++++++++++++++ 2 files changed, 40 insertions(+), 1 deletion(-) diff --git a/src/opendox/serve.py b/src/opendox/serve.py index 9ab261f8..998b4719 100644 --- a/src/opendox/serve.py +++ b/src/opendox/serve.py @@ -2445,7 +2445,14 @@ def _server_class_for(host: str) -> type[http.server.ThreadingHTTPServer]: def server_url(httpd: http.server.ThreadingHTTPServer, path: str = "/") -> str: host, port = httpd.server_address[:2] - if host in ("0.0.0.0", "", "::"): + # A WILDCARD BIND IS ANNOUNCED AT ITS OWN FAMILY'S LOOPBACK (Copilot at + # openDox-code#80, r4173481146). Since T103's fix round 1 `::` binds an + # `AF_INET6` socket, which is IPv6-only on some platforms, so announcing + # it at `127.0.0.1` could print a URL nothing answers. `::` is `::1`; + # the IPv4 wildcard stays `127.0.0.1`. + if host == "::": + host = "::1" + elif host in ("0.0.0.0", ""): host = "127.0.0.1" # AN IPv6 LITERAL IS BRACKETED in a URL (RFC 3986 § 3.2.2): `::1` is # `http://[::1]:/`, and `[::1]:` is also the `Host` a browser diff --git a/tests/test_loopback_host_gate.py b/tests/test_loopback_host_gate.py index c92176df..4f6ba7a2 100644 --- a/tests/test_loopback_host_gate.py +++ b/tests/test_loopback_host_gate.py @@ -559,6 +559,38 @@ def test_a_real_local_serve_binds_ipv6_loopback(tmp_path, monkeypatch) -> None: assert not child.state_dir.exists(), "the child's state dir outlived it" +@pytest.mark.parametrize("bound,announced", [ + (("127.0.0.1", 8123), "http://127.0.0.1:8123/index.html"), + (("0.0.0.0", 8123), "http://127.0.0.1:8123/index.html"), + (("", 8123), "http://127.0.0.1:8123/index.html"), + (("::1", 8123, 0, 0), "http://[::1]:8123/index.html"), + (("::", 8123, 0, 0), "http://[::1]:8123/index.html"), +]) +def test_server_url_announces_each_bind_at_its_own_familys_loopback( + bound, announced) -> None: + """An IPv6 literal is bracketed, and a wildcard bind is announced at its + own family's loopback: an `AF_INET6` socket can be IPv6-only, so `::` + printed as `127.0.0.1` could name nothing that answers (Copilot at + openDox-code#80, r4173481146).""" + class _Bound: + server_address = bound + + assert serve.server_url(_Bound(), "/index.html") == announced + + +def test_an_ipv6_wildcard_bind_answers_at_the_url_it_announces( + in_process) -> None: + """`host="::"` is a hosted plane (not `LOOPBACK_HOSTS`), so no loopback + gate applies; what this holds is that the URL it prints answers.""" + base, caps = in_process(identity=False, host="::") + url = in_process.urls[-1] + assert url == f"http://[::1]:{base[1]}/index.html", url + assert caps["actions"]["intent"] is True + status, _headers, payload, _response = _raw( + ("::1", base[1]), "GET", "/index.html", (f"[::1]:{base[1]}",)) + assert status == 200 and b" Date: Sat, 3 Oct 2026 15:07:05 +0000 Subject: [PATCH 66/88] T073: the install report reads the bundled server's process once; the /proc guard and a docstring's noun (Copilot review of openDox-code#72) Three of the four threads at 68942680, as the holder ruled them: - r4171180402: `BundledServer.report()` read `self.process` three times while `stop()`, on the lifecycle thread, takes it away. A `/capabilities` request on the document server's request thread could pass the liveness test and then dereference None, and the request ended in a dropped connection. It now reads the process ONCE into a local and answers from that snapshot. The new case `test_a_capabilities_request_racing_the_bundles_stop_answers_whole` drives the race deterministically (the process's `poll()` is where `stop()` clears the field). Before the fix: RemoteDisconnected, the server thread's AttributeError ('NoneType' object has no attribute 'pid'). After: 200, a whole `install` block. Two mutants killed: the pid read again from the field (RemoteDisconnected), and the field read a second time in the liveness test (a torn answer, pid None). - r4173559498: `test_the_serving_process_reports_its_own_install_shape` reads `/proc` (`_tcp_listeners`, `_parent_of`) and now carries the same skip guard as the F13.1 case beside it. - r4171090095: "given a hosted install's settings", the noun restored. Lane: openxfactory-4 (openXfactory-4-openDox_extraction) Co-Authored-By: Claude Opus 5.5 (1M context) --- src/opendox/runtime/bundle.py | 14 ++++++-- tests/test_served_install_block.py | 47 +++++++++++++++++++++++++- tests_runtime/test_bundled_postgres.py | 3 ++ 3 files changed, 61 insertions(+), 3 deletions(-) diff --git a/src/opendox/runtime/bundle.py b/src/opendox/runtime/bundle.py index da727389..134ff6f1 100644 --- a/src/opendox/runtime/bundle.py +++ b/src/opendox/runtime/bundle.py @@ -839,10 +839,20 @@ def log_path(self) -> Path: return self.bundle.data_dir.parent / "postgres.log" def report(self) -> dict[str, Any]: - live = self.process is not None and self.process.poll() is None + """The bundle's directories, and the server's pid while it lives. + + Asked from a request thread of the document server (`/capabilities`' + `install` block, plan 034 T073) while the lifecycle thread may be in + `stop()`, which takes `self.process` away. So the process is read ONCE, + into a local, and every answer comes from that one snapshot: a report + racing a stop answers whole, for the process as it was when read, and + never dereferences a field another thread has just cleared (Copilot + review of openDox-code#72, r4171180402).""" + process = self.process + live = process is not None and process.poll() is None return {"data_dir": str(self.bundle.data_dir), "socket_dir": str(self.bundle.socket_dir), - "pid": self.process.pid if live else None} + "pid": process.pid if live else None} # -- start ------------------------------------------------------------- diff --git a/tests/test_served_install_block.py b/tests/test_served_install_block.py index ae56c757..ba3d9323 100644 --- a/tests/test_served_install_block.py +++ b/tests/test_served_install_block.py @@ -16,6 +16,9 @@ nothing handed in, the payload carries no `install` block at all. 3. `cli._install_report` reads the settings the verb loaded and the bundled server it started, and answers None where nothing was resolved. +4. A `/capabilities` request racing the bundled server's `stop()` answers + whole: the report reads the process ONCE (Copilot review of + openDox-code#72, r4171180402). A CREATED FILE: no carve-manifest row (RULED OQ-C). """ @@ -70,7 +73,8 @@ def _get(base: tuple[str, int], path: str) -> tuple[int, dict]: def test_a_hosted_entry_point_reports_its_hosted_shape(tmp_path) -> None: """The child inherits no runtime setting from the runner - (`standalone_child.Child`), and is given a hosted install's, on purpose.""" + (`standalone_child.Child`), and is given a hosted install's settings, on + purpose.""" repo = fresh_repository(PLAIN, tmp_path) child = Child(tmp_path, "opendox.cli", "generate-and-open", "--repo-root", str(repo), "--repository", "fixture", @@ -175,3 +179,44 @@ def report(self): "data_dir": "/s/d", "socket_dir": "/s/r", "pid": 7}} server.pid = None # it stopped: asked again, it says so assert report()["database_bundle"]["pid"] is None + + +# --------------------------------------------------------------------------- +# 4 — a report racing the bundled server's stop answers whole +# --------------------------------------------------------------------------- + +def test_a_capabilities_request_racing_the_bundles_stop_answers_whole( + served) -> None: + """`/capabilities` asks the bundled server's `report()` on a request + thread while the lifecycle thread may be in `stop()`, which clears + `process` (Copilot review of openDox-code#72, r4171180402). The race is + driven here deterministically: the process's `poll()`, inside `report()`, + is where `stop()` takes the process away. A report that read the field a + second time dereferenced `None`, and the request ended in a dropped + connection. From one snapshot it answers whole, for the process as read.""" + from opendox.runtime import bundle as bundle_mod + + # A SHORT state path, and never created: `report()` only names the + # bundle's directories, and `tmp_path` grows past the kernel's socket + # bound with this case's name (`config.database_bundle` refuses it). + settings = types.SimpleNamespace( + install_mode=runtime_config.INSTALL_MODE_LOCAL, + state_dir=Path("/odx-never-created/state")) + server = bundle_mod.BundledServer(settings) + + class _StoppedMidReport: + pid = 4242 + + def poll(self): + server.process = None # `stop()`, on the lifecycle thread + return None # ...after the process was read as live + + server.process = _StoppedMidReport() + base = served(install_report=cli_mod._install_report(argparse.Namespace( + runtime_settings=settings, database_bundle=server))) + status, caps = _get(base, "/capabilities") + assert status == 200, caps + assert caps["install"]["mode"] == runtime_config.INSTALL_MODE_LOCAL + assert caps["install"]["database_bundle"]["pid"] == 4242, caps + # and once it has stopped, the next request says so + assert _get(base, "/capabilities")[1]["install"]["database_bundle"]["pid"] is None diff --git a/tests_runtime/test_bundled_postgres.py b/tests_runtime/test_bundled_postgres.py index a580a844..c8dfa688 100644 --- a/tests_runtime/test_bundled_postgres.py +++ b/tests_runtime/test_bundled_postgres.py @@ -443,6 +443,9 @@ def test_the_entry_point_owns_a_migrated_server_with_no_tcp_listener( "the data directory must survive a stop: it is the install's database" +# `/proc` as above (Copilot review of openDox-code#72, r4173559498): the pid's +# TCP listeners and its parent are the kernel's answers, read from it. +@pytest.mark.skipif(not Path("/proc/self").exists(), reason="asks Linux's /proc") def test_the_serving_process_reports_its_own_install_shape( corpus: Path, state_dir: Path, tmp_path: Path) -> None: """F13.1's `caps.json` block (T073; #1144 13.4a): the server the user From cb8995467127f55a5df91163633d7ca21e8f940a Mon Sep 17 00:00:00 2001 From: Brett Heap <1513478+brettheap@users.noreply.github.com> Date: Sat, 3 Oct 2026 15:55:44 +0000 Subject: [PATCH 67/88] T104 fix round 1: every served root, an exact removal, and a plain kill (Copilot review) Copilot's review of openDox-code#84 at 6db50b03, four threads: - r4173806506: the served-root boundary missed roots the server serves. build_server now reports every root it serves files from: the checkout, the static bundle's --web-dir (whose handler would serve a copy under it to anyone, its 0600 notwithstanding, since the server reads it as its owner), each declared source root, each registry entry's root (the bootstrapped session worktrees), and on a loopback plane the sessions container every later session worktree is made in. Cases: a state directory under the static bundle and under the sessions container are refused, nothing written; the reported set is asserted. - r4173806552: removing by identity check then unlink could delete a replacement serve's copy written in between. remove_private_copy now takes the name first (an atomic rename to a name only this process uses), judges what it took, removes its own, and links anything else back under the name (never over a still newer copy). Both entry points also remove the copy BEFORE closing the listening socket, so no later serve can bind the port and publish until this one's copy is gone. Case: a replacement written at the instant of removal survives, whole. - r4173806590: SIGTERM did not reach the cleanup for `python -m opendox.serve` or a hosted generate-and-open. While a copy exists, both entry points read SIGTERM as Ctrl-C (console_access.terminate_as_ interrupt), restoring the previous handler afterwards; a plane that wrote no copy keeps SIGTERM's default. Cases: the context manager, and a plain kill of serve, generate-and-open --local and a hosted generate-and-open, each exit 0 with the copy gone. - r4173806621: the end-to-end shutdown assertions ran after Child.interrupt(), whose kill() deletes the state directory. They now signal and wait (_stop), assert, and leave cleanup to the finally. Arc: neutral-product-standalone-operability Lane: openxfactory-4 (openXfactory-4-openDox_extraction) Co-Authored-By: Claude Opus 5.5 (1M context) --- src/opendox/cli.py | 19 +-- src/opendox/console_access.py | 80 +++++++++++-- src/opendox/serve.py | 36 ++++-- tests/test_console_token_delivery.py | 166 +++++++++++++++++++++++++-- 4 files changed, 270 insertions(+), 31 deletions(-) diff --git a/src/opendox/cli.py b/src/opendox/cli.py index 21dd0adf..af9ddded 100644 --- a/src/opendox/cli.py +++ b/src/opendox/cli.py @@ -937,21 +937,24 @@ def _generate_and_serve(args: argparse.Namespace, run_dir: Path, *, "above manually)") if args.no_serve: - httpd.server_close() return 0 print(" serving until interrupted (Ctrl-C to stop)", flush=True) - try: - httpd.serve_forever() - except KeyboardInterrupt: - pass - finally: - httpd.server_close() + # A plain `kill` stops a standalone console the way Ctrl-C does, so + # the copy below is removed (`terminate_as_interrupt`); a plane that + # wrote no copy keeps SIGTERM's default action. + with console_access.terminate_as_interrupt(console is not None): + try: + httpd.serve_forever() + except KeyboardInterrupt: + pass return 0 finally: # The copy goes with the server: its token is this serve's, and dies - # with it. + # with it. It goes FIRST, while this process still holds the port, so + # no later serve can bind it and write its own copy in between. console_access.remove_private_copy(console) + httpd.server_close() # ---- gate console (US9): human-only executable gate actions ---------------- diff --git a/src/opendox/console_access.py b/src/opendox/console_access.py index be7caf03..73090c71 100644 --- a/src/opendox/console_access.py +++ b/src/opendox/console_access.py @@ -71,6 +71,7 @@ import json import os import re +import signal import stat import urllib.parse from collections.abc import Iterable, Mapping @@ -80,10 +81,12 @@ from opendox.runtime import config as runtime_config __all__ = [ - "CONSOLE_DIRNAME", "ConsoleAccessRefused", "DELIVERY_CAPABILITIES", + "CONSOLE_DIRNAME", "ConsoleAccessRefused", "ConsoleTerminated", + "DELIVERY_CAPABILITIES", "DELIVERY_OPENED_URL", "FRAGMENT_KEY", "PrivateCopy", "RECORD_ELEMENT_ID", "RECORD_KIND", "delivery_for", "opened_url", "private_copy_path", "publish", - "read_private_copy", "remove_private_copy", "write_private_copy", + "read_private_copy", "remove_private_copy", "terminate_as_interrupt", + "write_private_copy", ] #: The token rides on `/capabilities`, as a host's plane has always read it. @@ -538,13 +541,76 @@ def remove_private_copy(copy: PrivateCopy | None) -> None: """Remove `copy` when the server stops, if it is still the file written. A later serve on the same port writes a file of its own, and that one is - left alone. Never raises: a copy already gone is the goal reached.""" + left alone. Never raises: a copy already gone is the goal reached. + + THE NAME IS TAKEN BEFORE IT IS JUDGED (Copilot at openDox-code#84, + r4173806552). Checking the name's identity and + then unlinking it are two steps, and a replacement written between them + would be the file unlinked. So the name is first RENAMED to a name only + this process uses, atomically, and what was renamed is judged: this + process's own file is removed, and anything else is linked back under the + name (never over a still newer copy) and its temporary name removed. The + entry points also remove the copy BEFORE they close the listening socket, + so no later serve can bind the port, and write its own copy, until this + one is gone.""" if copy is None: return - with contextlib.suppress(OSError): - info = os.lstat(copy.path) - if (info.st_dev, info.st_ino) == copy.identity: - os.unlink(copy.path) + try: + directory = os.open(copy.path.parent, + os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW) + except OSError: + return + name = copy.path.name + taken = f".{name}.removing-{os.getpid()}-{os.urandom(6).hex()}" + try: + try: + os.rename(name, taken, src_dir_fd=directory, dst_dir_fd=directory) + except OSError: + return # nothing there: already gone + with contextlib.suppress(OSError): + info = os.stat(taken, dir_fd=directory, follow_symlinks=False) + if (info.st_dev, info.st_ino) != copy.identity: + # ANOTHER SERVE'S COPY: put it back under its name, unless a + # still newer one has arrived there, which then stands. + with contextlib.suppress(FileExistsError): + os.link(taken, name, src_dir_fd=directory, + dst_dir_fd=directory, follow_symlinks=False) + with contextlib.suppress(OSError): + os.unlink(taken, dir_fd=directory) + finally: + os.close(directory) + + +class ConsoleTerminated(KeyboardInterrupt): + """SIGTERM, raised as the interrupt the serve loops already stop on.""" + + +def _terminate_as_interrupt(signum, frame): + raise ConsoleTerminated + + +@contextlib.contextmanager +def terminate_as_interrupt(enabled: bool): + """While a standalone console's private copy exists, read SIGTERM as the + Ctrl-C the serve loops already stop cleanly on, so a plain `kill ` + unwinds through the code that removes the copy (Copilot at + openDox-code#84, r4173806590). The handler it replaces is put back on the + way out. `enabled` is False wherever no copy was written, a host's plane + or a plane with no token, and then nothing changes: those planes keep the + signal's default action exactly as before. Off the main thread no handler + can be installed, and nothing is.""" + if not enabled: + yield + return + try: + previous = signal.signal(signal.SIGTERM, _terminate_as_interrupt) + except ValueError: + yield + return + try: + yield + finally: + signal.signal(signal.SIGTERM, previous) def publish(httpd: Any, *, page_url: str, diff --git a/src/opendox/serve.py b/src/opendox/serve.py index e80fe96e..8b66ab56 100644 --- a/src/opendox/serve.py +++ b/src/opendox/serve.py @@ -2443,9 +2443,23 @@ def build_server( # uses. Both `None` where no token was minted. httpd.console_token = console_token httpd.console_token_delivery = console_delivery - # ...and the roots `/source` serves, which the token's copy may not sit in. - httpd.served_roots = (checkout_root, *( - Path(path).resolve() for path in (source_roots or {}).values())) + # ...and EVERY root this plane serves files from, which the token's copy + # may not sit in (Copilot at openDox-code#84, r4173806506): the checkout, + # the static bundle's directory, each declared source root, the root of + # each entry the registry holds now (the bootstrapped session worktrees + # among them), and, on a loopback plane, the sessions container every + # later session worktree is made in (`branch_session.sessions_root`). + served = [checkout_root, web_dir, + *(Path(path).resolve() for path in (source_roots or {}).values())] + if loopback: + from opendox import branch_session as session_mod + served.append(session_mod.sessions_root(checkout_root)) + entries = getattr(source.registry, "entries", None) + for entry in (entries() if callable(entries) else ()): + root = getattr(entry, "source_root", None) + if root: + served.append(Path(root).resolve()) + httpd.served_roots = tuple(dict.fromkeys(served)) return httpd @@ -2572,14 +2586,18 @@ def serve( # never reaches a wrapper while the server runs, and the wrapper cannot # learn an ephemeral port or tell that the server started. print(f"serving ideation dashboard at {page}", flush=True) - try: - httpd.serve_forever() - except KeyboardInterrupt: - pass - finally: - httpd.server_close() + # A plain `kill` stops a standalone console the way Ctrl-C does, so the + # copy is removed; a plane that wrote none keeps SIGTERM's default. + with console_access.terminate_as_interrupt(console is not None): + try: + httpd.serve_forever() + except KeyboardInterrupt: + pass finally: + # The copy FIRST, while this process still holds the port, then the + # socket (`console_access.remove_private_copy`). console_access.remove_private_copy(console) + httpd.server_close() def _source_roots_from_args(values) -> dict: diff --git a/tests/test_console_token_delivery.py b/tests/test_console_token_delivery.py index 8adb60e0..62e8943b 100644 --- a/tests/test_console_token_delivery.py +++ b/tests/test_console_token_delivery.py @@ -33,6 +33,7 @@ import json import os import re +import signal import stat import threading import urllib.parse @@ -619,6 +620,17 @@ def test_every_route_that_requires_the_token_still_requires_it( assert json.loads(raw).get("error") != DOXBENCH_ERR_CONSOLE_REQUIRED, raw +def _stop(child: Child, signum: int) -> int: + """Signal the child and wait for it, WITHOUT `Child.interrupt()`, whose + `kill()` deletes the child's state directory and would hide a copy the + child failed to remove (Copilot at openDox-code#84, r4173806621). The + case's own `finally: child.kill()` cleans up afterwards.""" + import standalone_child + + child.process.send_signal(signum) + return child.process.wait(timeout=standalone_child.STOP_DEADLINE_SECONDS) + + # --------------------------------------------------------------------------- # 5 — the documented command, as a user runs it # --------------------------------------------------------------------------- @@ -649,7 +661,8 @@ def test_the_documented_command_delivers_the_token_only_through_its_copy( assert token.encode() not in raw status, _h, raw = _call(base, "GET", "/workbench/model-catalog", token=token) assert status == 200, raw - assert child.interrupt() == 0, child.stderr_text() + # STOPPED, and looked at BEFORE `Child.kill()` deletes the state dir + assert _stop(child, signal.SIGINT) == 0, child.stderr_text() assert not copy_path.exists(), "the copy outlived the server" finally: child.kill() @@ -686,7 +699,7 @@ def test_the_servers_own_entry_point_delivers_the_token_the_same_way( assert status == 200 and token.encode() not in raw status, _h, raw = _call(base, "GET", "/workbench/model-catalog", token=token) assert json.loads(raw).get("error") != "console_required", raw - assert child.interrupt() == 0, child.stderr_text() + assert _stop(child, signal.SIGINT) == 0, child.stderr_text() assert not copy_path.exists(), "the copy outlived the server" finally: child.kill() @@ -787,18 +800,157 @@ def test_generate_and_open_refuses_a_state_directory_inside_the_served_repositor def test_the_plane_reports_every_root_it_serves(tmp_path, monkeypatch, standalone_profile) -> None: - """`publish` reads the plane's own served roots: its checkout, and each - declared source root, resolved.""" - from opendox import serve + """`publish` reads every root the plane serves files from (Copilot at + openDox-code#84, r4173806506): the checkout, the static bundle's + directory, each declared source root, each registry entry's root, and the + sessions container every session worktree is made in.""" + from opendox import branch_session, serve with _serving(tmp_path, monkeypatch) as (httpd, _base, repo): - assert httpd.served_roots == (repo.resolve(),) + sessions = branch_session.sessions_root(repo) + assert httpd.served_roots[:2] == (repo.resolve(), WEB.resolve()) + assert sessions in httpd.served_roots + entries = httpd.RequestHandlerClass.func.source.registry.entries() + assert entries and all(Path(e.source_root).resolve() in httpd.served_roots + for e in entries if e.source_root) other = tmp_path / "other-source-root" other.mkdir() snapshot = tmp_path / "snapshot.json" declared = serve.build_server(WEB, snapshot, repo, port=0, quiet=True, source_roots={"other": str(other)}) try: - assert declared.served_roots == (repo.resolve(), other.resolve()) + assert other.resolve() in declared.served_roots + assert repo.resolve() in declared.served_roots finally: declared.server_close() + + +@pytest.mark.parametrize("inside", ["the static bundle", "the sessions container"]) +def test_a_state_directory_in_another_served_root_is_refused_by_the_entry_point( + tmp_path, monkeypatch, standalone_profile, inside) -> None: + """The static handler serves every file under `--web-dir`, and `/source` + serves every session worktree, so a copy there would be served to anyone + who asks, its 0600 notwithstanding: the server reads it as its owner. + `publish` refuses both, before anything is written.""" + import shutil as _shutil + + from opendox import branch_session, console_access, serve + + _clean_git(monkeypatch) + repo = _repository(tmp_path) + web = tmp_path / "web" + _shutil.copytree(WEB, web) + snapshot = tmp_path / "snapshot.json" + snapshot.write_text(json.dumps({"generation": {}}), encoding="utf-8") + httpd = serve.build_server(web, snapshot, repo, port=0, quiet=True) + try: + assert httpd.console_token + state = (web / "state" if inside == "the static bundle" + else branch_session.sessions_root(repo) / "state") + with pytest.raises(console_access.ConsoleAccessRefused, + match="lies inside the served repository"): + console_access.publish( + httpd, page_url=serve.server_url(httpd, "/index.html"), + env={"OPENDOX_STATE_DIR": str(state)}) + assert not state.exists(), "something was written" + finally: + httpd.server_close() + + +# --------------------------------------------------------------------------- +# 7 — the copy's removal: exact under a race, and on a plain kill +# --------------------------------------------------------------------------- + +def test_a_replacement_written_while_the_old_copy_is_removed_survives( + tmp_path, monkeypatch) -> None: + """Copilot at openDox-code#84, r4173806552. A replacement serve publishes + its copy for the same port at the instant the old serve removes its own: + the old serve takes the NAME first (an atomic rename), finds a file that + is not its own, and puts it back. The replacement's copy stands, whole.""" + from opendox import console_access + + state = _state(tmp_path) + first = _write(state) + second_token = _token() + real_rename = os.rename + raced: list = [] + + def racing(src, dst, *args, **kwargs): + if src == first.path.name and not raced: + raced.append(_write(state, token=second_token)) # the replacement + return real_rename(src, dst, *args, **kwargs) + + monkeypatch.setattr(console_access.os, "rename", racing) + console_access.remove_private_copy(first) + monkeypatch.undo() + assert raced, "the race was never staged" + record = console_access.read_private_copy(first.path) + assert record["console_token"] == second_token + assert sorted(p.name for p in first.path.parent.iterdir()) == [first.path.name] + console_access.remove_private_copy(raced[0]) + assert not first.path.exists() + + +def test_terminate_as_interrupt_reads_sigterm_as_ctrl_c_and_restores_the_handler( + ) -> None: + from opendox import console_access + + def sentinel(signum, frame): + raise AssertionError("the previous handler ran") + + previous = signal.signal(signal.SIGTERM, sentinel) + try: + with pytest.raises(KeyboardInterrupt): + with console_access.terminate_as_interrupt(True): + os.kill(os.getpid(), signal.SIGTERM) + signal.pthread_sigmask(signal.SIG_BLOCK, []) # deliver now + assert signal.getsignal(signal.SIGTERM) is sentinel + with console_access.terminate_as_interrupt(False): + assert signal.getsignal(signal.SIGTERM) is sentinel + finally: + signal.signal(signal.SIGTERM, previous) + + +_HOSTED = { + "OPENDOX_INSTALL_MODE": "hosted", + "OPENDOX_DATABASE_URL": "postgresql://serve@127.0.0.1:1/opendox", + "OPENDOX_MIGRATION_DATABASE_URL": "postgresql://migrate@127.0.0.1:1/opendox", + "OPENDOX_OIDC_AUDIENCE": "fixture", + "OPENDOX_OIDC_ISSUER": "https://issuer.example.invalid/realms/fixture", +} + + +@pytest.mark.parametrize("entry", ["serve", "generate-and-open --local", + "generate-and-open, hosted"]) +def test_a_plain_kill_removes_the_copy(tmp_path, monkeypatch, entry) -> None: + """Copilot at openDox-code#84, r4173806590. SIGTERM, which `kill` sends, + stops every standalone entry point through the code that removes its + copy, and the copy is looked for BEFORE the helper deletes the state + directory. Each exits 0, as Ctrl-C does.""" + from opendox import console_access + + _clean_git(monkeypatch) + repo = _repository(tmp_path) + if entry == "serve": + snapshot = tmp_path / "snapshot.json" + snapshot.write_text(json.dumps({"generation": {}}), encoding="utf-8") + child = Child(tmp_path, "opendox.serve", "--snapshot", str(snapshot), + "--checkout-root", str(repo), "--port", "0") + url = _SERVE_URL + else: + local = ["--local"] if entry.endswith("--local") else [] + child = Child(tmp_path, "opendox.cli", "generate-and-open", *local, + "--repo-root", str(repo), "--repository", "fixture", + "--no-open", "--port", "0", "--run-dir", str(tmp_path / "run"), + extra_env=None if local else _HOSTED) + url = _URL + try: + match = child.wait_for_line(url) + port = int(match.group(3)) + copy_path = console_access.private_copy_path(child.state_dir, port) + assert copy_path.exists(), child.stdout_text() + child.stderr_text() + assert _stop(child, signal.SIGTERM) == 0, child.stderr_text() + assert not copy_path.exists(), "a plain kill left the token's copy behind" + assert "Traceback" not in child.stderr_text(), child.stderr_text() + finally: + child.kill() From 1fb81cbd75ccd2b11438898b927e390f7a5a90e2 Mon Sep 17 00:00:00 2001 From: Brett Heap <1513478+brettheap@users.noreply.github.com> Date: Sat, 3 Oct 2026 16:06:06 +0000 Subject: [PATCH 68/88] T084 fix round 4: a selection's files are paths and a group's edges are ids, looked up apart; the server's help names loopback as the default (Copilot review) - r4173844321: fix round 3's index answered every reference id first, and it served a selection's `files` too, which are PATHS. The contract does not forbid one document's path from spelling another document's id, so a selection's file `x` resolved to the document whose ID is `x`. `default_columns._document_index` now returns a `_DocumentIndex` with two maps. `ids` answers an id first, then a path, as before, for group edges and a candidate's claiming groups. `paths` answers a path only, for a selection's files. tests/test_column_seams.py:: test_a_selection_file_is_a_path_where_it_spells_another_documents_id. Before (d556c3fb): failed, with [('sel.md', 'a.md', True)] where [('sel.md', 'sel.md', True)] was expected. Mutants killed: the files looked up as ids (1 failed); the edges looked up as paths (2 failed, fix round 3's case with it). - r4173844338: `serve.SERVE_DESCRIPTION` promised "locally ... on a loopback address", but `--host` takes any address and a hosted install serves through this entry point. It now reads "on a loopback address unless --host names another". tests/test_installed_help.py:: test_the_servers_help_states_loopback_as_the_default_bind. Before: failed. Mutant killed: the --host clause dropped, 1 failed. Lane: openxfactory-4 (openXfactory-4-openDox_extraction) Co-Authored-By: Claude Opus 5.5 (1M context) --- src/opendox/default_columns.py | 50 ++++++++++++++++++++++++---------- src/opendox/serve.py | 11 +++++--- tests/test_column_seams.py | 23 ++++++++++++++++ tests/test_installed_help.py | 13 +++++++++ 4 files changed, 78 insertions(+), 19 deletions(-) diff --git a/src/opendox/default_columns.py b/src/opendox/default_columns.py index 082a68b7..0502f94f 100644 --- a/src/opendox/default_columns.py +++ b/src/opendox/default_columns.py @@ -62,7 +62,7 @@ from dataclasses import dataclass from datetime import datetime, timezone from pathlib import Path, PurePosixPath -from typing import Any, Iterable, Mapping, Sequence +from typing import Any, Iterable, Mapping, NamedTuple, Sequence from opendox import defaults from opendox.column_seams import GATE_RECORDS_REFUSAL @@ -291,8 +291,16 @@ def _canonical(path: Any) -> str: return path -def _document_index(snapshot: Mapping[str, Any]) -> dict[str, str]: - """Each listed document's PATH, by its id and by its path. +class _DocumentIndex(NamedTuple): + """Each listed document's PATH, looked up in the namespace a reference + is written in: `ids` for a document ID, `paths` for a document PATH.""" + + ids: Mapping[str, str] + paths: Mapping[str, str] + + +def _document_index(snapshot: Mapping[str, Any]) -> _DocumentIndex: + """Each listed document's PATH, by its id and, apart, by its path. A group's `document_edges[].document` names a document by ID, and a selection's `files` by PATH (the snapshot schema's `$defs/id` and @@ -300,24 +308,36 @@ def _document_index(snapshot: Mapping[str, Any]) -> dict[str, str]: but the contract does not promise it: a snapshot with id `notes/soil-test` and path `notes/soil-test.md` is valid (Copilot review of openDox-code#77, r4171136778). So every reference is looked up here and - the section carries the document's path, as openXdox's authority does - (`_document_index`). An id is indexed first, so a path that equals some - other document's id cannot take that id's place.""" - index: dict[str, str] = {} + the section carries the document's path, as openXdox's authority does. + + THE TWO NAMESPACES ARE KEPT APART (r4173844321). Nor does the contract + forbid one document's path from equaling another document's id, so one + map from either spelling would resolve a selection's file `x` to the + document whose ID is `x`. `paths` answers a path only. `ids` answers an + id first and then, for an edge written as a path, a path: an id is + indexed first, so a path that equals some other document's id cannot + take that id's place.""" + ids: dict[str, str] = {} + paths: dict[str, str] = {} documents = [_mapping(d) for d in _sequence(snapshot.get("documents"))] for field in ("id", "path"): for document in documents: key, path = _text(document.get(field)), _text(document.get("path")) - if key and path and key not in index: - index[key] = path - return index + if key and path and key not in ids: + ids[key] = path + for document in documents: + path = _text(document.get("path")) + if path and path not in paths: + paths[path] = path + return _DocumentIndex(ids=ids, paths=paths) def _section(key: str, label: str, note: str, references: Iterable[Any], *, index: Mapping[str, str], seen: set[str], root: Path, inherited: bool, owned: bool) -> ScopeSection: - """One section of a tile: each reference (a document id or path) as the - document it names, by that document's path, confined to `root`. A + """One section of a tile: each reference as the document it names in + `index`, the namespace its references are written in (`_DocumentIndex`), + by that document's path, confined to `root`. A reference no listed document answers is kept, unresolved, under its own spelling, which must still be a safe path.""" from opendox import projection_seams @@ -450,7 +470,7 @@ def members(group: Mapping[str, Any]) -> list[Any]: keywords = tuple(_text(t) for t in _sequence(group.get("topics")) if _text(t)) sections.append(_section( "members", "group documents", "the group's own document edges", - members(group), index=index, seen=seen, root=root, inherited=False, + members(group), index=index.ids, seen=seen, root=root, inherited=False, owned=True)) elif key.tile_kind == "staged": selection = next((_mapping(s) for s in _sequence(snapshot.get("staged_topics")) @@ -460,7 +480,7 @@ def members(group: Mapping[str, Any]) -> list[Any]: title = key.tile_id sections.append(_section( "files", "selection files", "the documents this selection names", - _sequence(selection.get("files")), index=index, seen=seen, root=root, + _sequence(selection.get("files")), index=index.paths, seen=seen, root=root, inherited=False, owned=True)) elif key.tile_kind == "possible": candidate = next((_mapping(p) for p in _sequence(snapshot.get("possibles")) @@ -473,7 +493,7 @@ def members(group: Mapping[str, Any]) -> list[Any]: sections.append(_section( "claiming", "documents of the claiming groups", "membership inferred from the groups that claim this candidate", - claimed, index=index, seen=seen, root=root, inherited=True, + claimed, index=index.ids, seen=seen, root=root, inherited=True, owned=True)) else: return None diff --git a/src/opendox/serve.py b/src/opendox/serve.py index c18cf3bd..48db44ee 100644 --- a/src/opendox/serve.py +++ b/src/opendox/serve.py @@ -2428,12 +2428,15 @@ def _refuse_impossible_checkout_root(value: Path | str) -> int: #: `cli.PROG`; adversarial review 2). `python -m opendox.serve --help` printed #: `usage: ideation-dashboard-serve` and this module's docstring, which is #: openxFactory's pre-carve history. It names how it is run and openDox only. +#: Loopback is the DEFAULT bind, not a promise: `--host` takes any address, and +#: a hosted install serves through this entry point (Copilot review of +#: openDox-code#77, r4173844338). SERVE_PROG = "python -m opendox.serve" SERVE_DESCRIPTION = ( - "Serve an openDox snapshot locally: the browser bundle, the snapshot and " - "the read-only source of the checkout it was generated from, on a " - "loopback address. `opendox generate-and-open` generates a snapshot and " - "serves it in one command.") + "Serve an openDox snapshot: the browser bundle, the snapshot and the " + "read-only source of the checkout it was generated from, on a loopback " + "address unless --host names another. `opendox generate-and-open` " + "generates a snapshot and serves it in one command.") def main(argv: list[str] | None = None) -> int: diff --git a/tests/test_column_seams.py b/tests/test_column_seams.py index 7a6dc1fa..e4ff632d 100644 --- a/tests/test_column_seams.py +++ b/tests/test_column_seams.py @@ -363,6 +363,29 @@ def test_a_group_edge_names_its_document_by_id(corpus) -> None: assert staged.editable_paths == ("sel.md",) +def test_a_selection_file_is_a_path_where_it_spells_another_documents_id( + corpus) -> None: + """A selection's `files` are PATHS and a group's edges are IDS, and the + contract does not forbid one document's path from spelling another + document's id (Copilot review of openDox-code#77, r4173844321). Here + `sel.md` is the PATH of one document and the ID of another, `a.md`. The + selection's file is the document at `sel.md`, and a group edge naming + `sel.md` is the document whose id it is.""" + snapshot = _snapshot() + snapshot["documents"] = [ + {"id": "sel.md", "path": "a.md"}, # an id spelled as a path + {"id": "notes/sel", "path": "sel.md"}, + *({"id": p, "path": p} for p in ("b.md", "c.md", "gone.md"))] + snapshot["clusters"][0]["document_edges"] = [ + {"document": "sel.md"}, {"document": "b.md"}] + staged = dc.resolve_scope(snapshot, _key("staged", "s1"), source_root=corpus) + assert [(row.id, row.path, row.resolved) for row in staged.sections[0].documents] \ + == [("sel.md", "sel.md", True)] + assert staged.editable_paths == ("sel.md",) + group = dc.resolve_scope(snapshot, _key("cluster", "g1"), source_root=corpus) + assert group.editable_paths == ("a.md", "b.md") + + def test_a_listed_document_missing_from_the_tree_is_not_resolved(corpus) -> None: projection = dc.resolve_scope(_snapshot(), _key("cluster", "g2"), source_root=corpus) rows = {row.path: row.resolved for row in projection.sections[0].documents} diff --git a/tests/test_installed_help.py b/tests/test_installed_help.py index 2e9bad61..5dedcc79 100644 --- a/tests/test_installed_help.py +++ b/tests/test_installed_help.py @@ -97,3 +97,16 @@ def test_the_servers_own_help_names_openDox_only() -> None: done.stdout.splitlines()[0] found = sorted({m.group(0) for m in FOREIGN.finditer(done.stdout)}) assert found == [], f"the server's help names {found}:\n{done.stdout}" + + +def test_the_servers_help_states_loopback_as_the_default_bind() -> None: + """Loopback is the server's DEFAULT bind, not a promise: `--host` takes any + address, and a hosted install serves through this entry point (Copilot + review of openDox-code#77, r4173844338). The help said every run served + "locally ... on a loopback address", which a hosted run is not. Where it + names loopback, it names the option that replaces it.""" + from opendox import serve + text = " ".join(serve.SERVE_DESCRIPTION.split()) + assert "loopback" in text, text + assert "unless --host" in text, text + assert not re.search(r"\blocally\b", text), text From 213344addcd45f70a6d54e2c23bc68615a090176 Mon Sep 17 00:00:00 2001 From: Brett Heap <1513478+brettheap@users.noreply.github.com> Date: Sat, 3 Oct 2026 18:12:52 +0000 Subject: [PATCH 69/88] T084 fix round 5: an in-root alias of a settings document is the settings document; the typeless package marker is set aside by its reason (Copilot review) - r4173903232 (holder: ACCEPT): `resolve_within` follows a symlink to the canonical file, but a scope row keeps the spelling it was named by. So `alias.md -> ideation/dashboard/model-provider-bindings.yaml`, or a directory link on the way to one, compared unequal to every SETTINGS_DOCUMENTS path, and stayed owned and editable. That bypassed M1. `default_columns._settings_test(root)` now compares the file a row REACHES with the files the settings documents reach. `_without_settings` moves such an alias, under its own name, into the settings section that nothing owns: readable, never editable. tests/test_column_seams.py:: test_an_in_root_alias_of_a_settings_document_is_never_editable, with the file alias and the directory alias. Before (1fb81cbd): 2 failed, e.g. ('a.md', 'b.md', 'alias.md') where ('a.md', 'b.md') was expected. Mutants killed: the target comparison dropped (2 failed); the alias compared by spelling (2 failed). - "Previously missed" (holder: ACCEPT): tests/test_static_content_types.py's UNSHIPPED claimed the wheel leaves `.gitkeep` out. Since T075 it ships (`web/**/.*`). Renamed TYPELESS_MARKERS: the shipped, empty, extensionless package marker has no content type to pin. The new test_the_set_aside_markers_are_empty_and_extensionless holds that reason. Mutant killed: the set widened to index.html (2 failed). Lane: openxfactory-4 (openXfactory-4-openDox_extraction) Co-Authored-By: Claude Opus 5.5 (1M context) --- src/opendox/default_columns.py | 49 ++++++++++++++++++++++++------ tests/test_column_seams.py | 34 +++++++++++++++++++++ tests/test_static_content_types.py | 34 ++++++++++++++++++--- 3 files changed, 102 insertions(+), 15 deletions(-) diff --git a/src/opendox/default_columns.py b/src/opendox/default_columns.py index 0502f94f..b00b195e 100644 --- a/src/opendox/default_columns.py +++ b/src/opendox/default_columns.py @@ -62,7 +62,7 @@ from dataclasses import dataclass from datetime import datetime, timezone from pathlib import Path, PurePosixPath -from typing import Any, Iterable, Mapping, NamedTuple, Sequence +from typing import Any, Callable, Iterable, Mapping, NamedTuple, Sequence from opendox import defaults from opendox.column_seams import GATE_RECORDS_REFUSAL @@ -364,8 +364,9 @@ def _section(key: str, label: str, note: str, references: Iterable[Any], *, #: talks to, or approve a pending declaration, through a proposal. So #: `resolve_scope` keeps them out of every owned section, into a section of #: their own that is readable and owned by nothing, and `editable_paths` -#: refuses them whatever section carries them. The corpus scan's own exclusion -#: (openDox-code#76) is a second layer, not this one. +#: refuses them whatever section carries them. An in-root symlink that reaches +#: one is treated as the document it reaches (`_settings_test`). The corpus +#: scan's own exclusion (openDox-code#76) is a second layer, not this one. SETTINGS_DOCUMENTS: frozenset[str] = frozenset({ DEFAULT_BINDINGS_RELPATH, DEFAULT_DECLARATIONS_RELPATH}) @@ -374,18 +375,46 @@ def _section(key: str, label: str, note: str, references: Iterable[Any], *, "editable through a tile") -def _without_settings(sections: Sequence[ScopeSection]) -> list[ScopeSection]: - """`sections` with every settings document moved out of an OWNED section - into one trailing section that nothing owns, in the order they appeared.""" +def _settings_test(root: Path) -> Callable[[str], bool]: + """Whether a row's path IS one of openDox's settings documents, by its + spelling or by the file it reaches under `root`. + + A SYMLINK ALIAS IS ONE (Copilot review of openDox-code#77, r4173903232). + `resolve_within` follows links to the canonical file, but a row keeps the + spelling it was named by, so `alias.md -> ideation/dashboard/ + model-provider-bindings.yaml` (or a directory link on the way) compared + unequal to every settings path and stayed owned and editable. So the + file a row resolves to is compared with the files the settings documents + resolve to, and an alias is moved out of the owned sections under its own + name, like the document it reaches.""" + from opendox import projection_seams + + resolve_within = projection_seams.registry.current().resolve_within + targets = {target for target in (resolve_within(root, name) + for name in SETTINGS_DOCUMENTS) + if target is not None} + + def is_settings(path: str) -> bool: + return path in SETTINGS_DOCUMENTS or ( + bool(targets) and resolve_within(root, path) in targets) + + return is_settings + + +def _without_settings(sections: Sequence[ScopeSection], *, + root: Path) -> list[ScopeSection]: + """`sections` with every settings document, or an alias that reaches one + (`_settings_test`), moved out of an OWNED section into one trailing + section that nothing owns, in the order they appeared.""" + is_settings = _settings_test(root) kept: list[ScopeSection] = [] moved: list[ScopeDocument] = [] for section in sections: if not section.owned: kept.append(section) continue - rows = [row for row in section.documents if row.path not in SETTINGS_DOCUMENTS] - moved.extend(row for row in section.documents - if row.path in SETTINGS_DOCUMENTS) + rows = [row for row in section.documents if not is_settings(row.path)] + moved.extend(row for row in section.documents if is_settings(row.path)) kept.append(ScopeSection(key=section.key, label=section.label, note=section.note, inherited=section.inherited, owned=True, documents=tuple(rows))) @@ -497,7 +526,7 @@ def members(group: Mapping[str, Any]) -> list[Any]: owned=True)) else: return None - sections = _without_settings(sections) + sections = _without_settings(sections, root=root) context = [row.path for section in sections for row in section.documents if row.resolved] for raw in created: diff --git a/tests/test_column_seams.py b/tests/test_column_seams.py index e4ff632d..bfbd722e 100644 --- a/tests/test_column_seams.py +++ b/tests/test_column_seams.py @@ -431,6 +431,40 @@ def test_opendoxs_own_settings_documents_are_never_editable( assert holder.key == "settings" and holder.owned is False +@pytest.mark.skipif(sys.platform == "win32", reason="creates symlinks") +@pytest.mark.parametrize("alias, link, to", [ + # a file alias, as review r4173903232 names it + ("alias.md", "alias.md", "ideation/dashboard/model-provider-bindings.yaml"), + # a directory link on the way to one + ("cfg/model-declarations.yaml", "cfg", "ideation/dashboard"), +]) +def test_an_in_root_alias_of_a_settings_document_is_never_editable( + corpus, alias, link, to) -> None: + """Copilot review of openDox-code#77, r4173903232: `resolve_within` + follows a symlink to the canonical file, but the row keeps its alias, so + an alias of a settings document compared unequal to every settings path + and stayed owned and editable. The file it REACHES is compared now: the + alias is readable, in the section nothing owns, and never editable.""" + for name in dc.SETTINGS_DOCUMENTS: + (corpus / name).parent.mkdir(parents=True, exist_ok=True) + (corpus / name).write_text("schema_version: 1\n", encoding="utf-8") + (corpus / link).parent.mkdir(parents=True, exist_ok=True) + (corpus / link).symlink_to(corpus / to) + snapshot = _snapshot() + snapshot["documents"].append({"id": alias, "path": alias}) + snapshot["clusters"][0]["document_edges"].append({"document": alias}) + projection = dc.resolve_scope(snapshot, _key("cluster", "g1"), source_root=corpus) + assert projection.editable_paths == ("a.md", "b.md") + assert alias not in projection.active_document_candidates + owned = {row.path for section in projection.sections if section.owned + for row in section.documents} + assert alias not in owned + (holder,) = [section for section in projection.sections + if alias in {row.path for row in section.documents}] + assert holder.key == "settings" and holder.owned is False + assert alias in projection.context_paths, "readable, never editable" + + def test_the_editable_set_refuses_a_settings_document_in_any_section() -> None: """`editable_paths` itself, over an owned section that carries one.""" from opendox.doxbench_scope_types import ScopeDocument, ScopeSection diff --git a/tests/test_static_content_types.py b/tests/test_static_content_types.py index ff79ee27..cff4a808 100644 --- a/tests/test_static_content_types.py +++ b/tests/test_static_content_types.py @@ -15,7 +15,9 @@ 2. An extension the pin does not carry still falls back to the platform's table: the pin narrows nothing else. 3. Every extension the shipped bundle carries is pinned, so a file of a new - kind added to `src/opendox/web/` fails here until it is. + kind added to `src/opendox/web/` fails here until it is. The extensionless + package marker `vendor/.gitkeep` ships too, and is set aside with its + reason checked (`TYPELESS_MARKERS`): it has no type to pin. A CREATED FILE: no carve-manifest row (RULED OQ-C). """ @@ -36,14 +38,24 @@ PLAIN = ROOT / "tests" / "fixtures" / "plain-documents" WEB = ROOT / "src" / "opendox" / "web" -#: Files of the tree that the wheel does not ship (`pyproject.toml`'s -#: package data ships `web/**`, and setuptools leaves dotfiles out). -UNSHIPPED = {".gitkeep"} +#: The bundle's EXTENSIONLESS PACKAGE MARKERS, outside these content-type +#: checks. `vendor/.gitkeep` SHIPS: `pyproject.toml`'s package data carries +#: `web/**/.*` beside `web/**` (plan 034 T075), and +#: `tests_runtime/test_served_bundle.py` fetches it from an installed entry +#: point with every other bundle file. It is an empty file that keeps +#: `vendor/` in the tree, with no extension and so no content type to pin, +#: and no page loads it. So it is set aside here BY THAT REASON, which +#: `test_the_set_aside_markers_are_empty_and_extensionless` holds (Copilot +#: review of openDox-code#77, "previously missed": the name `UNSHIPPED` +#: claimed the wheel left it out, which T075 made untrue). +TYPELESS_MARKERS = {".gitkeep"} def _bundle() -> list[Path]: + """Every file of the bundle that carries a content type: every shipped + file but the typeless markers.""" return sorted(p for p in WEB.rglob("*") - if p.is_file() and p.name not in UNSHIPPED) + if p.is_file() and p.name not in TYPELESS_MARKERS) def _hostile(monkeypatch) -> None: @@ -141,6 +153,18 @@ def test_an_extension_outside_the_pin_still_reads_the_platform_table( assert _content_type(base, "/index.html") == (200, "text/html") +def test_the_set_aside_markers_are_empty_and_extensionless() -> None: + """What `TYPELESS_MARKERS` sets aside is what its reason says, and no + more: each name is present in the bundle, and every file by it is empty + and has no extension, so no file with a type escapes the checks.""" + markers = [p for p in WEB.rglob("*") + if p.is_file() and p.name in TYPELESS_MARKERS] + assert {p.name for p in markers} == TYPELESS_MARKERS, markers + for path in markers: + assert path.suffix == "", path + assert path.stat().st_size == 0, path + + def test_every_extension_the_bundle_ships_is_pinned() -> None: from opendox import serve shipped = {path.suffix for path in _bundle()} From c979747ad441bd001c2642bd2a0938f7e0269458 Mon Sep 17 00:00:00 2001 From: Brett Heap <1513478+brettheap@users.noreply.github.com> Date: Sat, 3 Oct 2026 18:13:22 +0000 Subject: [PATCH 70/88] T104 fix round 2: the copy is judged by its own path, and no static link serves it (Copilot review) Copilot's review of openDox-code#84 at cb899546, two threads, each with a case that failed first (review2-red.txt: DID NOT RAISE; GET /state-alias/console/.html answered 200): - r4173889265: the boundary judged the state directory only, so a served root equal to its console/ (OPENDOX_STATE_DIR outside every root, --web-dir = /console) let GET /.html serve the copy. The copy's own resolved path is now judged too: a served root that holds it refuses the write, by name, before anything is written. - r4173889294: the static handler follows links inside --web-dir, so web/state-alias -> served the copy. Links are not refused wholesale, because a governed host's composed web root (openxFactory scripts/ideation-dashboard-serve.py, _composed_web_root) is made of links out of the bundle. Instead publish marks the private-copy directory on the server (private_roots), and DashboardHandler.send_head answers 404 for any static target whose RESOLVED path is that directory or inside it, for GET and HEAD, files and listings, every port's copy included. A server that wrote no copy marks nothing and serves exactly as before. Arc: neutral-product-standalone-operability Lane: openxfactory-4 (openXfactory-4-openDox_extraction) Co-Authored-By: Claude Opus 5.5 (1M context) --- src/opendox/console_access.py | 27 +++++++++-- src/opendox/serve.py | 21 ++++++++ tests/test_console_token_delivery.py | 71 ++++++++++++++++++++++++++++ 3 files changed, 116 insertions(+), 3 deletions(-) diff --git a/src/opendox/console_access.py b/src/opendox/console_access.py index 73090c71..374d0754 100644 --- a/src/opendox/console_access.py +++ b/src/opendox/console_access.py @@ -366,7 +366,8 @@ def _opener_html(record: Mapping[str, Any]) -> str: def _refuse_a_served_state_dir(state_dir: Path, - served_roots: Iterable[Path | str]) -> None: + served_roots: Iterable[Path | str], *, + port: int) -> None: """The state directory may not BE, or lie inside, a root this plane serves. RULED by the holder on openxFactory#1220's review (Copilot @@ -387,6 +388,18 @@ def _refuse_a_served_state_dir(state_dir: Path, "copy is refused there and nothing is written. Set " f"{runtime_config.PREFIX}STATE_DIR to a directory outside the " "repositories this machine serves") + # AND THE COPY ITSELF (Copilot at openDox-code#84, r4173889265): a + # served root may be the state directory's `console/`, or anything + # else that holds the copy, while the state directory lies outside + # every served root. The copy's own resolved path is judged. + target = private_copy_path(resolved, port).resolve() + if served in target.parents: + raise ConsoleAccessRefused( + f"{runtime_config.PREFIX}STATE_DIR ({state_dir}) would put the " + f"console token's private copy at {target}, inside {served}, " + "which this plane serves, so the copy is refused there and " + f"nothing is written. Set {runtime_config.PREFIX}STATE_DIR to " + "a directory whose `console/` no served root holds") def write_private_copy(state_dir: Path | str, *, page_url: str, port: int, @@ -400,7 +413,7 @@ def write_private_copy(state_dir: Path | str, *, page_url: str, port: int, own earlier copy for the same port (a server restarted there), and refuses anything else already at that name.""" state = Path(state_dir) - _refuse_a_served_state_dir(state, tuple(served_roots)) + _refuse_a_served_state_dir(state, tuple(served_roots), port=port) record = { "schema_version": RECORD_SCHEMA_VERSION, "kind": RECORD_KIND, @@ -632,6 +645,14 @@ def publish(httpd: Any, *, page_url: str, raise ConsoleAccessRefused( f"the console token's private copy has no state directory: {exc}" ) from None - return write_private_copy(state, page_url=page_url, + copy = write_private_copy(state, page_url=page_url, port=int(httpd.server_address[1]), token=token, served_roots=getattr(httpd, "served_roots", ())) + # THE STATIC HANDLER NEVER SERVES A COPY (Copilot at openDox-code#84, + # r4173889294). It follows links inside `--web-dir` (a governed host's + # composed web root is made of them), so a link out of the bundle into the + # state directory would reach the copies. The handler refuses every static + # request whose resolved target is this directory or lies inside it + # (`serve.DashboardHandler.send_head`), every port's copy included. + httpd.private_roots = (copy.path.parent.resolve(),) + return copy diff --git a/src/opendox/serve.py b/src/opendox/serve.py index 8b66ab56..39f31e4d 100644 --- a/src/opendox/serve.py +++ b/src/opendox/serve.py @@ -1333,6 +1333,27 @@ def _route(self, head_only: bool) -> bool: return True return False + def send_head(self): + """The static bundle's file, as `SimpleHTTPRequestHandler` serves it, + except a target inside a console token's private-copy directory. + + The stdlib handler follows links inside `--web-dir`, and a governed + host's composed web root is MADE of links out of it, so links are not + refused wholesale. But a link into the state directory must never + serve the token's copy (plan 034 T104; Copilot at openDox-code#84, + r4173889294): a target whose RESOLVED path is a directory the entry + point marked private (`console_access.publish` sets + `private_roots` on the server), or lies inside one, is a 404, for GET + and HEAD, files and listings alike. A server with no copy marks + nothing, and serves exactly as before.""" + private = getattr(self.server, "private_roots", ()) + if private: + target = Path(self.translate_path(self.path)).resolve() + if any(target == root or root in target.parents for root in private): + self.send_error(404, "File not found") + return None + return super().send_head() + def do_GET(self): # noqa: N802 if not self._route(head_only=False): super().do_GET() diff --git a/tests/test_console_token_delivery.py b/tests/test_console_token_delivery.py index 62e8943b..c71186fc 100644 --- a/tests/test_console_token_delivery.py +++ b/tests/test_console_token_delivery.py @@ -954,3 +954,74 @@ def test_a_plain_kill_removes_the_copy(tmp_path, monkeypatch, entry) -> None: assert "Traceback" not in child.stderr_text(), child.stderr_text() finally: child.kill() + + +# --------------------------------------------------------------------------- +# 8 — the static handler never serves a private copy (Copilot review 2) +# --------------------------------------------------------------------------- + +def test_a_served_root_equal_to_the_console_directory_is_refused(tmp_path) -> None: + """Copilot at openDox-code#84, r4173889265. The state directory is outside + every served root, but the static bundle IS its `console/` directory, so + the copy would be `GET /.html`. The copy's own path is judged + against the served roots, so this is refused by name before anything is + written.""" + from opendox import console_access + + state = _state(tmp_path) + console = state / console_access.CONSOLE_DIRNAME + console.mkdir(mode=0o700) + with pytest.raises(console_access.ConsoleAccessRefused, + match="OPENDOX_STATE_DIR") as refused: + console_access.write_private_copy( + state, page_url="http://127.0.0.1:8080/index.html", port=8080, + token=_token(), served_roots=(console,)) + assert str(console.resolve()) in str(refused.value) + assert list(console.iterdir()) == [], "something was written" + + +def test_a_static_link_out_of_the_bundle_never_serves_a_private_copy( + tmp_path, monkeypatch, standalone_profile) -> None: + """Copilot at openDox-code#84, r4173889294. The static handler follows a + link inside `--web-dir` (a governed host's composed web root is MADE of + such links, so they cannot be refused wholesale). A link that leads into + the state directory must still never serve a copy: every static request + whose resolved target is the private-copy directory, or inside it, is + answered 404, for GET and HEAD, the copy, the directory listing, and + another port's copy alike. The bundle itself still answers.""" + import shutil as _shutil + + from opendox import console_access, serve + + _clean_git(monkeypatch) + repo = _repository(tmp_path) + web = tmp_path / "web" + _shutil.copytree(WEB, web) + state = _state(tmp_path) + (web / "state-alias").symlink_to(state) + snapshot = tmp_path / "snapshot.json" + snapshot.write_text(json.dumps({"generation": {}}), encoding="utf-8") + httpd = serve.build_server(web, snapshot, repo, port=0, quiet=True) + worker = threading.Thread(target=httpd.serve_forever, daemon=True) + worker.start() + try: + base = httpd.server_address[:2] + copy = console_access.publish( + httpd, page_url=serve.server_url(httpd, "/index.html"), + env={"OPENDOX_STATE_DIR": str(state)}) + other = _write(state, port=9, token=_token()) # another serve's copy + token = httpd.console_token + for path in (f"/state-alias/console/{copy.path.name}", + f"/state-alias/console/{other.path.name}", + "/state-alias/console/", "/state-alias/console"): + for method in ("GET", "HEAD"): + status, headers, raw = _call(base, method, path) + assert status == 404, (method, path, status) + assert token.encode() not in raw and copy.path.name.encode() not in raw + assert all(token not in str(v) for v in headers.values()) + status, _headers, raw = _call(base, "GET", "/index.html") + assert status == 200 and b" Date: Sat, 3 Oct 2026 18:42:51 +0000 Subject: [PATCH 71/88] T104: the state directory and every served root overlap in neither direction The holder's ruling on batch N's review (Copilot r4174345203, measured at cb899546): _refuse_a_served_state_dir checked one direction only (the state directory equal to, or inside, a served root), so a served root that is, or lies inside, the state directory (/console itself, or the bundle's tree beside it) let the plane serve the state directory's contents, the token's copy among them. The check is now two-way, on resolved paths, refused by name before anything is written. It subsumes fix round 2's check of the copy's own path (a root holding the copy is the state directory, inside it, or above it), which is dropped for the one rule. Falsifiers (ruling-reverse-red.txt): - test_a_served_root_inside_the_state_directory_is_refused, for /console, /postgres/run, a deep path and the state directory itself. At cb899546 the first three failed (DID NOT RAISE); at c979747a the console case already held (fix round 2) and the other two still failed. - test_a_source_link_into_the_state_directory_never_serves_the_copy pins that /source (default_registry.resolve_within) answers 404 for links in the checkout into the state directory, unkeyed and keyed, GET and HEAD. It held before the fix, as the ruling expected. Arc: neutral-product-standalone-operability Lane: openxfactory-4 (openXfactory-4-openDox_extraction) Co-Authored-By: Claude Opus 5.5 (1M context) --- src/opendox/console_access.py | 55 +++++++++++---------- tests/test_console_token_delivery.py | 73 ++++++++++++++++++++++++++++ 2 files changed, 103 insertions(+), 25 deletions(-) diff --git a/src/opendox/console_access.py b/src/opendox/console_access.py index 374d0754..30e54b92 100644 --- a/src/opendox/console_access.py +++ b/src/opendox/console_access.py @@ -368,38 +368,43 @@ def _opener_html(record: Mapping[str, Any]) -> str: def _refuse_a_served_state_dir(state_dir: Path, served_roots: Iterable[Path | str], *, port: int) -> None: - """The state directory may not BE, or lie inside, a root this plane serves. + """The state directory and every root this plane serves may not overlap, + in EITHER direction, or the copy is refused before anything is written. RULED by the holder on openxFactory#1220's review (Copilot `r4171166321`), mirroring T100's served-repository boundary (`doxbench_trust`'s state directory): the token's copy must never sit - inside what `/source` can serve, nor where a clone or an accidental commit - could carry it. Asked of the RESOLVED paths, before anything is written.""" + inside what the plane can serve, nor where a clone or an accidental + commit could carry it. So the state directory may not BE a served root + or lie inside one. And, by the holder's ruling on batch N's review + (Copilot `r4174345203`), no served root may be or lie inside the state + directory either: `/console` itself, or the bundle's tree beside + it, served as a root, would serve the copy (Copilot at openDox-code#84, + `r4173889265`, found the first of these). Asked of the RESOLVED paths, so + a link counts as where it leads. `port` names the copy the refusal is + about.""" resolved = Path(state_dir).resolve() + copy = private_copy_path(resolved, port) for root in served_roots: served = Path(root).resolve() - if resolved == served or served in resolved.parents: - where = ("is the served repository" if resolved == served - else "lies inside the served repository") - raise ConsoleAccessRefused( - f"{runtime_config.PREFIX}STATE_DIR ({state_dir}) {where} " - f"({served}), which this plane serves through `/source` and a " - "clone or a commit could carry, so the console token's private " - "copy is refused there and nothing is written. Set " - f"{runtime_config.PREFIX}STATE_DIR to a directory outside the " - "repositories this machine serves") - # AND THE COPY ITSELF (Copilot at openDox-code#84, r4173889265): a - # served root may be the state directory's `console/`, or anything - # else that holds the copy, while the state directory lies outside - # every served root. The copy's own resolved path is judged. - target = private_copy_path(resolved, port).resolve() - if served in target.parents: - raise ConsoleAccessRefused( - f"{runtime_config.PREFIX}STATE_DIR ({state_dir}) would put the " - f"console token's private copy at {target}, inside {served}, " - "which this plane serves, so the copy is refused there and " - f"nothing is written. Set {runtime_config.PREFIX}STATE_DIR to " - "a directory whose `console/` no served root holds") + if resolved == served: + where = f"is the served repository ({served})" + elif served in resolved.parents: + where = f"lies inside the served repository ({served})" + elif resolved in served.parents: + where = (f"holds {served}, a root this plane serves, so the plane " + f"would serve what the state directory keeps, the copy " + f"{copy} among it") + else: + continue + raise ConsoleAccessRefused( + f"{runtime_config.PREFIX}STATE_DIR ({state_dir}) {where}. The " + "state directory and every root this plane serves (through " + "`/source` or the static bundle) may not overlap, and a clone or a " + "commit could carry what lies inside a repository, so the console " + "token's private copy is refused there and nothing is written. Set " + f"{runtime_config.PREFIX}STATE_DIR to a directory apart from the " + "repositories and the bundle this machine serves") def write_private_copy(state_dir: Path | str, *, page_url: str, port: int, diff --git a/tests/test_console_token_delivery.py b/tests/test_console_token_delivery.py index c71186fc..e0845edd 100644 --- a/tests/test_console_token_delivery.py +++ b/tests/test_console_token_delivery.py @@ -1025,3 +1025,76 @@ def test_a_static_link_out_of_the_bundle_never_serves_a_private_copy( httpd.shutdown() httpd.server_close() worker.join(timeout=10) + + +# --------------------------------------------------------------------------- +# 9 — no overlap in EITHER direction (the holder's ruling on batch N's +# Copilot r4174345203) +# --------------------------------------------------------------------------- + +@pytest.mark.parametrize("inside", ["console", "postgres/run", "anything/else/deep", + "."]) +def test_a_served_root_inside_the_state_directory_is_refused(tmp_path, inside) -> None: + """The REVERSE nesting. A served root that IS, or lies inside, the state + directory (`/console` itself, the bundle's socket tree, anything) + would let the plane serve the state directory's contents, the copy among + them. The state directory and every served root may not overlap in either + direction: refused by name, before anything is written.""" + from opendox import console_access + + state = _state(tmp_path) + served = (state / inside).resolve() if inside != "." else state.resolve() + served.mkdir(parents=True, exist_ok=True, mode=0o700) + before = _tree(state) + with pytest.raises(console_access.ConsoleAccessRefused, + match="OPENDOX_STATE_DIR") as refused: + console_access.write_private_copy( + state, page_url="http://127.0.0.1:8080/index.html", port=8080, + token=_token(), served_roots=(served,)) + assert str(served) in str(refused.value), str(refused.value) + assert _tree(state) == before, "something was written" + + +def test_a_source_link_into_the_state_directory_never_serves_the_copy( + tmp_path, monkeypatch, standalone_profile) -> None: + """Pinned: `/source` confines every path to its root AFTER resolving links + (`default_registry.resolve_within`), so a link inside the served checkout + that points at the state directory reaches nothing in it. The copy, its + directory and the state directory itself answer 404, unkeyed and keyed, + for GET and HEAD, and no answer carries the token.""" + from opendox import console_access, serve + + _clean_git(monkeypatch) + repo = _repository(tmp_path) + state = _state(tmp_path) + (repo / "state-link").symlink_to(state) + (repo / "copy-link.md").symlink_to( + console_access.private_copy_path(state, 1)) # dangling until written + snapshot = tmp_path / "snapshot.json" + snapshot.write_text(json.dumps({"generation": {}}), encoding="utf-8") + httpd = serve.build_server(WEB, snapshot, repo, port=0, quiet=True) + worker = threading.Thread(target=httpd.serve_forever, daemon=True) + worker.start() + try: + base = httpd.server_address[:2] + copy = console_access.publish( + httpd, page_url=serve.server_url(httpd, "/index.html"), + env={"OPENDOX_STATE_DIR": str(state)}) + _write(state, port=1) # copy-link.md now resolves + token = httpd.console_token + name = copy.path.name + for tail in (f"state-link/console/{name}", "state-link/console/1.html", + "copy-link.md", "state-link/console/", "state-link/"): + for prefix in ("/source/", "/source/fixture@main/"): + for method in ("GET", "HEAD"): + status, headers, raw = _call(base, method, prefix + tail) + assert status == 404, (method, prefix + tail, status, raw[:200]) + assert token.encode() not in raw + assert all(token not in str(v) for v in headers.values()) + document = next(p.name for p in sorted(repo.glob("*.md")) if not p.is_symlink()) + status, _headers, raw = _call(base, "GET", f"/source/{document}") + assert status == 200 and raw == (repo / document).read_bytes() + finally: + httpd.shutdown() + httpd.server_close() + worker.join(timeout=10) From 219d7cf9d83c079bf841e2f73b724d746a447d11 Mon Sep 17 00:00:00 2001 From: Brett Heap <1513478+brettheap@users.noreply.github.com> Date: Sat, 3 Oct 2026 20:19:11 +0000 Subject: [PATCH 72/88] T104 fix round 3: a directory's index page is judged, and a FIFO never blocks a read (Copilot review) Copilot's review at 60bace00, two threads, each case failing first: - r4174674625, a directory index. For a directory request the stdlib handler serves the directory's first index page that exists (index_pages: index.html, then index.htm), and send_head judged only the directory. So GET /sub/ served web/sub/index.html, a link to the console token's copy, while /sub/index.html itself answered 404. The index page the handler would pick is now judged as well. Case: test_a_directory_index_linked_to_a_private_copy_is_never_served, for index.html and index.htm, GET and HEAD of /sub/, /sub/ and /sub/?x=1, and the bare /sub redirect. A directory whose index page is the bundle's own still serves it. It failed first: GET /sub/ answered 200. - r4174674702, a FIFO at the copy. read_private_copy's read-only open of a FIFO with no writer blocked forever, before the descriptor's regular-file check could refuse it. The open now adds O_NONBLOCK, so the read refuses it as not a regular file at once. A write already refused it by name. Case: test_a_fifo_at_the_copy_is_refused_without_blocking. The read and the write each run on a thread with a bounded join, and the open is released if blocked, so the case fails and never hangs. It failed first: "the read blocked on a FIFO". Arc: neutral-product-standalone-operability Lane: openxfactory-4 (openXfactory-4-openDox_extraction) Co-Authored-By: Claude Opus 5.5 (1M context) --- src/opendox/console_access.py | 12 +++- src/opendox/serve.py | 26 +++++-- tests/test_console_token_delivery.py | 100 +++++++++++++++++++++++++++ 3 files changed, 130 insertions(+), 8 deletions(-) diff --git a/src/opendox/console_access.py b/src/opendox/console_access.py index 30e54b92..6a9f5c9c 100644 --- a/src/opendox/console_access.py +++ b/src/opendox/console_access.py @@ -489,8 +489,9 @@ def read_private_copy(path: Path | str) -> dict: """The record in the copy at `path`, or a refusal naming why. The tree is judged again, and the file by its own descriptor, opened - without following a link: a regular file, this user's, exactly 0600, with - one link. A planted, linked or loosened copy is refused.""" + without following a link and without blocking: a regular file, this + user's, exactly 0600, with one link. A planted, linked or loosened copy is + refused, and so is a FIFO, without waiting on it.""" target = Path(path) state = target.parent.parent if target.parent.name != CONSOLE_DIRNAME: @@ -515,8 +516,13 @@ def read_private_copy(path: Path | str) -> dict: reason = _unsafe_because(os.fstat(directory), uid=uid, own=True) if reason is not None: raise _unsafe(target.parent, reason) + # `O_NONBLOCK` (Copilot at openDox-code#84, r4174674702): a FIFO + # planted at the name, with no writer, would block a plain read-only + # `open` forever, before the descriptor's regular-file check below + # could refuse it. A regular file reads the same either way. try: - handle = os.open(target.name, os.O_RDONLY | os.O_NOFOLLOW, + handle = os.open(target.name, + os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK, dir_fd=directory) except FileNotFoundError: raise ConsoleAccessRefused(f"{target} does not exist: no plane on " diff --git a/src/opendox/serve.py b/src/opendox/serve.py index f4a861f0..d7051e37 100644 --- a/src/opendox/serve.py +++ b/src/opendox/serve.py @@ -1345,13 +1345,29 @@ def send_head(self): point marked private (`console_access.publish` sets `private_roots` on the server), or lies inside one, is a 404, for GET and HEAD, files and listings alike. A server with no copy marks - nothing, and serves exactly as before.""" + nothing, and serves exactly as before. + + A DIRECTORY REQUEST IS JUDGED BY WHAT IT SERVES (Copilot at + openDox-code#84, r4174674625). For `/sub/` the stdlib handler serves + the directory's first index page that exists (`index_pages`: + `index.html`, then `index.htm`), so the index page it would pick is + judged as well as the directory, and `web/sub/index.html` linked to a + copy is a 404 like the link itself.""" private = getattr(self.server, "private_roots", ()) if private: - target = Path(self.translate_path(self.path)).resolve() - if any(target == root or root in target.parents for root in private): - self.send_error(404, "File not found") - return None + path = Path(self.translate_path(self.path)) + judged = [path] + if path.is_dir(): + for name in getattr(self, "index_pages", ("index.html", "index.htm")): + if (path / name).is_file(): + judged.append(path / name) + break + for candidate in judged: + target = candidate.resolve() + if any(target == root or root in target.parents + for root in private): + self.send_error(404, "File not found") + return None return super().send_head() def do_GET(self): # noqa: N802 diff --git a/tests/test_console_token_delivery.py b/tests/test_console_token_delivery.py index e0845edd..dcd25777 100644 --- a/tests/test_console_token_delivery.py +++ b/tests/test_console_token_delivery.py @@ -1098,3 +1098,103 @@ def test_a_source_link_into_the_state_directory_never_serves_the_copy( httpd.shutdown() httpd.server_close() worker.join(timeout=10) + + +# --------------------------------------------------------------------------- +# 10 — Copilot review 4: a directory's index page, and a FIFO at the copy +# --------------------------------------------------------------------------- + +@pytest.mark.parametrize("index", ["index.html", "index.htm"]) +def test_a_directory_index_linked_to_a_private_copy_is_never_served( + tmp_path, monkeypatch, standalone_profile, index) -> None: + """Copilot at openDox-code#84, r4174674625. For a directory request the + stdlib handler serves the directory's first index page that exists + (`index.html`, then `index.htm`), so judging only the directory let + `GET /sub/` serve `web/sub/`, a link to the console token's copy, + while `/sub/` itself answered 404. The index page the handler + would serve is judged too: GET and HEAD of the directory answer 404, the + redirect of the bare name carries nothing, and no answer carries the + token. A directory whose index page is the bundle's own still serves it.""" + import shutil as _shutil + + from opendox import console_access, serve + + _clean_git(monkeypatch) + repo = _repository(tmp_path) + web = tmp_path / "web" + _shutil.copytree(WEB, web) + (web / "sub").mkdir() + (web / "plain").mkdir() + (web / "plain" / index).write_text("plain index", encoding="utf-8") + state = _state(tmp_path) + snapshot = tmp_path / "snapshot.json" + snapshot.write_text(json.dumps({"generation": {}}), encoding="utf-8") + httpd = serve.build_server(web, snapshot, repo, port=0, quiet=True) + worker = threading.Thread(target=httpd.serve_forever, daemon=True) + worker.start() + try: + base = httpd.server_address[:2] + copy = console_access.publish( + httpd, page_url=serve.server_url(httpd, "/index.html"), + env={"OPENDOX_STATE_DIR": str(state)}) + (web / "sub" / index).symlink_to(copy.path) + token = httpd.console_token + for path in ("/sub/", f"/sub/{index}", "/sub/?x=1"): + for method in ("GET", "HEAD"): + status, headers, raw = _call(base, method, path) + assert status == 404, (method, path, status) + assert token.encode() not in raw + assert all(token not in str(v) for v in headers.values()) + status, headers, raw = _call(base, "GET", "/sub") + assert status != 200, status + assert token.encode() not in raw + assert all(token not in str(v) for v in headers.values()) + for method in ("GET", "HEAD"): + status, _headers, raw = _call(base, method, "/plain/") + assert status == 200, (method, status) + assert b"plain index" in _call(base, "GET", "/plain/")[2] + finally: + httpd.shutdown() + httpd.server_close() + worker.join(timeout=10) + + +@pytest.mark.skipif(not hasattr(os, "mkfifo"), reason="no FIFOs on this platform") +def test_a_fifo_at_the_copy_is_refused_without_blocking(tmp_path) -> None: + """Copilot at openDox-code#84, r4174674702. A FIFO planted at + `console/.html`, with no writer, blocked a read in its `open` + before the descriptor's regular-file check could run, so the reader + hung instead of refusing. The read opens without blocking and refuses it + as not a regular file. A write refuses it too, by its name, and neither + ever waits on it.""" + from opendox import console_access + + state = _state(tmp_path) + (state / console_access.CONSOLE_DIRNAME).mkdir(mode=0o700) + fifo = console_access.private_copy_path(state, 8080) + os.mkfifo(fifo, 0o600) + outcome: dict = {} + + def attempt(name, call) -> None: + try: + call() + except BaseException as exc: # noqa: BLE001 — judged below + outcome[name] = exc + else: + outcome[name] = None + + for name, call in (("read", lambda: console_access.read_private_copy(fifo)), + ("write", lambda: _write(state))): + worker = threading.Thread(target=attempt, args=(name, call), daemon=True) + worker.start() + worker.join(timeout=10) + if worker.is_alive(): + # Release the blocked open, so the case fails rather than hangs. + with contextlib.suppress(OSError): + os.close(os.open(fifo, os.O_WRONLY | os.O_NONBLOCK)) + worker.join(timeout=10) + pytest.fail(f"the {name} blocked on a FIFO at {fifo}") + assert isinstance(outcome[name], console_access.ConsoleAccessRefused), ( + name, outcome[name]) + assert "not a regular file" in str(outcome[name]), str(outcome[name]) + assert stat.S_ISFIFO(os.lstat(fifo).st_mode), "the FIFO was replaced" From a13857ad26a1dbdca8717334fbd43c11a1332df7 Mon Sep 17 00:00:00 2001 From: Brett Heap <1513478+brettheap@users.noreply.github.com> Date: Sat, 3 Oct 2026 21:40:22 +0000 Subject: [PATCH 73/88] T104: every clause of #1144 12.4a as batch N amends it, and the opener file's lifecycle self-pass openxFactory#1222 (T007 batch N) landed as bdd0f586, and its amended 12.4a is the normative text for T104. Each clause was checked against d4b99436. The gaps batch N named at c979747a, (c) a non-blocking reader and (d) the automatic index file, closed in fix round 3 (219d7cf9). This closes the rest. Each new case failed first at d4b99436, unless it is marked a pin. 12.4a: - (a) "replaced only when it is this user's own regular file of mode 0600 with one link". The writer asked for the type, the owner and the link count, not the mode, so a loosened own copy was REPLACED. It is now judged by the reader's own rule (_file_unsafe_because), refused by name and left as it is. - "in a directory of mode 0700". A console/ loosened after it was made (0755, 0750, 0711) was accepted wherever no one else could write it. The writer and the reader now refuse it by name. Only the permission bits count, so a setgid bit inherited from a setgid parent is accepted (a pin, held by mutant M17b). - (b) the writer's and the entry points' regressions. Another user's file at the name (a pin). Through generate-and-open and python -m opendox.serve, each of a symbolic link, a directory, a FIFO, a hard-linked copy and a loosened copy refuses the START by name: exit 1, nothing that serves printed, no browser, the planted thing untouched, the socket closed (the loosened copy failed first; the rest are pins). - (e) a served root named through a symbolic link that leads to the state directory, or into it (console, postgres/run, the directory itself), is judged where it leads. Also through the entry point, as a --web-dir link to /console (pins, held by mutant M24). The lifecycle self-pass (write, replace, read, remove at stop, refuse before write): - An operating-system refusal on the way (a parent that will not let this user make the state directory, a full disk) escaped as a raw OSError, a traceback and no refusal by name. It is now a ConsoleAccessRefused naming the copy, through both entry points. - A write that fails part way removes its temporary file. - A copy whose read-back fails is removed with the refusal. - Removal put another serve's copy back by a hard link only. Where links fail (EPERM: a filesystem without them, or a directory) that copy was DELETED. It is renamed back where the name is free. - SIGHUP, which a closed terminal sends, ended the process with the copy left behind. It is read as Ctrl-C while a copy exists, as SIGTERM is, unless the process was started ignoring it (nohup). - The signal handling covered only the serve loop. A SIGTERM while the browser opener ran, which can take seconds, took SIGTERM's default on a hosted standalone plane, and the copy was left. It now covers the whole window from the write to the stop, in both entry points. Arc: neutral-product-standalone-operability Lane: openxfactory-4 (openXfactory-4-openDox_extraction) Co-Authored-By: Claude Opus 5.5 (1M context) --- src/opendox/cli.py | 63 ++-- src/opendox/console_access.py | 196 ++++++++--- src/opendox/serve.py | 23 +- tests/test_console_token_delivery.py | 468 +++++++++++++++++++++++++++ 4 files changed, 661 insertions(+), 89 deletions(-) diff --git a/src/opendox/cli.py b/src/opendox/cli.py index af9ddded..c0a986f0 100644 --- a/src/opendox/cli.py +++ b/src/opendox/cli.py @@ -914,37 +914,42 @@ def _generate_and_serve(args: argparse.Namespace, run_dir: Path, *, print(f"generate-and-open refused: {exc}", file=sys.stderr) return 1 try: - print(f" serving {url}") - print(f" snapshot {serve_mod.server_url(httpd, '/snapshot.json')}") - if console is not None: - print(f" console {console.file_url} (this user's private copy, " - "mode 0600: open it to open the console page again)") - # The URL is ALWAYS printed on its own line, AND FLUSHED (plan 034 - # T056). Where standard output is a pipe or a file, Python buffers it - # by block, and the process is about to block in `serve_forever()`. - # So without the flush, a wrapper reading this line never sees it - # while the server runs, and it cannot learn an ephemeral port or tell - # that the server started. Measured at openDox-code#59 e3ef506a: zero - # lines in 20 s on a pipe. It carries no token. - print(url, flush=True) - - if not args.no_open: - try: - opener(console.file_url if console is not None else url) - except Exception as exc: # a headless box has no browser — never fatal - print(f" (could not open a browser: {exc}; open the " - f"{'console file' if console is not None else 'URL'} " - "above manually)") - - if args.no_serve: - return 0 - - print(" serving until interrupted (Ctrl-C to stop)", flush=True) - # A plain `kill` stops a standalone console the way Ctrl-C does, so - # the copy below is removed (`terminate_as_interrupt`); a plane that - # wrote no copy keeps SIGTERM's default action. + # A plain `kill`, or a closed terminal, stops a standalone console the + # way Ctrl-C does, so the copy below is removed + # (`terminate_as_interrupt`); a plane that wrote no copy keeps the + # signals' default actions. It covers the WHOLE window from the write + # to the stop, the browser opener included, which can take seconds + # (T104's self-pass), and a stop asked for there is a clean stop too. with console_access.terminate_as_interrupt(console is not None): try: + print(f" serving {url}") + print(f" snapshot {serve_mod.server_url(httpd, '/snapshot.json')}") + if console is not None: + print(f" console {console.file_url} (this user's private " + "copy, mode 0600: open it to open the console page " + "again)") + # The URL is ALWAYS printed on its own line, AND FLUSHED (plan + # 034 T056). Where standard output is a pipe or a file, Python + # buffers it by block, and the process is about to block in + # `serve_forever()`. So without the flush, a wrapper reading + # this line never sees it while the server runs, and it cannot + # learn an ephemeral port or tell that the server started. + # Measured at openDox-code#59 e3ef506a: zero lines in 20 s on + # a pipe. It carries no token. + print(url, flush=True) + + if not args.no_open: + try: + opener(console.file_url if console is not None else url) + except Exception as exc: # a headless box has no browser — never fatal + print(f" (could not open a browser: {exc}; open the " + f"{'console file' if console is not None else 'URL'} " + "above manually)") + + if args.no_serve: + return 0 + + print(" serving until interrupted (Ctrl-C to stop)", flush=True) httpd.serve_forever() except KeyboardInterrupt: pass diff --git a/src/opendox/console_access.py b/src/opendox/console_access.py index 6a9f5c9c..707095a6 100644 --- a/src/opendox/console_access.py +++ b/src/opendox/console_access.py @@ -41,24 +41,34 @@ are that module's private helpers and its refusal names a socket: * the state directory and `console/` must be real directories, this user's, - writable by no one else; every directory above them must be this user's or - root's, and one that others can write must be sticky; every symbolic link - on the configured path must be this user's or root's; + writable by no one else, and `console/` exactly 0700; every directory + above them must be this user's or root's, and one that others can write + must be sticky; every symbolic link on the configured path must be this + user's or root's; * a missing directory is made relative to its parent's DESCRIPTOR, born 0700, and opened without following a link before anything is made under it; * the file is created exclusively, without following a link, set to exactly 0600 by its descriptor, and renamed into place. A name already at the - target that is not this user's own regular file (a link, a directory, a - file another user owns, a file with a second hard link) is REFUSED, never - followed or replaced; + target that is not this user's own regular file of mode 0600 with one + link (a link, a directory, a FIFO, a file another user owns, a file with + a second hard link, a loosened file) is REFUSED, never followed or + replaced; * a READ asks all of it again of what exists, and of the file by its - descriptor: a regular file, this user's, exactly 0600, one link; - * and the state directory may not BE, or lie inside, a root the plane - serves (its checkout and any declared source root), by name and before - any write, as T100's served-repository boundary refuses its own: the - token must never sit inside what `/source` can serve (holder's ruling on - openxFactory#1220's review, Copilot `r4171166321`). + descriptor, opened without blocking: a regular file, this user's, + exactly 0600, one link; + * the state directory and every root the plane serves may not overlap in + either direction, by name and before any write, as T100's + served-repository boundary refuses its own (holder's rulings on + openxFactory#1220's review, Copilot `r4171166321`, and on batch N's, + `r4174345203`); + * every refusal names its path, an operating-system one included, so an + entry point refuses its start by name; and the copy is removed when the + server stops, by Ctrl-C, SIGTERM or SIGHUP, or when its start is refused + after it was written. + +#1144 12.4a, as T007 batch N amends it (openxFactory#1222), is the normative +text this module realizes. A CREATED FILE, with no carve-manifest row (RULED OQ-C). """ @@ -105,6 +115,9 @@ RECORD_ELEMENT_ID = "opendox-console" #: The one mode a private copy may have. PRIVATE_MODE = 0o600 +#: The one mode the copies' directory, `console/`, may have (#1144 12.4a: the +#: copy is mode 0600 "in a directory of mode 0700"). +CONSOLE_DIR_MODE = 0o700 #: A copy is a few hundred bytes; a read stops well past that. _READ_LIMIT = 64 * 1024 #: `secrets.token_urlsafe` spells a token in these characters only, so a token @@ -208,6 +221,20 @@ def _unsafe_because(info: os.stat_result, *, uid: int, own: bool) -> str | None: return None +def _console_dir_unsafe_because(info: os.stat_result, *, uid: int) -> str | None: + """Why the copies' own directory is unsafe, or `None`: the rules for + every directory this user owns on the path, and exactly mode 0700 (#1144 + 12.4a). A `console/` loosened after it was made is refused by name, as a + loosened copy is, even where no one else can write it.""" + reason = _unsafe_because(info, uid=uid, own=True) + # The PERMISSION bits only: a directory made under a setgid parent + # inherits the setgid bit, which grants no one access. + permissions = stat.S_IMODE(info.st_mode) & 0o777 + if reason is None and permissions != CONSOLE_DIR_MODE: + reason = f"has mode {permissions:o}, not {CONSOLE_DIR_MODE:o}" + return reason + + def _unsafe(path: Path, reason: str) -> ConsoleAccessRefused: return ConsoleAccessRefused( f"{path} {reason}, so another user could replace or read the console " @@ -247,7 +274,9 @@ def present(path: Path) -> bool: if not present(directory): continue info = os.lstat(directory) if mine else os.stat(directory) - reason = _unsafe_because(info, uid=uid, own=mine) + reason = (_console_dir_unsafe_because(info, uid=uid) + if directory == tree[1] + else _unsafe_because(info, uid=uid, own=mine)) if reason is not None: raise _unsafe(directory, reason) @@ -412,11 +441,20 @@ def write_private_copy(state_dir: Path | str, *, page_url: str, port: int, served_roots: Iterable[Path | str]) -> PrivateCopy: """Write the copy for the plane on `port`, mode 0600, or refuse. - `served_roots` are the roots this plane serves (`/source`'s checkout and - any declared source root): a state directory that is one of them, or lies - inside one, is refused before anything is written. Replaces this user's - own earlier copy for the same port (a server restarted there), and refuses - anything else already at that name.""" + `served_roots` are the roots this plane serves: the state directory and + any of them may not overlap, in either direction, and that is asked + before anything is written. A file already at the copy's path is replaced + ONLY when it is this user's own regular file of mode 0600 with one link, + an earlier serve's copy for this port (#1144 12.4a). Anything else there, + a loosened copy included, is refused by name and never replaced. + + EVERY REFUSAL NAMES ITS PATH (the opener file's lifecycle, T104's + self-pass). An operating-system refusal on the way (a parent that will + not let this user make the state directory, a full disk) is a + `ConsoleAccessRefused` naming the copy, so the entry point refuses its + start by name instead of ending in a traceback. And a copy whose + read-back fails is removed with the refusal, so a start that never served + leaves no copy behind.""" state = Path(state_dir) _refuse_a_served_state_dir(state, tuple(served_roots), port=port) record = { @@ -428,28 +466,53 @@ def write_private_copy(state_dir: Path | str, *, page_url: str, port: int, "pid": os.getpid(), FRAGMENT_KEY: token, } - _refuse_an_unsafe_tree(state, existing_only=True) target = private_copy_path(state, port) + try: + identity = _write_the_copy(state, target, record) + except OSError as exc: + raise ConsoleAccessRefused( + f"{target} cannot be written ({exc}), so the console token has no " + "private copy and the start is refused. Use a state directory this " + f"user can write ({runtime_config.PREFIX}STATE_DIR)") from None + copy = PrivateCopy(path=target, page_url=page_url, + opened_url=record["opened_url"], identity=identity) + try: + read_private_copy(target) # what was written is what a reader accepts + except BaseException: + remove_private_copy(copy) + raise + return copy + + +def _write_the_copy(state: Path, target: Path, + record: Mapping[str, Any]) -> tuple[int, int]: + """`write_private_copy`'s writing half: the tree judged and made, the + name judged, the file written beside it and renamed into place. Returns + the written file's `(st_dev, st_ino)`.""" + _refuse_an_unsafe_tree(state, existing_only=True) directory = _open_private_directory(target.parent, state=state) uid = os.getuid() temporary = f".{target.name}.opendox-{os.getpid()}" try: # Judged only after the directories exist: what `existing_only` could - # not see before they were made, it sees now. + # not see before they were made, it sees now. `console/` is judged by + # its descriptor too, its exact mode included. _refuse_an_unsafe_tree(state, existing_only=False) + reason = _console_dir_unsafe_because(os.fstat(directory), uid=uid) + if reason is not None: + raise _unsafe(target.parent, reason) try: present = os.stat(target.name, dir_fd=directory, follow_symlinks=False) except FileNotFoundError: present = None - # THIS USER'S OWN regular file, with one link, is an earlier serve's - # copy for this port, and is replaced. Anything else was PLANTED or - # LINKED there, and is refused, never followed or replaced. - if present is not None and not ( - stat.S_ISREG(present.st_mode) and present.st_uid == uid - and present.st_nlink == 1): - reason = (_file_unsafe_because(present, uid=uid) - or "is not this user's own file") + # THIS USER'S OWN regular file, of mode 0600, with one link, is an + # earlier serve's copy for this port, and is replaced. Anything else + # was PLANTED, LINKED or LOOSENED there, and is refused, never + # followed or replaced (#1144 12.4a). + reason = (None if present is None + else _file_unsafe_because(present, uid=uid)) + if reason is not None: raise ConsoleAccessRefused( f"{target} {reason}: something other than this user's own " "private copy is at that name, so it is refused, never " @@ -459,17 +522,19 @@ def write_private_copy(state_dir: Path | str, *, page_url: str, port: int, handle = os.open(temporary, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, PRIVATE_MODE, dir_fd=directory) + # From here a failure (a full disk, an interrupt) removes the + # temporary file it made, so no partial copy is left beside the name. try: - os.fchmod(handle, PRIVATE_MODE) - data = _opener_html(record).encode("utf-8") - view = memoryview(data) - while view: - view = view[os.write(handle, view):] - os.fsync(handle) - written = os.fstat(handle) - finally: - os.close(handle) - try: + try: + os.fchmod(handle, PRIVATE_MODE) + data = _opener_html(record).encode("utf-8") + view = memoryview(data) + while view: + view = view[os.write(handle, view):] + os.fsync(handle) + written = os.fstat(handle) + finally: + os.close(handle) os.replace(temporary, target.name, src_dir_fd=directory, dst_dir_fd=directory) except BaseException: @@ -478,11 +543,7 @@ def write_private_copy(state_dir: Path | str, *, page_url: str, port: int, raise finally: os.close(directory) - copy = PrivateCopy(path=target, page_url=page_url, - opened_url=record["opened_url"], - identity=(written.st_dev, written.st_ino)) - read_private_copy(target) # what was written is what a reader accepts - return copy + return (written.st_dev, written.st_ino) def read_private_copy(path: Path | str) -> dict: @@ -513,7 +574,7 @@ def read_private_copy(path: Path | str) -> dict: raise _unsafe(target.parent, _unsafe_because(info, uid=uid, own=True) or "cannot be opened") from None try: - reason = _unsafe_because(os.fstat(directory), uid=uid, own=True) + reason = _console_dir_unsafe_because(os.fstat(directory), uid=uid) if reason is not None: raise _unsafe(target.parent, reason) # `O_NONBLOCK` (Copilot at openDox-code#84, r4174674702): a FIFO @@ -576,7 +637,13 @@ def remove_private_copy(copy: PrivateCopy | None) -> None: name (never over a still newer copy) and its temporary name removed. The entry points also remove the copy BEFORE they close the listening socket, so no later serve can bind the port, and write its own copy, until this - one is gone.""" + one is gone. + + PUT BACK BY A RENAME WHERE A HARD LINK CANNOT BE MADE (T104's self-pass). + A filesystem without hard links refuses the link (EPERM), and so does a + directory, and the other serve's copy used to be deleted with the + temporary name. It is renamed back instead, where the name is still + free.""" if copy is None: return try: @@ -596,17 +663,32 @@ def remove_private_copy(copy: PrivateCopy | None) -> None: if (info.st_dev, info.st_ino) != copy.identity: # ANOTHER SERVE'S COPY: put it back under its name, unless a # still newer one has arrived there, which then stands. - with contextlib.suppress(FileExistsError): + try: os.link(taken, name, src_dir_fd=directory, dst_dir_fd=directory, follow_symlinks=False) + except FileExistsError: + pass + except OSError: + if not _name_exists(name, directory): + os.rename(taken, name, src_dir_fd=directory, + dst_dir_fd=directory) with contextlib.suppress(OSError): os.unlink(taken, dir_fd=directory) finally: os.close(directory) +def _name_exists(name: str, directory: int) -> bool: + try: + os.stat(name, dir_fd=directory, follow_symlinks=False) + except FileNotFoundError: + return False + return True + + class ConsoleTerminated(KeyboardInterrupt): - """SIGTERM, raised as the interrupt the serve loops already stop on.""" + """SIGTERM or SIGHUP, raised as the interrupt the serve loops already + stop on.""" def _terminate_as_interrupt(signum, frame): @@ -622,19 +704,33 @@ def terminate_as_interrupt(enabled: bool): way out. `enabled` is False wherever no copy was written, a host's plane or a plane with no token, and then nothing changes: those planes keep the signal's default action exactly as before. Off the main thread no handler - can be installed, and nothing is.""" + can be installed, and nothing is. + + AND SIGHUP (T104's self-pass), which a closed terminal sends and whose + default action ends the process with the copy left behind. It is read the + same way, but only where it still has its default action: a process + started ignoring it (`nohup`) keeps ignoring it.""" if not enabled: yield return + signals = [signal.SIGTERM] + hangup = getattr(signal, "SIGHUP", None) + if hangup is not None and signal.getsignal(hangup) == signal.SIG_DFL: + signals.append(hangup) + previous: dict = {} try: - previous = signal.signal(signal.SIGTERM, _terminate_as_interrupt) + for signum in signals: + previous[signum] = signal.signal(signum, _terminate_as_interrupt) except ValueError: + for signum, handler in previous.items(): + signal.signal(signum, handler) yield return try: yield finally: - signal.signal(signal.SIGTERM, previous) + for signum, handler in previous.items(): + signal.signal(signum, handler) def publish(httpd: Any, *, page_url: str, diff --git a/src/opendox/serve.py b/src/opendox/serve.py index d7051e37..5a49e58e 100644 --- a/src/opendox/serve.py +++ b/src/opendox/serve.py @@ -2615,18 +2615,21 @@ def serve( httpd.server_close() raise try: - if console is not None: - print(f"console {console.file_url} (this user's private copy, " - "mode 0600: open it to open the console page)") - # FLUSHED before the process blocks (plan 034 T056): where standard - # output is a pipe or a file it is block-buffered, so an unflushed line - # never reaches a wrapper while the server runs, and the wrapper cannot - # learn an ephemeral port or tell that the server started. - print(f"serving ideation dashboard at {page}", flush=True) - # A plain `kill` stops a standalone console the way Ctrl-C does, so the - # copy is removed; a plane that wrote none keeps SIGTERM's default. + # A plain `kill`, or a closed terminal, stops a standalone console the + # way Ctrl-C does, so the copy is removed; a plane that wrote none + # keeps the signals' defaults. It covers the whole window from the + # write to the stop (T104's self-pass). with console_access.terminate_as_interrupt(console is not None): try: + if console is not None: + print(f"console {console.file_url} (this user's private " + "copy, mode 0600: open it to open the console page)") + # FLUSHED before the process blocks (plan 034 T056): where + # standard output is a pipe or a file it is block-buffered, so + # an unflushed line never reaches a wrapper while the server + # runs, and the wrapper cannot learn an ephemeral port or tell + # that the server started. + print(f"serving ideation dashboard at {page}", flush=True) httpd.serve_forever() except KeyboardInterrupt: pass diff --git a/tests/test_console_token_delivery.py b/tests/test_console_token_delivery.py index dcd25777..11c5e92a 100644 --- a/tests/test_console_token_delivery.py +++ b/tests/test_console_token_delivery.py @@ -28,12 +28,15 @@ import argparse import contextlib +import errno import html.parser import http.client import json import os import re import signal +import socket +import socketserver import stat import threading import urllib.parse @@ -1198,3 +1201,468 @@ def attempt(name, call) -> None: name, outcome[name]) assert "not a regular file" in str(outcome[name]), str(outcome[name]) assert stat.S_ISFIFO(os.lstat(fifo).st_mode), "the FIFO was replaced" + + +# --------------------------------------------------------------------------- +# 11 — #1144 12.4a as amended by T007 batch N (openxFactory#1222, landed +# bdd0f586), clause by clause, and the opener file's lifecycle +# --------------------------------------------------------------------------- + +def _free_port() -> int: + with socket.socket() as probe: + probe.bind(("127.0.0.1", 0)) + return probe.getsockname()[1] + + +#: What 12.4a says may be at the copy's path, other than an earlier copy of +#: this user's, and the reason each is refused by. +_PLANTED = { + "a symbolic link": "is a symbolic link", + "a directory": "is not a regular file", + "a FIFO": "is not a regular file", + "a hard-linked copy": "has 2 hard links", + "a loosened copy": "has mode 644, not 600", +} + + +def _plant(state: Path, port: int, kind: str, tmp_path: Path) -> Path: + from opendox import console_access + + (state / console_access.CONSOLE_DIRNAME).mkdir(mode=0o700, exist_ok=True) + path = console_access.private_copy_path(state, port) + if kind == "a symbolic link": + bait = tmp_path / "bait.html" + bait.write_text("bait\n", encoding="utf-8") + path.symlink_to(bait) + elif kind == "a directory": + path.mkdir() + elif kind == "a FIFO": + os.mkfifo(path, 0o600) + else: + written = _write(state, port=port) + if kind == "a hard-linked copy": + os.link(written.path, tmp_path / "second-name.html") + else: + written.path.chmod(0o644) + return path + + +def _fingerprint(path: Path) -> tuple: + info = os.lstat(path) + return (stat.S_IFMT(info.st_mode), stat.S_IMODE(info.st_mode), info.st_ino, + info.st_nlink, os.readlink(path) if stat.S_ISLNK(info.st_mode) else None) + + +def _port_is_free(port: int) -> bool: + with socket.socket() as probe: + try: + probe.bind(("127.0.0.1", port)) + except OSError: + return False + return True + + +def test_a_loosened_own_copy_is_refused_by_the_writer_and_left_as_it_is( + tmp_path) -> None: + """12.4a: a file at the copy's path is replaced ONLY when it is this + user's own regular file of mode 0600 with one link. A loosened one is + refused by name, never replaced (batch N's gap (a) at `c979747a`: the + writer asked for the type, the owner and the link count, not the mode).""" + from opendox import console_access + + state = _state(tmp_path) + first = _write(state) + first.path.chmod(0o644) + before = (first.path.read_bytes(), _fingerprint(first.path)) + with pytest.raises(console_access.ConsoleAccessRefused, + match="has mode 644, not 600") as refused: + _write(state) + assert str(first.path) in str(refused.value) + assert (first.path.read_bytes(), _fingerprint(first.path)) == before + + +def test_another_users_file_at_the_copy_is_refused_by_the_writer( + tmp_path, monkeypatch) -> None: + """12.4a: another user's file at the copy's path refuses the write by + name and is never replaced. Simulated through the writer's own `stat` of + that name, so the tree above it is still this user's.""" + from opendox import console_access + + state = _state(tmp_path) + first = _write(state) + before = (first.path.read_bytes(), _fingerprint(first.path)) + real_stat = os.stat + + def foreign(path, *args, **kwargs): + info = real_stat(path, *args, **kwargs) + if path == first.path.name and kwargs.get("dir_fd") is not None: + values = list(info[:10]) + values[4] = info.st_uid + 1 # st_uid + return os.stat_result(values) + return info + + monkeypatch.setattr(console_access.os, "stat", foreign) + with pytest.raises(console_access.ConsoleAccessRefused, + match="not by this user"): + _write(state) + monkeypatch.undo() + assert (first.path.read_bytes(), _fingerprint(first.path)) == before + + +@pytest.mark.parametrize("kind", sorted(_PLANTED)) +def test_generate_and_open_refuses_by_name_what_was_planted_at_the_copy( + tmp_path, monkeypatch, capsys, standalone_profile, kind) -> None: + """12.4a, through the entry point: anything at the copy's path but an + earlier copy of this user's refuses the START by name. Nothing is printed + that serves, no browser is opened, the planted thing is untouched, and the + listening socket is closed, so the server never serves.""" + from opendox import cli + + _clean_git(monkeypatch) + repo = _repository(tmp_path) + state = _state(tmp_path) + monkeypatch.setenv("OPENDOX_STATE_DIR", str(state)) + port = _free_port() + planted = _plant(state, port, kind, tmp_path) + before = _fingerprint(planted) + opened: list[str] = [] + assert cli._generate_and_open( + _generate_and_open_args(tmp_path, repo, "--port", str(port)), + opener=opened.append) == 1 + out, err = capsys.readouterr() + assert opened == [] + assert "generate-and-open refused:" in err, err + assert str(planted) in err and _PLANTED[kind] in err, err + assert " console " not in out and f":{port}/" not in out, out + assert _fingerprint(planted) == before + assert _port_is_free(port), "the refused start kept its socket" + + +@pytest.mark.parametrize("kind", sorted(_PLANTED)) +def test_the_servers_own_entry_point_refuses_by_name_what_was_planted_at_the_copy( + tmp_path, monkeypatch, capsys, standalone_profile, kind) -> None: + """The same, through `python -m opendox.serve`'s `main`. A start that + did not refuse would serve: the serve loop here fails the case instead of + blocking it.""" + from opendox import serve + + _clean_git(monkeypatch) + repo = _repository(tmp_path) + snapshot = tmp_path / "snapshot.json" + snapshot.write_text(json.dumps({"generation": {}}), encoding="utf-8") + state = _state(tmp_path) + monkeypatch.setenv("OPENDOX_STATE_DIR", str(state)) + monkeypatch.setattr(serve, "real_notebook_adapter", lambda *a, **k: None) + + def served(self, *args, **kwargs): + raise AssertionError("the server served") + + monkeypatch.setattr(socketserver.BaseServer, "serve_forever", served) + port = _free_port() + planted = _plant(state, port, kind, tmp_path) + before = _fingerprint(planted) + assert serve.main(["--snapshot", str(snapshot), "--checkout-root", str(repo), + "--port", str(port)]) == 1 + out, err = capsys.readouterr() + assert "serve refused:" in err and str(planted) in err, err + assert _PLANTED[kind] in err, err + assert "serving ideation dashboard at" not in out, out + assert _fingerprint(planted) == before + assert _port_is_free(port), "the refused start kept its socket" + + +@pytest.mark.parametrize("inside", ["console", "postgres/run", "."]) +def test_a_served_root_that_is_a_link_into_the_state_directory_is_refused( + tmp_path, inside) -> None: + """Batch N's gap (e): a served root named through a SYMBOLIC LINK that + leads to the state directory, or into it, is judged where it leads. It + is refused by name, and nothing is written.""" + from opendox import console_access + + state = _state(tmp_path) + target = state if inside == "." else state / inside + target.mkdir(parents=True, exist_ok=True, mode=0o700) + alias = tmp_path / "served-alias" + alias.symlink_to(target) + before = _tree(state) + with pytest.raises(console_access.ConsoleAccessRefused, + match="OPENDOX_STATE_DIR") as refused: + console_access.write_private_copy( + state, page_url="http://127.0.0.1:8080/index.html", port=8080, + token=_token(), served_roots=(alias,)) + assert str(target.resolve()) in str(refused.value), str(refused.value) + assert _tree(state) == before, "something was written" + + +def test_the_servers_own_entry_point_refuses_a_bundle_linked_into_the_state_directory( + tmp_path, monkeypatch, capsys, standalone_profile) -> None: + """The same, through the entry point: `--web-dir` names a link that leads + to `/console`. The start is refused by name, and the static + handler never serves the directory the copies live in.""" + from opendox import console_access, serve + + _clean_git(monkeypatch) + repo = _repository(tmp_path) + snapshot = tmp_path / "snapshot.json" + snapshot.write_text(json.dumps({"generation": {}}), encoding="utf-8") + state = _state(tmp_path) + console = state / console_access.CONSOLE_DIRNAME + console.mkdir(mode=0o700) + alias = tmp_path / "web-alias" + alias.symlink_to(console) + monkeypatch.setenv("OPENDOX_STATE_DIR", str(state)) + monkeypatch.setattr(serve, "real_notebook_adapter", lambda *a, **k: None) + + def served(self, *args, **kwargs): + raise AssertionError("the server served") + + monkeypatch.setattr(socketserver.BaseServer, "serve_forever", served) + assert serve.main(["--web-dir", str(alias), "--snapshot", str(snapshot), + "--checkout-root", str(repo), "--port", "0"]) == 1 + err = capsys.readouterr().err + assert "serve refused:" in err and str(console.resolve()) in err, err + assert list(console.iterdir()) == [] + + +@pytest.mark.parametrize("mode", [0o755, 0o750, 0o711]) +def test_a_console_directory_that_is_not_0700_is_refused(tmp_path, mode) -> None: + """12.4a: the copy is mode 0600 "in a directory of mode 0700". A + `console/` directory loosened after it was made, even where no one else + can write it, is refused by name by the reader and by the writer, and the + copy in it is left as it is.""" + from opendox import console_access + + state = _state(tmp_path) + copy = _write(state) + copy.path.parent.chmod(mode) + try: + expected = f"has mode {mode:o}, not 700" + with pytest.raises(console_access.ConsoleAccessRefused, match=expected): + console_access.read_private_copy(copy.path) + with pytest.raises(console_access.ConsoleAccessRefused, match=expected): + _write(state) + assert copy.path.exists() + finally: + copy.path.parent.chmod(0o700) + + +@pytest.mark.skipif(os.geteuid() == 0, reason="root can write any directory") +def test_a_copy_that_cannot_be_written_refuses_the_start_by_name( + tmp_path, monkeypatch, capsys, standalone_profile) -> None: + """The lifecycle self-pass: a state directory its parent will not let + this user make (an operating-system refusal, not a rule of this module) + used to escape as a raw `PermissionError`, a traceback and no refusal. + It refuses by name, through the writer and through the entry point.""" + from opendox import cli, console_access + + locked = tmp_path / "locked" + locked.mkdir(mode=0o700) + locked.chmod(0o500) + try: + state = locked / "state" + with pytest.raises(console_access.ConsoleAccessRefused, + match="cannot be written") as refused: + _write(state) + assert str(console_access.private_copy_path(state, 8080)) in str(refused.value) + _clean_git(monkeypatch) + repo = _repository(tmp_path) + monkeypatch.setenv("OPENDOX_STATE_DIR", str(state)) + opened: list[str] = [] + assert cli._generate_and_open(_generate_and_open_args(tmp_path, repo), + opener=opened.append) == 1 + err = capsys.readouterr().err + assert opened == [] + assert "generate-and-open refused:" in err and "cannot be written" in err, err + assert not state.exists() + finally: + locked.chmod(0o700) + + +def test_a_copy_that_fails_its_own_read_back_is_not_left_behind( + tmp_path, monkeypatch) -> None: + """The lifecycle self-pass: the writer reads back what it wrote, as any + reader would, and refuses on a failure. The copy it wrote then goes with + the refusal, rather than outliving a start that never served.""" + from opendox import console_access + + state = _state(tmp_path) + + def refusing(path): + raise console_access.ConsoleAccessRefused("staged: the read-back refused") + + monkeypatch.setattr(console_access, "read_private_copy", refusing) + with pytest.raises(console_access.ConsoleAccessRefused, match="staged"): + _write(state) + monkeypatch.undo() + assert list((state / console_access.CONSOLE_DIRNAME).iterdir()) == [] + + +def test_another_serves_copy_survives_a_removal_where_hard_links_fail( + tmp_path, monkeypatch) -> None: + """The lifecycle self-pass, on removal: the name is taken and judged, and + another serve's copy is put back. Putting it back by a hard link fails on + a filesystem without them (EPERM), and that used to DELETE the other + serve's copy. It is put back by renaming it, where the name is free.""" + from opendox import console_access + + state = _state(tmp_path) + first = _write(state) + second_token = _token() + real_rename = os.rename + raced: list = [] + + def racing(src, dst, *args, **kwargs): + if src == first.path.name and not raced: + raced.append(_write(state, token=second_token)) # the replacement + return real_rename(src, dst, *args, **kwargs) + + def no_hard_links(*args, **kwargs): + raise PermissionError(errno.EPERM, "Operation not permitted") + + monkeypatch.setattr(console_access.os, "rename", racing) + monkeypatch.setattr(console_access.os, "link", no_hard_links) + console_access.remove_private_copy(first) + monkeypatch.undo() + assert raced, "the race was never staged" + assert console_access.read_private_copy(first.path)["console_token"] == second_token + assert sorted(p.name for p in first.path.parent.iterdir()) == [first.path.name] + + +def test_terminate_as_interrupt_reads_a_hangup_as_ctrl_c_unless_it_is_ignored( + ) -> None: + """The lifecycle self-pass, on stopping: closing the terminal sends + SIGHUP, whose default action ends the process without removing the copy. + While a copy exists, SIGHUP is read as Ctrl-C, as SIGTERM is. A SIGHUP + the process was started ignoring (`nohup`) stays ignored.""" + from opendox import console_access + + previous = signal.signal(signal.SIGHUP, signal.SIG_DFL) + try: + with pytest.raises(KeyboardInterrupt): + with console_access.terminate_as_interrupt(True): + # never deliver a hangup that would END this test process + assert signal.getsignal(signal.SIGHUP) not in ( + signal.SIG_DFL, signal.SIG_IGN), "no hangup handler" + os.kill(os.getpid(), signal.SIGHUP) + signal.pthread_sigmask(signal.SIG_BLOCK, []) # deliver now + assert signal.getsignal(signal.SIGHUP) == signal.SIG_DFL + signal.signal(signal.SIGHUP, signal.SIG_IGN) # as `nohup` starts it + with console_access.terminate_as_interrupt(True): + assert signal.getsignal(signal.SIGHUP) == signal.SIG_IGN + assert signal.getsignal(signal.SIGHUP) == signal.SIG_IGN + finally: + signal.signal(signal.SIGHUP, previous) + + +@pytest.mark.parametrize("entry", ["serve", "generate-and-open --local", + "generate-and-open, hosted"]) +def test_a_hangup_removes_the_copy(tmp_path, monkeypatch, entry) -> None: + """SIGHUP, as a closed terminal sends it, stops every standalone entry + point through the code that removes its copy, and each exits 0. + + The child is started with SIGHUP at its DEFAULT action, whatever this + runner inherited: an ignored signal stays ignored across `exec`, so a + suite run under `nohup` would hand every child an ignored SIGHUP, which + the entry points rightly keep ignoring.""" + from opendox import console_access + + _clean_git(monkeypatch) + repo = _repository(tmp_path) + inherited = signal.signal(signal.SIGHUP, signal.SIG_DFL) + try: + if entry == "serve": + snapshot = tmp_path / "snapshot.json" + snapshot.write_text(json.dumps({"generation": {}}), encoding="utf-8") + child = Child(tmp_path, "opendox.serve", "--snapshot", str(snapshot), + "--checkout-root", str(repo), "--port", "0") + url = _SERVE_URL + else: + local = ["--local"] if entry.endswith("--local") else [] + child = Child(tmp_path, "opendox.cli", "generate-and-open", *local, + "--repo-root", str(repo), "--repository", "fixture", + "--no-open", "--port", "0", + "--run-dir", str(tmp_path / "run"), + extra_env=None if local else _HOSTED) + url = _URL + finally: + signal.signal(signal.SIGHUP, inherited) + try: + match = child.wait_for_line(url) + copy_path = console_access.private_copy_path(child.state_dir, + int(match.group(3))) + assert copy_path.exists(), child.stdout_text() + child.stderr_text() + assert _stop(child, signal.SIGHUP) == 0, child.stderr_text() + assert not copy_path.exists(), "a hangup left the token's copy behind" + assert "Traceback" not in child.stderr_text(), child.stderr_text() + finally: + child.kill() + + +@pytest.mark.parametrize("local", [False, True], ids=["hosted", "--local"]) +def test_a_kill_while_the_browser_opens_removes_the_copy( + tmp_path, monkeypatch, local) -> None: + """The lifecycle self-pass: the copy exists from the moment it is + written, and the browser opener can take seconds. A SIGTERM then, before + the serve loop, used to take SIGTERM's default action on a hosted + standalone plane, ending the process with the copy left behind. The + window from the write to the stop is covered whole. Here the browser + (`BROWSER`, a script) kills its own parent while it opens.""" + import standalone_child + from opendox import console_access + + _clean_git(monkeypatch) + repo = _repository(tmp_path) + browser = tmp_path / "kill-the-opener" + browser.write_text('#!/bin/sh\nkill -TERM "$PPID"\n', encoding="utf-8") + browser.chmod(0o700) + env = {"BROWSER": str(browser), **({} if local else _HOSTED)} + child = Child(tmp_path, "opendox.cli", "generate-and-open", + *(["--local"] if local else []), + "--repo-root", str(repo), "--repository", "fixture", + "--port", "0", "--run-dir", str(tmp_path / "run"), extra_env=env) + try: + match = child.wait_for_line(_URL) + copy_path = console_access.private_copy_path(child.state_dir, + int(match.group(3))) + code = child.process.wait(timeout=standalone_child.STOP_DEADLINE_SECONDS) + assert code == 0, (code, child.stderr_text()) + assert not copy_path.exists(), "a kill while the browser opened left the copy" + assert "Traceback" not in child.stderr_text(), child.stderr_text() + finally: + child.kill() + + +def test_a_console_directory_with_a_setgid_bit_is_still_0700(tmp_path) -> None: + """Only the permission bits are judged: a directory made under a setgid + parent inherits the setgid bit, which grants no one access, so a + `console/` of mode 2700 is accepted by the writer and the reader alike.""" + from opendox import console_access + + state = _state(tmp_path) + copy = _write(state) + copy.path.parent.chmod(0o2700) + if not os.lstat(copy.path.parent).st_mode & stat.S_ISGID: + pytest.skip("this filesystem keeps no setgid bit on a directory") + assert console_access.read_private_copy(copy.path)["port"] == 8080 + assert _write(state).path == copy.path + + +def test_a_write_that_fails_part_way_leaves_no_partial_copy( + tmp_path, monkeypatch) -> None: + """The lifecycle self-pass, on writing: the copy is written under a + temporary name and renamed into place. A failure part way (here the + disk is full at the `fsync`) refuses by name and removes the temporary + file, so nothing partial is left beside the name.""" + from opendox import console_access + + state = _state(tmp_path) + + def full(handle): + raise OSError(errno.ENOSPC, "No space left on device") + + monkeypatch.setattr(console_access.os, "fsync", full) + with pytest.raises(console_access.ConsoleAccessRefused, + match="cannot be written.*No space left on device"): + _write(state) + monkeypatch.undo() + assert list((state / console_access.CONSOLE_DIRNAME).iterdir()) == [] From 14285fbbf26e9f2de2ead07ff0d4e3194c3fbefa Mon Sep 17 00:00:00 2001 From: Brett Heap <1513478+brettheap@users.noreply.github.com> Date: Sat, 3 Oct 2026 22:40:40 +0000 Subject: [PATCH 74/88] T104: the state directory is walked once, every link and directory on the way judged, and the write anchored to that walk (Copilot review) WIP AT BREAKPOINT 6, NOT YET PUSHED: the targeted cases pass, and the full suite and the mutants have not run on this commit. Copilot's review at d4b99436 (5402659135), r4174785933, with a probe confirmed here first. For OPENDOX_STATE_DIR=alias/state, alias -> shared/hop and hop -> private, the tree rules judged the configured path's components and the directories above the RESOLVED path. shared, reached only through a link's target, was neither: at mode 0777 and not sticky, it went unjudged. And the write walked the configured path AGAIN after the served-root check, so a hop re-pointed in between landed the token's copy in a served root (served/state/console/8080.html). _walked resolves the state directory once, component by component as the kernel does. Every directory passed through is judged by the rule for the directories above the state directory, and every link followed by the rule for a link. The served-root boundary, the tree rules, the write and the read all work on the walked path. Cases, both failing first at 182cac76: - test_a_directory_passed_through_by_an_intermediate_link_is_judged: DID NOT RAISE, for the write and for the read. - test_a_link_swapped_after_the_checks_never_redirects_the_write: the copy landed in the served root. test_no_route_serves_the_token_to_another_local_user's simulated other uid is now refused at the first of this user's directories that the walk passes through, so its match takes that wording as well. Arc: neutral-product-standalone-operability Lane: openxfactory-4 (openXfactory-4-openDox_extraction) Co-Authored-By: Claude Opus 5.5 (1M context) --- src/opendox/console_access.py | 93 ++++++++++++++++++++++++++-- tests/test_console_token_delivery.py | 83 ++++++++++++++++++++++++- 2 files changed, 168 insertions(+), 8 deletions(-) diff --git a/src/opendox/console_access.py b/src/opendox/console_access.py index 707095a6..d2a1b511 100644 --- a/src/opendox/console_access.py +++ b/src/opendox/console_access.py @@ -242,6 +242,79 @@ def _unsafe(path: Path, reason: str) -> ConsoleAccessRefused: f"({runtime_config.PREFIX}STATE_DIR)") +#: How many symbolic links one walk of the state directory may follow, the +#: kernel's own `MAXSYMLINKS` on Linux. +_MAX_LINKS = 40 + + +def _walked(configured: Path | str) -> Path: + """The state directory, resolved ONCE, as the kernel walks it, with every + directory it passes through and every symbolic link it follows judged on + the way (Copilot at openDox-code#84, r4174785933). + + The tree rules below judge the configured path's own components and the + directories above the RESOLVED path. A directory reached only through a + link's target (`alias -> shared/hop`, `hop -> private`) is neither, so a + `shared` that others could write went unjudged, and another user could + re-point `hop` between the checks and the write, which walked the + configured path again. So every directory passed through is judged by + the rule for the directories above the state directory (this user's or + root's, and sticky if others can write it), every link followed by the + rule for a link (this user's or root's), and the caller works on the + path returned, never on the configured one again. Nothing on that path + can then be replaced by another user. A missing tail is appended as + named, to be made by descriptor under the deepest directory that + exists.""" + uid = os.getuid() + configured = Path(configured) + if not configured.is_absolute() or ".." in configured.parts: + raise ConsoleAccessRefused( + f"the state directory {str(configured)!r} is not an absolute path " + "without `..`, so the copy's path is not the one the kernel walks") + pending = list(reversed(configured.parts[1:])) + current = Path(configured.anchor) + links = 0 + while pending: + name = pending.pop() + if name in ("", "."): + continue + if name == "..": # only ever from a link's target + current = current.parent + continue + candidate = current / name + try: + info = os.lstat(candidate) + except FileNotFoundError: + rest = [name, *reversed(pending)] + if ".." in rest: + raise ConsoleAccessRefused( + f"{candidate} does not exist, and the state directory " + f"{configured} would climb out of it with `..`") from None + return current.joinpath(*rest) + if stat.S_ISLNK(info.st_mode): + if info.st_uid not in (uid, 0): + raise _unsafe(candidate, f"is a symbolic link owned by uid " + f"{info.st_uid}, neither this user nor root, who " + "could point it elsewhere") + links += 1 + if links > _MAX_LINKS: + raise ConsoleAccessRefused( + f"the state directory {configured} passes through more " + f"than {_MAX_LINKS} symbolic links") + target = Path(os.readlink(candidate)) + if target.is_absolute(): + current = Path(target.anchor) + pending.extend(reversed(target.parts[1:])) + else: + pending.extend(reversed(target.parts)) + continue + reason = _unsafe_because(info, uid=uid, own=False) + if reason is not None: + raise _unsafe(candidate, reason) + current = candidate + return current + + def _refuse_an_unsafe_tree(state_dir: Path, *, existing_only: bool) -> None: """The copy's whole path is this user's to change, or it is refused. @@ -454,8 +527,14 @@ def write_private_copy(state_dir: Path | str, *, page_url: str, port: int, `ConsoleAccessRefused` naming the copy, so the entry point refuses its start by name instead of ending in a traceback. And a copy whose read-back fails is removed with the refusal, so a start that never served - leaves no copy behind.""" - state = Path(state_dir) + leaves no copy behind. + + THE STATE DIRECTORY IS WALKED ONCE (`_walked`, Copilot at + openDox-code#84, r4174785933), and the served-root boundary, the tree's + rules and the write all work on the path that walk returned. A link on + the configured path that is re-pointed after the checks cannot redirect + the write. The copy's `path` is that walked path.""" + state = _walked(state_dir) _refuse_a_served_state_dir(state, tuple(served_roots), port=port) record = { "schema_version": RECORD_SCHEMA_VERSION, @@ -553,11 +632,13 @@ def read_private_copy(path: Path | str) -> dict: without following a link and without blocking: a regular file, this user's, exactly 0600, with one link. A planted, linked or loosened copy is refused, and so is a FIFO, without waiting on it.""" - target = Path(path) - state = target.parent.parent - if target.parent.name != CONSOLE_DIRNAME: - raise ConsoleAccessRefused(f"{target} is not in a `{CONSOLE_DIRNAME}/` " + given = Path(path) + if given.parent.name != CONSOLE_DIRNAME: + raise ConsoleAccessRefused(f"{given} is not in a `{CONSOLE_DIRNAME}/` " "directory of a state directory") + # Walked once, as the writer walks it, and read from where the walk led. + state = _walked(given.parent.parent) + target = state / CONSOLE_DIRNAME / given.name try: _refuse_an_unsafe_tree(state, existing_only=False) except FileNotFoundError: diff --git a/tests/test_console_token_delivery.py b/tests/test_console_token_delivery.py index 11c5e92a..c6e2a7b5 100644 --- a/tests/test_console_token_delivery.py +++ b/tests/test_console_token_delivery.py @@ -302,10 +302,12 @@ def test_no_route_serves_the_token_to_another_local_user( assert stat.S_IMODE(info.st_mode) == mode, (path, oct(info.st_mode)) assert info.st_uid == os.getuid(), path assert stat.S_ISREG(os.lstat(copy.path).st_mode) - # A copy ANOTHER user owns, as this module's reader sees one: refused. + # A copy ANOTHER user owns, as this module's reader sees one: refused, at + # the first directory of this user's that its walk passes through. real = os.getuid() monkeypatch.setattr(console_access.os, "getuid", lambda: real + 1) - with pytest.raises(console_access.ConsoleAccessRefused, match="not by this user"): + with pytest.raises(console_access.ConsoleAccessRefused, + match="not by this user|neither this user nor root"): console_access.read_private_copy(copy.path) @@ -1666,3 +1668,80 @@ def full(handle): _write(state) monkeypatch.undo() assert list((state / console_access.CONSOLE_DIRNAME).iterdir()) == [] + + +# --------------------------------------------------------------------------- +# 12 — the state directory is walked ONCE, every link and directory on the +# way judged, and the write is anchored to that walk (Copilot at +# openDox-code#84, r4174785933) +# --------------------------------------------------------------------------- + +def _hop_layout(tmp_path: Path, shared_mode: int) -> dict: + """Copilot's layout: `OPENDOX_STATE_DIR=alias/state`, `alias -> + shared/hop`, `hop -> private`. Only `alias` is on the configured path; + `shared` is passed through by way of a link's target.""" + shared = tmp_path / "shared" + shared.mkdir() + private = tmp_path / "private" + private.mkdir(mode=0o700) + served = tmp_path / "served" + served.mkdir(mode=0o700) + (shared / "hop").symlink_to(private) + (tmp_path / "alias").symlink_to(shared / "hop") + shared.chmod(shared_mode) + return {"shared": shared, "private": private, "served": served, + "hop": shared / "hop", "state": tmp_path / "alias" / "state"} + + +def test_a_directory_passed_through_by_an_intermediate_link_is_judged( + tmp_path) -> None: + """`shared` is neither on the configured path nor above the resolved + one, and it was never judged: mode 0777 and not sticky, so another user + could replace `hop`. It is judged now, as every directory the walk passes + through is, and the write and the read are both refused by name.""" + from opendox import console_access + + layout = _hop_layout(tmp_path, 0o777) + try: + with pytest.raises(console_access.ConsoleAccessRefused, + match="writable by every user and is not sticky") as refused: + _write(layout["state"]) + assert str(layout["shared"]) in str(refused.value), str(refused.value) + assert _tree(layout["private"]) == [], "something was written" + # a copy that is there already is refused when read through that way + written = _write(layout["private"] / "state") + through = layout["state"] / console_access.CONSOLE_DIRNAME / written.path.name + with pytest.raises(console_access.ConsoleAccessRefused, + match="writable by every user and is not sticky"): + console_access.read_private_copy(through) + finally: + layout["shared"].chmod(0o755) + + +def test_a_link_swapped_after_the_checks_never_redirects_the_write( + tmp_path, monkeypatch) -> None: + """The race: `hop` is pointed at a served root after the served-root + check, and the write used to walk the configured path AGAIN, so the + token's copy landed in the served root, where `/source` serves it. The + path is walked once, and the write is anchored to that walk: the copy + is where the checks saw the state directory, and the served root gains + nothing.""" + from opendox import console_access + + layout = _hop_layout(tmp_path, 0o755) + real = console_access._refuse_a_served_state_dir + + def then_swap(*args, **kwargs): + real(*args, **kwargs) + layout["hop"].unlink() + layout["hop"].symlink_to(layout["served"]) + + monkeypatch.setattr(console_access, "_refuse_a_served_state_dir", then_swap) + copy = console_access.write_private_copy( + layout["state"], page_url="http://127.0.0.1:8080/index.html", port=8080, + token=_token(), served_roots=(layout["served"],)) + monkeypatch.undo() + assert _tree(layout["served"]) == [], "the swapped link redirected the write" + expected = (layout["private"] / "state").resolve() + assert copy.path == console_access.private_copy_path(expected, 8080) + assert console_access.read_private_copy(copy.path)["port"] == 8080 From 9f3288921631ffb4b0b761a0a46130791a2a15a1 Mon Sep 17 00:00:00 2001 From: Brett Heap <1513478+brettheap@users.noreply.github.com> Date: Sun, 4 Oct 2026 14:02:06 +0000 Subject: [PATCH 75/88] T104 fix round 5: the snapshot files are served roots, publication and removal take one lock, and a stop is held through publication (Copilot review) Copilot's review at 182cac76 (5403191194, "Changes recommended"), three threads. Each case failed first at ddb26c34: - r4175213798, the snapshot files. /snapshot.json reads its file directly, not through the static handler, so --snapshot named at an earlier copy (/console/.html) would be replaced by the new copy and served to anyone. The served roots now hold the configured snapshot, each registered entry's snapshot and, on a loopback plane, the session snapshots' container (branch_session.snapshots_root). So the reverse boundary refuses the start by name, through a link as well. Cases: - test_a_snapshot_inside_the_state_directory_refuses_the_start (before the fix: "the server served"); - test_the_plane_reports_its_snapshot_files_as_served_roots, which includes a registered entry whose snapshot is another file. - r4175213842, the rename-back race. Where a hard link fails, another serve's copy is renamed back where the name is free, and a newer copy published between that check and the rename was overwritten. Every writer and remover of console/ now takes the directory's exclusive flock (_lock), so publication and removal are serialized. Case: test_a_copy_published_during_a_rename_back_is_never_overwritten. A third copy is published from another thread at exactly that moment, waits, then stands. Before the fix, "an older copy overwrote the newest". The two earlier removal-race cases now publish from another thread too, because a publication from inside a removal would wait on the lock forever. - r4175213864, a stop during publication. SIGTERM was read as Ctrl-C only after publish() returned, so a SIGTERM during the copy's read-back took its default action and left the copy. Both entry points now install the handler BEFORE publication on a plane that writes a copy (console_access.needs_copy), and keep it through removal. A stop that arrives while the copy is written or removed is HELD (console_access.deferred_termination): publication raises it once the copy is in hand, and removal, already a stop, lets it go. Cases: - test_a_stop_during_publication_removes_the_copy, for serve and for generate-and-open (before the fix: "no handler during publication"); - test_a_stop_during_removal_lets_the_removal_finish (before the fix: "no handler during removal"); - test_deferred_termination_holds_a_stop_until_the_block_ends. Arc: neutral-product-standalone-operability Lane: openxfactory-4 (openXfactory-4-openDox_extraction) Co-Authored-By: Claude Opus 5.5 (1M context) --- src/opendox/cli.py | 45 ++--- src/opendox/console_access.py | 81 ++++++++- src/opendox/serve.py | 50 +++-- tests/test_console_token_delivery.py | 263 +++++++++++++++++++++++++-- 4 files changed, 379 insertions(+), 60 deletions(-) diff --git a/src/opendox/cli.py b/src/opendox/cli.py index 23169107..9a40e2b3 100644 --- a/src/opendox/cli.py +++ b/src/opendox/cli.py @@ -908,21 +908,19 @@ def _generate_and_serve(args: argparse.Namespace, run_dir: Path, *, # the page. `None` on a host's plane and where no token was minted, and # then nothing changes. A copy that cannot be written safely refuses the # run before it serves. - try: - console = console_access.publish(httpd, page_url=url) - except console_access.ConsoleAccessRefused as exc: - httpd.server_close() - print(f"generate-and-open refused: {exc}", file=sys.stderr) - return 1 - try: - # A plain `kill`, or a closed terminal, stops a standalone console the - # way Ctrl-C does, so the copy below is removed - # (`terminate_as_interrupt`); a plane that wrote no copy keeps the - # signals' default actions. It covers the WHOLE window from the write - # to the stop, the browser opener included, which can take seconds - # (T104's self-pass), and a stop asked for there is a clean stop too. - with console_access.terminate_as_interrupt(console is not None): + # + # A plain `kill`, or a closed terminal, stops a standalone console the way + # Ctrl-C does (`terminate_as_interrupt`), from BEFORE the copy is written + # to after it is removed (Copilot at openDox-code#84, r4175213864), and a + # stop that arrives while the copy is being written or removed is held + # until that is done (`deferred_termination`), so no copy is ever left + # half handled. A plane that writes no copy keeps the signals' defaults. + console = None + with console_access.terminate_as_interrupt(console_access.needs_copy(httpd)): + try: try: + with console_access.deferred_termination(): + console = console_access.publish(httpd, page_url=url) print(f" serving {url}") print(f" snapshot {serve_mod.server_url(httpd, '/snapshot.json')}") if console is not None: @@ -952,15 +950,20 @@ def _generate_and_serve(args: argparse.Namespace, run_dir: Path, *, print(" serving until interrupted (Ctrl-C to stop)", flush=True) httpd.serve_forever() + except console_access.ConsoleAccessRefused as exc: + print(f"generate-and-open refused: {exc}", file=sys.stderr) + return 1 except KeyboardInterrupt: pass - return 0 - finally: - # The copy goes with the server: its token is this serve's, and dies - # with it. It goes FIRST, while this process still holds the port, so - # no later serve can bind it and write its own copy in between. - console_access.remove_private_copy(console) - httpd.server_close() + return 0 + finally: + # The copy goes with the server: its token is this serve's, and + # dies with it. It goes FIRST, while this process still holds the + # port, so no later serve can bind it and write its own copy in + # between. A stop that arrives meanwhile lets it finish. + with console_access.deferred_termination(raise_pending=False): + console_access.remove_private_copy(console) + httpd.server_close() # ---- gate console (US9): human-only executable gate actions ---------------- diff --git a/src/opendox/console_access.py b/src/opendox/console_access.py index d2a1b511..ba310076 100644 --- a/src/opendox/console_access.py +++ b/src/opendox/console_access.py @@ -65,7 +65,10 @@ * every refusal names its path, an operating-system one included, so an entry point refuses its start by name; and the copy is removed when the server stops, by Ctrl-C, SIGTERM or SIGHUP, or when its start is refused - after it was written. + after it was written. A stop is read as Ctrl-C from before the copy is + written to after it is removed, and held while a copy is being written or + removed (`deferred_termination`), and every writer and remover of + `console/` takes the directory's lock (`_lock`). #1144 12.4a, as T007 batch N amends it (openxFactory#1222), is the normative text this module realizes. @@ -90,13 +93,18 @@ from opendox.runtime import config as runtime_config +try: # POSIX; the copy's rules are POSIX's + import fcntl +except ImportError: # pragma: no cover + fcntl = None + __all__ = [ "CONSOLE_DIRNAME", "ConsoleAccessRefused", "ConsoleTerminated", "DELIVERY_CAPABILITIES", "DELIVERY_OPENED_URL", "FRAGMENT_KEY", "PrivateCopy", "RECORD_ELEMENT_ID", - "RECORD_KIND", "delivery_for", "opened_url", "private_copy_path", "publish", - "read_private_copy", "remove_private_copy", "terminate_as_interrupt", - "write_private_copy", + "RECORD_KIND", "deferred_termination", "delivery_for", "needs_copy", + "opened_url", "private_copy_path", "publish", "read_private_copy", + "remove_private_copy", "terminate_as_interrupt", "write_private_copy", ] #: The token rides on `/capabilities`, as a host's plane has always read it. @@ -235,6 +243,23 @@ def _console_dir_unsafe_because(info: os.stat_result, *, uid: int) -> str | None return reason +def _lock(directory: int) -> None: + """Hold the console directory's lock on its descriptor until it closes. + + PUBLICATION AND REMOVAL ARE SERIALIZED (Copilot at openDox-code#84, + r4175213842). The copy's name is per PORT, and two serves can share one + (127.0.0.1 and ::1), so one serve's removal can find another's copy at + the name and must put it back. Where that takes a rename, a check that + the name is free and the rename are two steps, and a third copy published + between them would be overwritten by an older one. Every writer and + remover of `console/` takes this exclusive lock, an advisory `flock` the + kernel drops when the descriptor closes or the process dies. Where the + filesystem keeps no such locks, nothing is held, as before.""" + if fcntl is not None: + with contextlib.suppress(OSError): + fcntl.flock(directory, fcntl.LOCK_EX) + + def _unsafe(path: Path, reason: str) -> ConsoleAccessRefused: return ConsoleAccessRefused( f"{path} {reason}, so another user could replace or read the console " @@ -580,6 +605,7 @@ def _write_the_copy(state: Path, target: Path, reason = _console_dir_unsafe_because(os.fstat(directory), uid=uid) if reason is not None: raise _unsafe(target.parent, reason) + _lock(directory) # until the copy is in place (`_lock`) try: present = os.stat(target.name, dir_fd=directory, follow_symlinks=False) @@ -724,7 +750,8 @@ def remove_private_copy(copy: PrivateCopy | None) -> None: A filesystem without hard links refuses the link (EPERM), and so does a directory, and the other serve's copy used to be deleted with the temporary name. It is renamed back instead, where the name is still - free.""" + free, and the console directory's lock (`_lock`) keeps any newer copy + from being published between that check and the rename.""" if copy is None: return try: @@ -735,6 +762,7 @@ def remove_private_copy(copy: PrivateCopy | None) -> None: name = copy.path.name taken = f".{name}.removing-{os.getpid()}-{os.urandom(6).hex()}" try: + _lock(directory) # no copy is published meanwhile (`_lock`) try: os.rename(name, taken, src_dir_fd=directory, dst_dir_fd=directory) except OSError: @@ -772,10 +800,40 @@ class ConsoleTerminated(KeyboardInterrupt): stop on.""" +#: Whether a stop is being HELD (`deferred_termination`), and the one that +#: arrived meanwhile. Python runs signal handlers in the main thread only, as +#: the entry points publish and remove there, so plain module state serves. +_held = {"depth": 0, "pending": None} + + def _terminate_as_interrupt(signum, frame): + if _held["depth"]: + _held["pending"] = signum + return raise ConsoleTerminated +@contextlib.contextmanager +def deferred_termination(*, raise_pending: bool = True): + """Hold a SIGTERM or SIGHUP that arrives inside the block, rather than + raising it in the middle of publishing or removing a copy (Copilot at + openDox-code#84, r4175213864): a copy half published, or half removed, + is one nothing cleans up. On the way out, a held stop is raised as + `ConsoleTerminated` once the block is done, when the copy is in the + caller's hands, or dropped with `raise_pending=False`, for a removal, + which is a stop already. Only the handler `terminate_as_interrupt` + installs holds anything; Ctrl-C keeps Python's own.""" + _held["depth"] += 1 + try: + yield + finally: + _held["depth"] -= 1 + if not _held["depth"]: + pending, _held["pending"] = _held["pending"], None + if pending is not None and raise_pending: + raise ConsoleTerminated + + @contextlib.contextmanager def terminate_as_interrupt(enabled: bool): """While a standalone console's private copy exists, read SIGTERM as the @@ -814,6 +872,14 @@ def terminate_as_interrupt(enabled: bool): signal.signal(signum, handler) +def needs_copy(httpd: Any) -> bool: + """Whether the plane `httpd` delivers its console token through a + private copy: a standalone plane that minted one. The entry points ask it + BEFORE `publish`, to read a stop as Ctrl-C from before the copy exists.""" + return bool(getattr(httpd, "console_token", None)) and getattr( + httpd, "console_token_delivery", None) == DELIVERY_OPENED_URL + + def publish(httpd: Any, *, page_url: str, env: Mapping[str, str] | None = None) -> PrivateCopy | None: """What an ENTRY POINT does after `serve.build_server`: on a standalone @@ -823,10 +889,9 @@ def publish(httpd: Any, *, page_url: str, A state directory that cannot be named, or a tree that is not this user's alone, refuses (`ConsoleAccessRefused`), and the entry point refuses with it: a console nobody can open is not served as if it could be.""" - token = getattr(httpd, "console_token", None) - if not token or getattr(httpd, "console_token_delivery", - None) != DELIVERY_OPENED_URL: + if not needs_copy(httpd): return None + token = httpd.console_token try: state = runtime_config.state_dir(env) except runtime_config.ConfigurationError as exc: diff --git a/src/opendox/serve.py b/src/opendox/serve.py index e9221050..b8f2249e 100644 --- a/src/opendox/serve.py +++ b/src/opendox/serve.py @@ -2488,16 +2488,24 @@ def build_server( # each entry the registry holds now (the bootstrapped session worktrees # among them), and, on a loopback plane, the sessions container every # later session worktree is made in (`branch_session.sessions_root`). - served = [checkout_root, web_dir, + # + # AND THE SNAPSHOT FILES `/snapshot.json` READS DIRECTLY (Copilot at + # openDox-code#84, r4175213798), not through the static handler: the + # configured snapshot, each registered entry's, and, on a loopback plane, + # the container every session's snapshot is written in. A snapshot named + # at an earlier copy would otherwise be replaced by the new one and served. + served = [checkout_root, web_dir, snapshot_path, *(Path(path).resolve() for path in (source_roots or {}).values())] if loopback: from opendox import branch_session as session_mod served.append(session_mod.sessions_root(checkout_root)) + served.append(session_mod.snapshots_root(checkout_root)) entries = getattr(source.registry, "entries", None) for entry in (entries() if callable(entries) else ()): - root = getattr(entry, "source_root", None) - if root: - served.append(Path(root).resolve()) + for root in (getattr(entry, "source_root", None), + getattr(entry, "snapshot_path", None)): + if root: + served.append(Path(root).resolve()) httpd.served_roots = tuple(dict.fromkeys(served)) return httpd @@ -2611,18 +2619,18 @@ def serve( # as `cli.cmd_generate_and_open` writes it: its PATH is printed, never the # token, and it goes when the server does. A copy that cannot be written # safely refuses the start (`console_access.ConsoleAccessRefused`). - try: - console = console_access.publish(httpd, page_url=page) - except BaseException: - httpd.server_close() - raise - try: - # A plain `kill`, or a closed terminal, stops a standalone console the - # way Ctrl-C does, so the copy is removed; a plane that wrote none - # keeps the signals' defaults. It covers the whole window from the - # write to the stop (T104's self-pass). - with console_access.terminate_as_interrupt(console is not None): + # + # A plain `kill`, or a closed terminal, stops a standalone console the way + # Ctrl-C does, from BEFORE the copy is written to after it is removed, and + # a stop that arrives while the copy is written or removed is held until + # that is done (Copilot at openDox-code#84, r4175213864). A plane that + # writes no copy keeps the signals' defaults. + console = None + with console_access.terminate_as_interrupt(console_access.needs_copy(httpd)): + try: try: + with console_access.deferred_termination(): + console = console_access.publish(httpd, page_url=page) if console is not None: print(f"console {console.file_url} (this user's private " "copy, mode 0600: open it to open the console page)") @@ -2635,11 +2643,13 @@ def serve( httpd.serve_forever() except KeyboardInterrupt: pass - finally: - # The copy FIRST, while this process still holds the port, then the - # socket (`console_access.remove_private_copy`). - console_access.remove_private_copy(console) - httpd.server_close() + finally: + # The copy FIRST, while this process still holds the port, then + # the socket (`console_access.remove_private_copy`). A stop that + # arrives meanwhile lets both finish. + with console_access.deferred_termination(raise_pending=False): + console_access.remove_private_copy(console) + httpd.server_close() def _source_roots_from_args(values) -> dict: diff --git a/tests/test_console_token_delivery.py b/tests/test_console_token_delivery.py index c6e2a7b5..bebff65d 100644 --- a/tests/test_console_token_delivery.py +++ b/tests/test_console_token_delivery.py @@ -882,17 +882,18 @@ def test_a_replacement_written_while_the_old_copy_is_removed_survives( def racing(src, dst, *args, **kwargs): if src == first.path.name and not raced: - raced.append(_write(state, token=second_token)) # the replacement + raced.append(_publish_concurrently(state, second_token)) return real_rename(src, dst, *args, **kwargs) monkeypatch.setattr(console_access.os, "rename", racing) console_access.remove_private_copy(first) monkeypatch.undo() assert raced, "the race was never staged" + second = _settle(raced[0]) record = console_access.read_private_copy(first.path) assert record["console_token"] == second_token assert sorted(p.name for p in first.path.parent.iterdir()) == [first.path.name] - console_access.remove_private_copy(raced[0]) + console_access.remove_private_copy(second) assert not first.path.exists() @@ -1210,6 +1211,32 @@ def attempt(name, call) -> None: # bdd0f586), clause by clause, and the opener file's lifecycle # --------------------------------------------------------------------------- +def _publish_concurrently(state: Path, token: str, port: int = 8080) -> dict: + """Publish a copy from ANOTHER thread, as another serve would, and give it + time to finish or to wait on the console directory's lock.""" + import time + + outcome: dict = {} + + def publish() -> None: + try: + outcome["copy"] = _write(state, port=port, token=token) + except BaseException as exc: # noqa: BLE001 — judged by `_settle` + outcome["error"] = exc + + outcome["thread"] = threading.Thread(target=publish, daemon=True) + outcome["thread"].start() + time.sleep(0.5) + return outcome + + +def _settle(outcome: dict): + outcome["thread"].join(timeout=30) + assert not outcome["thread"].is_alive(), "the concurrent publication never finished" + assert "error" not in outcome, outcome.get("error") + return outcome["copy"] + + def _free_port() -> int: with socket.socket() as probe: probe.bind(("127.0.0.1", 0)) @@ -1510,22 +1537,14 @@ def test_another_serves_copy_survives_a_removal_where_hard_links_fail( state = _state(tmp_path) first = _write(state) second_token = _token() - real_rename = os.rename - raced: list = [] - - def racing(src, dst, *args, **kwargs): - if src == first.path.name and not raced: - raced.append(_write(state, token=second_token)) # the replacement - return real_rename(src, dst, *args, **kwargs) + _write(state, token=second_token) # another serve's, over the first def no_hard_links(*args, **kwargs): raise PermissionError(errno.EPERM, "Operation not permitted") - monkeypatch.setattr(console_access.os, "rename", racing) monkeypatch.setattr(console_access.os, "link", no_hard_links) console_access.remove_private_copy(first) monkeypatch.undo() - assert raced, "the race was never staged" assert console_access.read_private_copy(first.path)["console_token"] == second_token assert sorted(p.name for p in first.path.parent.iterdir()) == [first.path.name] @@ -1745,3 +1764,225 @@ def then_swap(*args, **kwargs): expected = (layout["private"] / "state").resolve() assert copy.path == console_access.private_copy_path(expected, 8080) assert console_access.read_private_copy(copy.path)["port"] == 8080 + + +# --------------------------------------------------------------------------- +# 13 — Copilot's review at 182cac76: the snapshot files are served roots; +# publication and removal are serialized; a stop during publication +# --------------------------------------------------------------------------- + +def test_a_snapshot_inside_the_state_directory_refuses_the_start( + tmp_path, monkeypatch, capsys, standalone_profile) -> None: + """Copilot at openDox-code#84, r4175213798. `/snapshot.json` reads its + file directly, not through the static handler, so a `--snapshot` named + at an earlier copy, `/console/.html`, would have been + replaced by the new copy and served to anyone. The snapshot file, each + registered entry's snapshot and the session snapshots' container are + served roots, so the start is refused by name, through a link to it as + well, and the earlier copy is left as it was.""" + from opendox import console_access, serve + + _clean_git(monkeypatch) + repo = _repository(tmp_path) + state = _state(tmp_path) + monkeypatch.setenv("OPENDOX_STATE_DIR", str(state)) + monkeypatch.setattr(serve, "real_notebook_adapter", lambda *a, **k: None) + + def served(self, *args, **kwargs): + raise AssertionError("the server served") + + monkeypatch.setattr(socketserver.BaseServer, "serve_forever", served) + port = _free_port() + earlier = _write(state, port=port) + alias = tmp_path / "snapshot-alias.json" + alias.symlink_to(earlier.path) + for named in (earlier.path, alias): + before = (earlier.path.read_bytes(), _fingerprint(earlier.path)) + assert serve.main(["--snapshot", str(named), "--checkout-root", str(repo), + "--port", str(port)]) == 1 + err = capsys.readouterr().err + assert "serve refused:" in err and "OPENDOX_STATE_DIR" in err, err + assert str(earlier.path.resolve()) in err, err + assert (earlier.path.read_bytes(), _fingerprint(earlier.path)) == before + assert _port_is_free(port) + + +def test_the_plane_reports_its_snapshot_files_as_served_roots( + tmp_path, monkeypatch, standalone_profile) -> None: + """The roots `/snapshot.json` serves from: the configured snapshot, each + registered entry's snapshot file, and, on a loopback plane, the container + each session's snapshot is written in.""" + from opendox import branch_session + + with _serving(tmp_path, monkeypatch) as (httpd, _base, repo): + snapshot = (tmp_path / "snapshot.json").resolve() + assert snapshot in httpd.served_roots + assert branch_session.snapshots_root(repo) in httpd.served_roots + entries = httpd.RequestHandlerClass.func.source.registry.entries() + for entry in entries: + if getattr(entry, "snapshot_path", None): + assert Path(entry.snapshot_path).resolve() in httpd.served_roots + # ...and a registered entry whose snapshot is ANOTHER file + from opendox import default_registry, serve + + other = tmp_path / "elsewhere" / "other.snapshot.json" + other.parent.mkdir() + other.write_text(json.dumps({"generation": {}}), encoding="utf-8") + source = default_registry.SnapshotSource( + baked_snapshot=tmp_path / "snapshot.json", checkout_root=repo) + source.registry.register(default_registry.entry_from_snapshot_file( + other, repository="other", ref="main")) + declared = serve.build_server(WEB, tmp_path / "snapshot.json", repo, + port=0, quiet=True, snapshot_source=source) + try: + assert other.resolve() in declared.served_roots + finally: + declared.server_close() + + +def test_a_copy_published_during_a_rename_back_is_never_overwritten( + tmp_path, monkeypatch) -> None: + """Copilot at openDox-code#84, r4175213842. Where a hard link cannot be + made, another serve's copy is put back by a rename where the name is + free, and a still newer copy published between that check and the + rename would have been overwritten by the older one. Publication and + removal are serialized on the console directory's lock, so the newer + copy, published at exactly that moment, waits and then stands.""" + from opendox import console_access + + state = _state(tmp_path) + first = _write(state) + second_token, third_token = _token(), _token() + _write(state, token=second_token) # another serve's, over the first + real_exists = console_access._name_exists + raced: list = [] + + def then_publish(name, directory): + free = real_exists(name, directory) + if not raced: + raced.append(_publish_concurrently(state, third_token)) + return free + + def no_hard_links(*args, **kwargs): + raise PermissionError(errno.EPERM, "Operation not permitted") + + monkeypatch.setattr(console_access, "_name_exists", then_publish) + monkeypatch.setattr(console_access.os, "link", no_hard_links) + console_access.remove_private_copy(first) + monkeypatch.undo() + assert raced, "the race was never staged" + _settle(raced[0]) + record = console_access.read_private_copy(first.path) + assert record["console_token"] == third_token, "an older copy overwrote the newest" + assert sorted(p.name for p in first.path.parent.iterdir()) == [first.path.name] + + +def test_deferred_termination_holds_a_stop_until_the_block_ends() -> None: + """While a copy is being published or removed, SIGTERM is held and not + raised in the middle of it: publication raises it once the copy is + in hand, and removal, already a stop, lets it go.""" + from opendox import console_access + + with console_access.terminate_as_interrupt(True): + assert signal.getsignal(signal.SIGTERM) not in (signal.SIG_DFL, signal.SIG_IGN) + with pytest.raises(KeyboardInterrupt): + with console_access.deferred_termination(): + os.kill(os.getpid(), signal.SIGTERM) + signal.pthread_sigmask(signal.SIG_BLOCK, []) # deliver now + reached = True # not raised here + assert reached + with console_access.deferred_termination(raise_pending=False): + os.kill(os.getpid(), signal.SIGTERM) + signal.pthread_sigmask(signal.SIG_BLOCK, []) + with console_access.deferred_termination(): + pass # nothing left over + + +@pytest.mark.parametrize("entry", ["serve", "generate-and-open"]) +def test_a_stop_during_publication_removes_the_copy( + tmp_path, monkeypatch, capsys, standalone_profile, entry) -> None: + """Copilot at openDox-code#84, r4175213864. SIGTERM's handling began + only after the copy was published, so a SIGTERM during publication (here + in the copy's read-back, after its rename into place) took its default + action and left the token's copy behind. It is installed BEFORE + publication on a plane that writes a copy, and held through it, so the + stop ends the start cleanly, before the startup line, with no copy left.""" + from opendox import cli, console_access, serve + + _clean_git(monkeypatch) + repo = _repository(tmp_path) + state = _state(tmp_path) + monkeypatch.setenv("OPENDOX_STATE_DIR", str(state)) + monkeypatch.setattr(serve, "real_notebook_adapter", lambda *a, **k: None) + real_read = console_access.read_private_copy + stopped: list = [] + + def read_back(path): + if not stopped: + stopped.append(path) + # never deliver a SIGTERM that would END this test process + assert signal.getsignal(signal.SIGTERM) not in ( + signal.SIG_DFL, signal.SIG_IGN), "no handler during publication" + os.kill(os.getpid(), signal.SIGTERM) + return real_read(path) + + def served(self, *args, **kwargs): + raise AssertionError("the server served after a stop") + + monkeypatch.setattr(console_access, "read_private_copy", read_back) + monkeypatch.setattr(socketserver.BaseServer, "serve_forever", served) + if entry == "serve": + snapshot = tmp_path / "snapshot.json" + snapshot.write_text(json.dumps({"generation": {}}), encoding="utf-8") + assert serve.main(["--snapshot", str(snapshot), "--checkout-root", str(repo), + "--port", "0"]) == 0 + startup = "serving ideation dashboard at" + else: + args = cli.build_parser().parse_args([ + "generate-and-open", "--repo-root", str(repo), "--repository", "fixture", + "--run-dir", str(tmp_path / "run"), "--port", "0", "--no-validate", + "--no-open"]) + assert cli._generate_and_open(args, opener=lambda url: None) == 0 + startup = " serving " + out = capsys.readouterr().out + assert stopped, "the stop was never staged" + assert startup not in out and "console " not in out, out + assert list((state / console_access.CONSOLE_DIRNAME).iterdir()) == [] + + +def test_a_stop_during_removal_lets_the_removal_finish( + tmp_path, monkeypatch, capsys, standalone_profile) -> None: + """The handler stays installed through removal, and a SIGTERM that + arrives while the copy is being removed is held, not raised in the middle + of it: the removal finishes, nothing is left, and the stop is clean.""" + from opendox import console_access, serve + + _clean_git(monkeypatch) + repo = _repository(tmp_path) + state = _state(tmp_path) + monkeypatch.setenv("OPENDOX_STATE_DIR", str(state)) + monkeypatch.setattr(serve, "real_notebook_adapter", lambda *a, **k: None) + + def stopped_at_once(self, *args, **kwargs): + raise KeyboardInterrupt # Ctrl-C, at once + + real_rename = os.rename + taken: list = [] + + def take(src, dst, *args, **kwargs): + if not taken and ".removing-" in str(dst): + taken.append(dst) + assert signal.getsignal(signal.SIGTERM) not in ( + signal.SIG_DFL, signal.SIG_IGN), "no handler during removal" + os.kill(os.getpid(), signal.SIGTERM) + return real_rename(src, dst, *args, **kwargs) + + monkeypatch.setattr(socketserver.BaseServer, "serve_forever", stopped_at_once) + monkeypatch.setattr(console_access.os, "rename", take) + snapshot = tmp_path / "snapshot.json" + snapshot.write_text(json.dumps({"generation": {}}), encoding="utf-8") + assert serve.main(["--snapshot", str(snapshot), "--checkout-root", str(repo), + "--port", "0"]) == 0 + monkeypatch.undo() + assert taken, "the stop was never staged" + assert list((state / console_access.CONSOLE_DIRNAME).iterdir()) == [] From 9dcc9bb5be2911610721ebb3faf63a4649660650 Mon Sep 17 00:00:00 2001 From: Brett Heap <1513478+brettheap@users.noreply.github.com> Date: Sun, 4 Oct 2026 14:20:04 +0000 Subject: [PATCH 76/88] T104 fix round 6: the walk's operating-system errors are refusals by name, for the writer and the reader (Copilot review) Copilot's review at ddb26c34 (5406571882), r4177975898. The walk and the served-root check ran outside the writer's conversion of OSError, so an overlong state-path component (ENAMETOOLONG) or an unsearchable parent (EACCES) escaped as a raw OSError. Both entry points then ended in a traceback instead of a named refusal. Both are inside the conversion now. The reader converts the same way (read_private_copy wraps _read_the_copy). Cases, each failing first at 9f328892 with the raw OSError or PermissionError: - test_a_state_path_the_walk_cannot_take_is_refused_by_name, for an overlong component and an unsearchable parent, through the writer and the reader; - test_a_state_path_the_walk_cannot_take_refuses_the_start, for the same two through serve and generate-and-open: exit 1 with the refusal named, nothing printed that serves, and the socket closed. The review's other thread, r4177975845 (the handler was installed only after publication, in cli), was fixed in 9f328892 (fix round 5). Arc: neutral-product-standalone-operability Lane: openxfactory-4 (openXfactory-4-openDox_extraction) Co-Authored-By: Claude Opus 5.5 (1M context) --- src/opendox/console_access.py | 43 +++++++++----- tests/test_console_token_delivery.py | 85 ++++++++++++++++++++++++++++ 2 files changed, 115 insertions(+), 13 deletions(-) diff --git a/src/opendox/console_access.py b/src/opendox/console_access.py index ba310076..4f36fa40 100644 --- a/src/opendox/console_access.py +++ b/src/opendox/console_access.py @@ -558,20 +558,25 @@ def write_private_copy(state_dir: Path | str, *, page_url: str, port: int, openDox-code#84, r4174785933), and the served-root boundary, the tree's rules and the write all work on the path that walk returned. A link on the configured path that is re-pointed after the checks cannot redirect - the write. The copy's `path` is that walked path.""" - state = _walked(state_dir) - _refuse_a_served_state_dir(state, tuple(served_roots), port=port) - record = { - "schema_version": RECORD_SCHEMA_VERSION, - "kind": RECORD_KIND, - "page_url": page_url, - "opened_url": opened_url(page_url, token), - "port": int(port), - "pid": os.getpid(), - FRAGMENT_KEY: token, - } - target = private_copy_path(state, port) + the write. The copy's `path` is that walked path. + + THE WALK IS INSIDE THE CONVERSION TOO (Copilot at openDox-code#84, + r4177975898): an overlong component (ENAMETOOLONG) or an unsearchable + parent (EACCES) on the way is a refusal by name, like any other.""" + target = private_copy_path(state_dir, port) try: + state = _walked(state_dir) + _refuse_a_served_state_dir(state, tuple(served_roots), port=port) + record = { + "schema_version": RECORD_SCHEMA_VERSION, + "kind": RECORD_KIND, + "page_url": page_url, + "opened_url": opened_url(page_url, token), + "port": int(port), + "pid": os.getpid(), + FRAGMENT_KEY: token, + } + target = private_copy_path(state, port) identity = _write_the_copy(state, target, record) except OSError as exc: raise ConsoleAccessRefused( @@ -652,6 +657,18 @@ def _write_the_copy(state: Path, target: Path, def read_private_copy(path: Path | str) -> dict: + """The record in the copy at `path`, or a refusal naming why: an + operating-system error on the way (an overlong component, an unsearchable + parent) included (Copilot at openDox-code#84, r4177975898).""" + try: + return _read_the_copy(path) + except OSError as exc: + raise ConsoleAccessRefused( + f"{path} cannot be read ({exc}), so it is not a private copy this " + "user can use") from None + + +def _read_the_copy(path: Path | str) -> dict: """The record in the copy at `path`, or a refusal naming why. The tree is judged again, and the file by its own descriptor, opened diff --git a/tests/test_console_token_delivery.py b/tests/test_console_token_delivery.py index bebff65d..866db3fa 100644 --- a/tests/test_console_token_delivery.py +++ b/tests/test_console_token_delivery.py @@ -1986,3 +1986,88 @@ def take(src, dst, *args, **kwargs): monkeypatch.undo() assert taken, "the stop was never staged" assert list((state / console_access.CONSOLE_DIRNAME).iterdir()) == [] + + +# --------------------------------------------------------------------------- +# 14 — Copilot's review at ddb26c34: every operating-system refusal on the +# way to the copy is a refusal by name, the walk's included +# --------------------------------------------------------------------------- + +def _overlong(tmp_path: Path) -> Path: + return tmp_path / ("x" * 300) / "state" # ENAMETOOLONG (errno 36) + + +def _unsearchable(tmp_path: Path) -> Path: + locked = tmp_path / "unsearchable" + locked.mkdir(mode=0o700) + locked.chmod(0o600) # no search (x) bit + return locked / "state" + + +@pytest.mark.parametrize("make", [_overlong, _unsearchable], + ids=["an overlong component", "an unsearchable parent"]) +def test_a_state_path_the_walk_cannot_take_is_refused_by_name(tmp_path, make) -> None: + """Copilot at openDox-code#84, r4177975898. The walk and the served-root + check ran outside the writer's conversion of operating-system errors, so + an overlong component (ENAMETOOLONG) or an unsearchable parent (EACCES) + escaped as a raw `OSError`, a traceback and no refusal by name. Each is + a `ConsoleAccessRefused` naming the copy, for the writer and the reader.""" + from opendox import console_access + + if make is _unsearchable and os.geteuid() == 0: + pytest.skip("root searches any directory") + state = make(tmp_path) + try: + with pytest.raises(console_access.ConsoleAccessRefused, + match="cannot be written") as refused: + _write(state) + assert str(console_access.private_copy_path(state, 8080)) in str(refused.value) + with pytest.raises(console_access.ConsoleAccessRefused, match="cannot be read"): + console_access.read_private_copy( + console_access.private_copy_path(state, 8080)) + finally: + if make is _unsearchable: + state.parent.chmod(0o700) + + +@pytest.mark.parametrize("make", [_overlong, _unsearchable], + ids=["an overlong component", "an unsearchable parent"]) +@pytest.mark.parametrize("entry", ["serve", "generate-and-open"]) +def test_a_state_path_the_walk_cannot_take_refuses_the_start( + tmp_path, monkeypatch, capsys, standalone_profile, make, entry) -> None: + """The same, through both entry points: exit 1 with the refusal named, + nothing printed that serves, and the listening socket closed.""" + from opendox import cli, serve + + if make is _unsearchable and os.geteuid() == 0: + pytest.skip("root searches any directory") + _clean_git(monkeypatch) + repo = _repository(tmp_path) + state = make(tmp_path) + monkeypatch.setenv("OPENDOX_STATE_DIR", str(state)) + monkeypatch.setattr(serve, "real_notebook_adapter", lambda *a, **k: None) + + def served(self, *args, **kwargs): + raise AssertionError("the server served") + + monkeypatch.setattr(socketserver.BaseServer, "serve_forever", served) + port = _free_port() + try: + if entry == "serve": + snapshot = tmp_path / "snapshot.json" + snapshot.write_text(json.dumps({"generation": {}}), encoding="utf-8") + assert serve.main(["--snapshot", str(snapshot), "--checkout-root", + str(repo), "--port", str(port)]) == 1 + refused, startup = "serve refused:", "serving ideation dashboard at" + else: + assert cli._generate_and_open( + _generate_and_open_args(tmp_path, repo, "--port", str(port)), + opener=lambda url: None) == 1 + refused, startup = "generate-and-open refused:", " serving " + out, err = capsys.readouterr() + assert refused in err and "cannot be written" in err, err + assert startup not in out, out + assert _port_is_free(port), "the refused start kept its socket" + finally: + if make is _unsearchable: + state.parent.chmod(0o700) From fb8a1cc4aeaa8d30f40cdf7fb3e93369258092ec Mon Sep 17 00:00:00 2001 From: Brett Heap <1513478+brettheap@users.noreply.github.com> Date: Sun, 4 Oct 2026 14:33:07 +0000 Subject: [PATCH 77/88] T104 fix round 7: Ctrl-C is held like SIGTERM while a copy is written or removed (Copilot review) Copilot's review at 9f328892 (5406635828), r4178041022. SIGINT kept Python's immediate handler, so deferred_termination never held it. A Ctrl-C just after publication's os.replace raised before the caller held its PrivateCopy, and the cleanup, holding None, left the copy behind. A second Ctrl-C just after a removal's take left a .removing-* file holding the token. terminate_as_interrupt now takes SIGINT as well, still raised as a KeyboardInterrupt, but only where it has Python's own handler (signal.default_int_handler). An ignored SIGINT, or a host's own handler, is left exactly as it was. Cases, each failing first at 9dcc9bb5. Each one checks for the handler before it sends SIGINT, so a raw KeyboardInterrupt never aborts the pytest session. - test_ctrl_c_just_after_the_copys_rename_leaves_no_copy, for serve and generate-and-open; - test_a_second_ctrl_c_after_the_removal_rename_leaves_nothing; - test_terminate_as_interrupt_takes_ctrl_c_only_from_its_default. Arc: neutral-product-standalone-operability Lane: openxfactory-4 (openXfactory-4-openDox_extraction) Co-Authored-By: Claude Opus 5.5 (1M context) --- src/opendox/console_access.py | 19 +++- tests/test_console_token_delivery.py | 129 +++++++++++++++++++++++++++ 2 files changed, 144 insertions(+), 4 deletions(-) diff --git a/src/opendox/console_access.py b/src/opendox/console_access.py index 4f36fa40..98bd4334 100644 --- a/src/opendox/console_access.py +++ b/src/opendox/console_access.py @@ -813,8 +813,8 @@ def _name_exists(name: str, directory: int) -> bool: class ConsoleTerminated(KeyboardInterrupt): - """SIGTERM or SIGHUP, raised as the interrupt the serve loops already - stop on.""" + """SIGTERM, SIGHUP or Ctrl-C, raised as the interrupt the serve loops + already stop on.""" #: Whether a stop is being HELD (`deferred_termination`), and the one that @@ -839,7 +839,8 @@ def deferred_termination(*, raise_pending: bool = True): `ConsoleTerminated` once the block is done, when the copy is in the caller's hands, or dropped with `raise_pending=False`, for a removal, which is a stop already. Only the handler `terminate_as_interrupt` - installs holds anything; Ctrl-C keeps Python's own.""" + installs holds anything: SIGTERM, SIGHUP where it has its default, and + Ctrl-C where it has Python's own.""" _held["depth"] += 1 try: yield @@ -865,7 +866,15 @@ def terminate_as_interrupt(enabled: bool): AND SIGHUP (T104's self-pass), which a closed terminal sends and whose default action ends the process with the copy left behind. It is read the same way, but only where it still has its default action: a process - started ignoring it (`nohup`) keeps ignoring it.""" + started ignoring it (`nohup`) keeps ignoring it. + + AND CTRL-C (Copilot at openDox-code#84, r4178041022). Python's own + SIGINT handler raises at once, so a Ctrl-C just after the copy's rename + into place, or just after a removal's take, bypassed + `deferred_termination` and left a copy, or a `.removing-*` file, behind. + It is taken the same way, still raised as a `KeyboardInterrupt`, but only + where it still has Python's own handler: an ignored SIGINT, or a host's + own handler, is left exactly as it was.""" if not enabled: yield return @@ -873,6 +882,8 @@ def terminate_as_interrupt(enabled: bool): hangup = getattr(signal, "SIGHUP", None) if hangup is not None and signal.getsignal(hangup) == signal.SIG_DFL: signals.append(hangup) + if signal.getsignal(signal.SIGINT) is signal.default_int_handler: + signals.append(signal.SIGINT) previous: dict = {} try: for signum in signals: diff --git a/tests/test_console_token_delivery.py b/tests/test_console_token_delivery.py index 866db3fa..e79cfad6 100644 --- a/tests/test_console_token_delivery.py +++ b/tests/test_console_token_delivery.py @@ -2071,3 +2071,132 @@ def served(self, *args, **kwargs): finally: if make is _unsearchable: state.parent.chmod(0o700) + + +# --------------------------------------------------------------------------- +# 15 — Copilot's review at 9f328892: Ctrl-C is held like SIGTERM while a copy +# is written or removed (r4178041022) +# --------------------------------------------------------------------------- + +def _ctrl_c_is_held() -> None: + """Never send a SIGINT that Python's own handler would raise at once: a + raw KeyboardInterrupt aborts the whole pytest session, not one case.""" + assert signal.getsignal(signal.SIGINT) is not signal.default_int_handler, ( + "Ctrl-C still has Python's immediate handler") + + +def test_terminate_as_interrupt_takes_ctrl_c_only_from_its_default() -> None: + """Ctrl-C is held like SIGTERM, where it still has Python's own handler; + an ignored one, or a host's own handler, is left exactly as it was.""" + from opendox import console_access + + previous = signal.getsignal(signal.SIGINT) + try: + signal.signal(signal.SIGINT, signal.default_int_handler) + with console_access.terminate_as_interrupt(True): + _ctrl_c_is_held() + assert signal.getsignal(signal.SIGINT) is signal.default_int_handler + + def host(signum, frame): # a host's own handler + raise AssertionError("never sent") + + for kept in (signal.SIG_IGN, host): + signal.signal(signal.SIGINT, kept) + with console_access.terminate_as_interrupt(True): + assert signal.getsignal(signal.SIGINT) is kept + assert signal.getsignal(signal.SIGINT) is kept + finally: + signal.signal(signal.SIGINT, previous) + + +@pytest.mark.parametrize("entry", ["serve", "generate-and-open"]) +def test_ctrl_c_just_after_the_copys_rename_leaves_no_copy( + tmp_path, monkeypatch, capsys, standalone_profile, entry) -> None: + """The publication window: Ctrl-C right after the copy is renamed into + place, before the caller holds it, used to raise at once, and the + caller's cleanup, holding nothing, left the copy behind. It is held until + the copy is in hand, then the start stops cleanly with no copy left.""" + from opendox import cli, console_access, serve + + _clean_git(monkeypatch) + repo = _repository(tmp_path) + state = _state(tmp_path) + monkeypatch.setenv("OPENDOX_STATE_DIR", str(state)) + monkeypatch.setattr(serve, "real_notebook_adapter", lambda *a, **k: None) + previous = signal.signal(signal.SIGINT, signal.default_int_handler) + real_replace = os.replace + sent: list = [] + + def then_ctrl_c(src, dst, *args, **kwargs): + real_replace(src, dst, *args, **kwargs) + if not sent and str(dst).endswith(".html"): + sent.append(dst) + _ctrl_c_is_held() + os.kill(os.getpid(), signal.SIGINT) + signal.pthread_sigmask(signal.SIG_BLOCK, []) # deliver now + + def served(self, *args, **kwargs): + raise AssertionError("the server served after a stop") + + monkeypatch.setattr(console_access.os, "replace", then_ctrl_c) + monkeypatch.setattr(socketserver.BaseServer, "serve_forever", served) + try: + if entry == "serve": + snapshot = tmp_path / "snapshot.json" + snapshot.write_text(json.dumps({"generation": {}}), encoding="utf-8") + assert serve.main(["--snapshot", str(snapshot), "--checkout-root", + str(repo), "--port", "0"]) == 0 + else: + args = cli.build_parser().parse_args([ + "generate-and-open", "--repo-root", str(repo), "--repository", + "fixture", "--run-dir", str(tmp_path / "run"), "--port", "0", + "--no-validate", "--no-open"]) + assert cli._generate_and_open(args, opener=lambda url: None) == 0 + finally: + signal.signal(signal.SIGINT, previous) + out = capsys.readouterr().out + assert sent, "the Ctrl-C was never staged" + assert "console " not in out, out + assert list((state / console_access.CONSOLE_DIRNAME).iterdir()) == [] + + +def test_a_second_ctrl_c_after_the_removal_rename_leaves_nothing( + tmp_path, monkeypatch, capsys, standalone_profile) -> None: + """The removal window: a second Ctrl-C right after the removal renamed + the copy to its temporary name used to raise there and leave a + `.removing-*` file holding the token. It is held, the removal finishes, + and `console/` is left empty.""" + from opendox import console_access, serve + + _clean_git(monkeypatch) + repo = _repository(tmp_path) + state = _state(tmp_path) + monkeypatch.setenv("OPENDOX_STATE_DIR", str(state)) + monkeypatch.setattr(serve, "real_notebook_adapter", lambda *a, **k: None) + previous = signal.signal(signal.SIGINT, signal.default_int_handler) + real_rename = os.rename + sent: list = [] + + def stopped_at_once(self, *args, **kwargs): + raise KeyboardInterrupt # the first Ctrl-C + + def then_ctrl_c(src, dst, *args, **kwargs): + real_rename(src, dst, *args, **kwargs) + if not sent and ".removing-" in str(dst): + sent.append(dst) + _ctrl_c_is_held() + os.kill(os.getpid(), signal.SIGINT) # the second + signal.pthread_sigmask(signal.SIG_BLOCK, []) + + monkeypatch.setattr(socketserver.BaseServer, "serve_forever", stopped_at_once) + monkeypatch.setattr(console_access.os, "rename", then_ctrl_c) + snapshot = tmp_path / "snapshot.json" + snapshot.write_text(json.dumps({"generation": {}}), encoding="utf-8") + try: + assert serve.main(["--snapshot", str(snapshot), "--checkout-root", + str(repo), "--port", "0"]) == 0 + finally: + signal.signal(signal.SIGINT, previous) + monkeypatch.undo() + assert sent, "the second Ctrl-C was never staged" + assert list((state / console_access.CONSOLE_DIRNAME).iterdir()) == [] From 66cffdb87e40304a7b627afe75d3bcc49724a3a2 Mon Sep 17 00:00:00 2001 From: Brett Heap <1513478+brettheap@users.noreply.github.com> Date: Sun, 4 Oct 2026 15:24:47 +0000 Subject: [PATCH 78/88] T104 fix round 8: a running console's copy is reserved, and every unguarded read judges the file it opened (Copilot review) Copilot's review at fb8a1cc4 (5406735555), two threads. Each case failed first at fb8a1cc4: - r4178133814, two consoles sharing a port number. The copy's name is per port, so a console on 127.0.0.1 and one on ::1, sharing a state directory, replaced each other's copy. The writer now takes an exclusive flock on the file it wrote, before the rename, on its own descriptor (the _Reservation in PrivateCopy.reservation), and keeps it until removal releases it. A later publication on that port probes the lock without waiting. A held lock is a running console's, refused by name ("belongs to a console that is still running (pid N)") and left as it was. A free lock is a stale copy's, replaced. Cases: - test_a_running_consoles_copy_is_never_replaced (failed first: DID NOT RAISE); - test_two_consoles_on_one_port_number_never_share_a_copy, with real IPv4 and IPv6 planes (failed first: DID NOT RAISE); - test_a_copy_whose_console_died_is_replaced, a pin: a subprocess writes a copy and exits, and the kernel releases its lock. The cases that model a serve gone without removing its copy now say so (_abandon). - r4178133842, a source root retargeted after publication. /source resolves a declared root again on every request, so a root re-pointed at the state directory after publication served the copy. Every read that serves a file to any caller without the console check now judges the file it OPENED, by (st_dev, st_ino), against every name in the private-copy directory (console_access.is_private_file): - /source and /snapshot.json read through serve.read_unless_private; - the static handler judges the file it would serve before the stdlib opens it (console_access.opens_a_private_file), and judges again what the stdlib opened. If a link was swapped in between, that file is closed unsent and the connection closed. The identity check closes hard links to the copy as well, which a path cannot tell apart. test_source_core_arm's read-only pin now names the guarded reader (read_unless_private, itself only opened "rb") in place of read_bytes(). It stays AST-only and --noconftest safe. Cases: - test_a_source_root_retargeted_after_publication_never_serves_the_copy; - test_a_snapshot_retargeted_after_publication_never_serves_the_copy; - test_a_hard_link_to_the_copy_is_never_served, for the static bundle and the served checkout. Each failed first: GET answered 200 with the copy. The backstop's case, test_the_static_backstop_never_sends_a_copy_swapped_in_after_the_check, blinds the first check and reads the raw response: the copy's bytes are never sent. Arc: neutral-product-standalone-operability Lane: openxfactory-4 (openXfactory-4-openDox_extraction) Co-Authored-By: Claude Opus 5.5 (1M context) --- src/opendox/console_access.py | 167 ++++++++++++++-- src/opendox/serve.py | 76 ++++++- tests/test_console_token_delivery.py | 283 ++++++++++++++++++++++++++- tests/test_source_core_arm.py | 13 +- 4 files changed, 512 insertions(+), 27 deletions(-) diff --git a/src/opendox/console_access.py b/src/opendox/console_access.py index 98bd4334..0b154dcd 100644 --- a/src/opendox/console_access.py +++ b/src/opendox/console_access.py @@ -102,7 +102,8 @@ "CONSOLE_DIRNAME", "ConsoleAccessRefused", "ConsoleTerminated", "DELIVERY_CAPABILITIES", "DELIVERY_OPENED_URL", "FRAGMENT_KEY", "PrivateCopy", "RECORD_ELEMENT_ID", - "RECORD_KIND", "deferred_termination", "delivery_for", "needs_copy", + "RECORD_KIND", "deferred_termination", "delivery_for", "is_private_file", + "needs_copy", "opens_a_private_file", "opened_url", "private_copy_path", "publish", "read_private_copy", "remove_private_copy", "terminate_as_interrupt", "write_private_copy", ] @@ -151,6 +152,10 @@ class PrivateCopy: #: `(st_dev, st_ino)` of the file this process wrote, so a removal at #: shutdown removes that file and never one written after it. identity: tuple[int, int] + #: The open descriptor that RESERVES the copy for its server's life + #: (`_Reservation`), or None where no reservation could be taken. + reservation: "_Reservation | None" = dataclasses.field( + default=None, compare=False, repr=False) @property def file_url(self) -> str: @@ -158,6 +163,32 @@ def file_url(self) -> str: return self.path.as_uri() +class _Reservation: + """The open descriptor that holds a copy's lock while its server runs. + + A COPY IS RESERVED FOR ITS SERVER'S LIFE (Copilot at openDox-code#84, + r4178133814). The copy's name is per PORT, and two consoles can share a + port number and a state directory, one on 127.0.0.1 and one on ::1. So + the writer takes an exclusive `flock` on the file it wrote, on its own + descriptor, and keeps it until the copy is removed. A later publication + on that port finds the lock held, and refuses rather than replace a + RUNNING console's copy. A copy whose server died holds no lock, since the + kernel drops it with the process, and is replaced as a stale one. + `close` is idempotent, and so is dropping the reservation.""" + + def __init__(self, fd: int) -> None: + self._fd: int | None = fd + + def close(self) -> None: + fd, self._fd = self._fd, None + if fd is not None: + with contextlib.suppress(OSError): + os.close(fd) + + def __del__(self) -> None: + self.close() + + def delivery_for(profile: Any) -> str: """Which delivery a plane built from `profile` uses. @@ -577,14 +608,15 @@ def write_private_copy(state_dir: Path | str, *, page_url: str, port: int, FRAGMENT_KEY: token, } target = private_copy_path(state, port) - identity = _write_the_copy(state, target, record) + identity, reservation = _write_the_copy(state, target, record) except OSError as exc: raise ConsoleAccessRefused( f"{target} cannot be written ({exc}), so the console token has no " "private copy and the start is refused. Use a state directory this " f"user can write ({runtime_config.PREFIX}STATE_DIR)") from None copy = PrivateCopy(path=target, page_url=page_url, - opened_url=record["opened_url"], identity=identity) + opened_url=record["opened_url"], identity=identity, + reservation=reservation) try: read_private_copy(target) # what was written is what a reader accepts except BaseException: @@ -627,6 +659,8 @@ def _write_the_copy(state: Path, target: Path, f"{target} {reason}: something other than this user's own " "private copy is at that name, so it is refused, never " "followed or replaced") + if present is not None: + _refuse_a_running_consoles_copy(target, directory, present) with contextlib.suppress(FileNotFoundError): os.unlink(temporary, dir_fd=directory) # an interrupted start's; a link itself, never its target handle = os.open(temporary, @@ -634,26 +668,82 @@ def _write_the_copy(state: Path, target: Path, PRIVATE_MODE, dir_fd=directory) # From here a failure (a full disk, an interrupt) removes the # temporary file it made, so no partial copy is left beside the name. + # The descriptor stays open on success: it is the copy's RESERVATION + # (`_Reservation`), locked before the copy takes its name, so no other + # publication can find the name unreserved. + reservation = _Reservation(handle) try: - try: - os.fchmod(handle, PRIVATE_MODE) - data = _opener_html(record).encode("utf-8") - view = memoryview(data) - while view: - view = view[os.write(handle, view):] - os.fsync(handle) - written = os.fstat(handle) - finally: - os.close(handle) + os.fchmod(handle, PRIVATE_MODE) + data = _opener_html(record).encode("utf-8") + view = memoryview(data) + while view: + view = view[os.write(handle, view):] + os.fsync(handle) + written = os.fstat(handle) + if fcntl is not None: + with contextlib.suppress(OSError): # no locks here: unreserved + fcntl.flock(handle, fcntl.LOCK_EX | fcntl.LOCK_NB) os.replace(temporary, target.name, src_dir_fd=directory, dst_dir_fd=directory) except BaseException: + reservation.close() with contextlib.suppress(FileNotFoundError): os.unlink(temporary, dir_fd=directory) raise finally: os.close(directory) - return (written.st_dev, written.st_ino) + return (written.st_dev, written.st_ino), reservation + + +def _refuse_a_running_consoles_copy(target: Path, directory: int, + present: os.stat_result) -> None: + """Refuse when the copy at `target` is RESERVED by a console that is still + running (`_Reservation`); return where it is a stale copy, to be replaced. + + The copy is opened without following a link or blocking, checked to be + the file judged a moment ago, and its lock asked for WITHOUT waiting: a + lock that is held is a running console's, and one that is free is a + stale copy's. Where the filesystem keeps no locks, nothing can be told, + and the copy is replaced, as before. The console directory's lock + (`_lock`) is held throughout, so no publication races this one.""" + try: + held = os.open(target.name, os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK, + dir_fd=directory) + except FileNotFoundError: + return + try: + info = os.fstat(held) + if (info.st_dev, info.st_ino) != (present.st_dev, present.st_ino): + raise ConsoleAccessRefused( + f"{target} changed while it was judged, so it is refused, " + "never replaced") + if fcntl is None: + return + try: + fcntl.flock(held, fcntl.LOCK_EX | fcntl.LOCK_NB) + except BlockingIOError: + raise ConsoleAccessRefused( + f"{target} belongs to a console that is still running" + f"{_writer_of(held)}. Two serves on this port number share " + f"this state directory (for example one on 127.0.0.1 and one " + "on ::1), and a running console's copy is never replaced. " + "Stop that console, or serve on another port or with another " + f"{runtime_config.PREFIX}STATE_DIR") from None + except OSError: + return # no locks here: replace, as before + finally: + os.close(held) + + +def _writer_of(handle: int) -> str: + """`" (pid N)"` from the copy's own record, or nothing.""" + with contextlib.suppress(OSError, ValueError, AttributeError): + found = _RECORD_PATTERN.search( + os.pread(handle, _READ_LIMIT, 0).decode("utf-8", "replace")) + pid = json.loads(found.group("record")).get("pid") + if isinstance(pid, int): + return f" (pid {pid})" + return "" def read_private_copy(path: Path | str) -> dict: @@ -775,6 +865,8 @@ def remove_private_copy(copy: PrivateCopy | None) -> None: directory = os.open(copy.path.parent, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW) except OSError: + if copy.reservation is not None: # nothing left to remove: unreserve + copy.reservation.close() return name = copy.path.name taken = f".{name}.removing-{os.getpid()}-{os.urandom(6).hex()}" @@ -801,6 +893,10 @@ def remove_private_copy(copy: PrivateCopy | None) -> None: with contextlib.suppress(OSError): os.unlink(taken, dir_fd=directory) finally: + # Its RESERVATION goes with it, the file gone and the console + # directory still locked, so no publication sees it in between. + if copy.reservation is not None: + copy.reservation.close() os.close(directory) @@ -900,6 +996,49 @@ def terminate_as_interrupt(enabled: bool): signal.signal(signum, handler) +def is_private_file(handle: int, private_roots: Iterable[Path | str]) -> bool: + """Whether the file open on `handle` lives in a private-copy directory. + + Judged by the FILE'S OWN IDENTITY, `(st_dev, st_ino)`, against every name + in each directory `publish` marked private (Copilot at openDox-code#84, + r4178133842). A path cannot tell: a served root re-pointed at the state + directory after publication, a link swapped after a check, or a hard link + made anywhere under a served root all reach the copy by a name that looks + like something else. The identity of what was OPENED cannot be swapped + afterwards. Every name in the directory counts, every port's copy and a + temporary name included. A directory that cannot be listed counts for + nothing, as no copy can be published in it either.""" + info = os.fstat(handle) + key = (info.st_dev, info.st_ino) + for root in private_roots: + try: + with os.scandir(root) as entries: + for entry in entries: + with contextlib.suppress(OSError): + found = entry.stat(follow_symlinks=False) + if (found.st_dev, found.st_ino) == key: + return True + except OSError: + continue + return False + + +def opens_a_private_file(path: Path | str, + private_roots: Iterable[Path | str]) -> bool: + """Whether what opens at `path` is a file in a private-copy directory + (`is_private_file`). Opened without blocking, so a FIFO cannot stall the + check; a path that does not open is no private file, and is left to the + caller to answer as it always has.""" + try: + handle = os.open(path, os.O_RDONLY | os.O_NONBLOCK) + except OSError: + return False + try: + return is_private_file(handle, private_roots) + finally: + os.close(handle) + + def needs_copy(httpd: Any) -> bool: """Whether the plane `httpd` delivers its console token through a private copy: a standalone plane that minted one. The entry points ask it diff --git a/src/opendox/serve.py b/src/opendox/serve.py index b8f2249e..50a3279a 100644 --- a/src/opendox/serve.py +++ b/src/opendox/serve.py @@ -812,6 +812,25 @@ def host_names_this_loopback_serve(host_lines, port: int, # --------------------------- source-path containment (pure) --------------------------- +def read_unless_private(path: Path | str, private_roots) -> bytes | None: + """The bytes of `path`, or None where the file it OPENS is a console + token's private copy (`console_access.is_private_file`, plan 034 T104). + + Every route that reads a file for any caller, with no console check, + reads through this (Copilot at openDox-code#84, r4178133842): `/source` + and `/snapshot.json`. A root or a snapshot named through a link is + resolved again on every request, and could be re-pointed at the state + directory after the copy was published; a hard link reaches the copy by + another name. The file actually opened is judged, so neither is served. + With no private root (a host's plane, or a plane that wrote no copy) it + reads as before.""" + with open(path, "rb") as stream: + if private_roots and console_access.is_private_file( + stream.fileno(), private_roots): + return None + return stream.read() + + def resolve_source_path(checkout_root: Path, url_tail: str) -> Path | None: """Resolve a `/source/` request to an absolute file under `checkout_root`, or None to reject. Rejects absolute paths, NUL bytes, any @@ -1352,7 +1371,16 @@ def send_head(self): the directory's first index page that exists (`index_pages`: `index.html`, then `index.htm`), so the index page it would pick is judged as well as the directory, and `web/sub/index.html` linked to a - copy is a 404 like the link itself.""" + copy is a 404 like the link itself. + + AND BY THE IDENTITY OF THE FILE (Copilot at openDox-code#84, + r4178133842). A path cannot tell a hard link to the copy from any + other file, so the file the handler would serve is opened and judged + by `(st_dev, st_ino)` first (`console_access.is_private_file`), and a + private copy is a 404. The file the stdlib handler then opens is + judged the same way, against a link swapped in between: its headers + are already sent by then, so it is closed unread and its bytes are + never written.""" private = getattr(self.server, "private_roots", ()) if private: path = Path(self.translate_path(self.path)) @@ -1364,11 +1392,22 @@ def send_head(self): break for candidate in judged: target = candidate.resolve() - if any(target == root or root in target.parents - for root in private): + if (any(target == root or root in target.parents + for root in private) + or console_access.opens_a_private_file(candidate, private)): self.send_error(404, "File not found") return None - return super().send_head() + stream = super().send_head() + if private and stream is not None: + try: + handle = stream.fileno() + except (AttributeError, OSError, ValueError): + handle = None # a directory listing, in memory + if handle is not None and console_access.is_private_file(handle, private): + stream.close() + self.close_connection = True # its promised body never comes + return None + return stream def do_GET(self): # noqa: N802 if not self._route(head_only=False): @@ -1407,12 +1446,26 @@ def _read_snapshot(self, entry=None) -> bytes | None: pass the hosted refusal with one entry and serve another's bytes (FR-048).""" if entry is not None: - return entry.read_bytes() + return self._entry_bytes(entry) try: - return Path(self.snapshot_path).read_bytes() + return read_unless_private( + self.snapshot_path, getattr(self.server, "private_roots", ())) except OSError: return None + def _entry_bytes(self, entry) -> bytes | None: + """A registered entry's snapshot bytes, read as `read_unless_private` + reads, where this plane marked a private-copy directory and the entry + names its file. Otherwise the entry reads itself, as before.""" + private = getattr(self.server, "private_roots", ()) + path = getattr(entry, "snapshot_path", None) + if private and path is not None: + try: + return read_unless_private(path, private) + except OSError: + return None + return entry.read_bytes() + def _serve_snapshot(self, head_only: bool) -> None: """`/snapshot.json`: the active snapshot, or the registered `(repository, ref)` the query names. An unknown pair is a 404, and the @@ -1445,7 +1498,7 @@ def _serve_snapshot(self, head_only: bool) -> None: return if self._hosted_entry_refused(entry): return - self._serve_bytes(entry.read_bytes(), JSON_CTYPE, head_only, + self._serve_bytes(self._entry_bytes(entry), JSON_CTYPE, head_only, entry=entry) return if repository: @@ -1560,10 +1613,17 @@ def _serve_source(self, tail: str, head_only: bool) -> None: self.end_headers() return try: - body = target.read_bytes() + body = read_unless_private( + target, getattr(self.server, "private_roots", ())) except OSError: self.send_error(404, "unreadable source") return + if body is None: # a console token's private copy + self.send_response(404) + self._divergence_headers(entry) + self.send_header("Content-Length", "0") + self.end_headers() + return ctype = "text/markdown; charset=utf-8" if target.suffix == ".md" else "text/plain; charset=utf-8" self._serve_bytes(body, ctype, head_only, entry=entry) diff --git a/tests/test_console_token_delivery.py b/tests/test_console_token_delivery.py index e79cfad6..17ed72d0 100644 --- a/tests/test_console_token_delivery.py +++ b/tests/test_console_token_delivery.py @@ -576,12 +576,14 @@ def test_a_state_directory_below_a_non_sticky_shared_directory_is_refused(tmp_pa def test_a_later_copy_replaces_this_users_earlier_one_and_removal_is_exact( tmp_path) -> None: - """A server restarted on the same port replaces its own earlier copy. The - first server's removal at shutdown leaves the later copy in place.""" + """A server restarted on the same port, after the first ended without + removing its copy, replaces that copy. The first server's removal, had it + still run, leaves the later copy in place.""" from opendox import console_access state = _state(tmp_path) first = _write(state) + _abandon(first) # the first serve is gone second_token = _token() second = _write(state, token=second_token) assert first.path == second.path @@ -1536,8 +1538,9 @@ def test_another_serves_copy_survives_a_removal_where_hard_links_fail( state = _state(tmp_path) first = _write(state) + _abandon(first) # its reservation lost second_token = _token() - _write(state, token=second_token) # another serve's, over the first + second = _write(state, token=second_token) # another serve's, over the first def no_hard_links(*args, **kwargs): raise PermissionError(errno.EPERM, "Operation not permitted") @@ -1665,6 +1668,7 @@ def test_a_console_directory_with_a_setgid_bit_is_still_0700(tmp_path) -> None: if not os.lstat(copy.path.parent).st_mode & stat.S_ISGID: pytest.skip("this filesystem keeps no setgid bit on a directory") assert console_access.read_private_copy(copy.path)["port"] == 8080 + _abandon(copy) assert _write(state).path == copy.path @@ -1852,8 +1856,10 @@ def test_a_copy_published_during_a_rename_back_is_never_overwritten( state = _state(tmp_path) first = _write(state) + _abandon(first) # its reservation lost second_token, third_token = _token(), _token() - _write(state, token=second_token) # another serve's, over the first + second = _write(state, token=second_token) # another serve's, over the first + _abandon(second) # ...whose serve is gone too real_exists = console_access._name_exists raced: list = [] @@ -2200,3 +2206,272 @@ def then_ctrl_c(src, dst, *args, **kwargs): monkeypatch.undo() assert sent, "the second Ctrl-C was never staged" assert list((state / console_access.CONSOLE_DIRNAME).iterdir()) == [] + + +# --------------------------------------------------------------------------- +# 16 — Copilot's review at fb8a1cc4: a live console's copy is reserved for +# its server's life (r4178133814); every unguarded file read refuses a +# private copy by the file's own identity (r4178133842) +# --------------------------------------------------------------------------- + +def _abandon(copy) -> None: + """The serve that wrote `copy` is gone without removing it (a crash, a + SIGKILL): its reservation is released, as the kernel releases it.""" + reservation = getattr(copy, "reservation", None) + if reservation is not None: + reservation.close() + + +def test_a_running_consoles_copy_is_never_replaced(tmp_path) -> None: + """Two consoles can share a port number (127.0.0.1 and ::1) and a state + directory. The first's copy is reserved while it runs: a second + publication on that port is refused by name, and the first's copy is left + exactly as it was, still opening the first console. Once the first stops + and removes its copy, the port's copy can be written again.""" + from opendox import console_access + + state = _state(tmp_path) + first = _write(state) + before = (first.path.read_bytes(), _fingerprint(first.path)) + with pytest.raises(console_access.ConsoleAccessRefused, + match="still running") as refused: + _write(state) + assert str(first.path) in str(refused.value) + assert (first.path.read_bytes(), _fingerprint(first.path)) == before + console_access.remove_private_copy(first) + assert _write(state).path == first.path + + +def test_a_copy_whose_console_died_is_replaced(tmp_path) -> None: + """A copy whose writer died without removing it (here a process that + writes one and exits at once) holds no reservation, since the kernel + released it with the process. It is a stale copy, and is replaced.""" + import subprocess + import sys + + from opendox import console_access + + state = _state(tmp_path) + env = {k: v for k, v in os.environ.items() if not k.startswith(("GIT_", "XF_"))} + env["PYTHONPATH"] = os.pathsep.join([str(ROOT / "src"), env.get("PYTHONPATH", "")]) + done = subprocess.run( + [sys.executable, "-c", + "import sys; from opendox import console_access, serve\n" + "console_access.write_private_copy(sys.argv[1], " + "page_url='http://127.0.0.1:8080/index.html', port=8080, " + "token=serve.mint_console_token(), served_roots=())\n", + str(state)], env=env, capture_output=True, text=True, timeout=60) + assert done.returncode == 0, done.stderr + stale = console_access.private_copy_path(state, 8080) + old = console_access.read_private_copy(stale)["console_token"] + token = _token() + assert _write(state, token=token).path == stale + assert console_access.read_private_copy(stale)["console_token"] == token != old + + +@pytest.mark.skipif(not socket.has_ipv6, reason="no IPv6 on this platform") +def test_two_consoles_on_one_port_number_never_share_a_copy( + tmp_path, monkeypatch, standalone_profile) -> None: + """Copilot's layout, as two real planes: one bound to 127.0.0.1 and one + to ::1, on the same port number and the same state directory. The second + is refused by name, and the first's copy still opens the first.""" + from opendox import console_access, serve + + _clean_git(monkeypatch) + repo = _repository(tmp_path) + state = _state(tmp_path) + snapshot = tmp_path / "snapshot.json" + snapshot.write_text(json.dumps({"generation": {}}), encoding="utf-8") + v4 = serve.build_server(WEB, snapshot, repo, host="127.0.0.1", port=0, quiet=True) + port = v4.server_address[1] + try: + v6 = serve.build_server(WEB, snapshot, repo, host="::1", port=port, quiet=True) + except OSError as exc: + v4.server_close() + pytest.skip(f"no IPv6 loopback here: {exc}") + try: + env = {"OPENDOX_STATE_DIR": str(state)} + first = console_access.publish( + v4, page_url=serve.server_url(v4, "/index.html"), env=env) + with pytest.raises(console_access.ConsoleAccessRefused, match="still running"): + console_access.publish( + v6, page_url=serve.server_url(v6, "/index.html"), env=env) + record = console_access.read_private_copy(first.path) + assert record["console_token"] == v4.console_token + assert "127.0.0.1" in record["page_url"] + console_access.remove_private_copy(first) + finally: + v4.server_close() + v6.server_close() + + +def _guarded_plane(tmp_path, monkeypatch, **build): + from opendox import serve + + _clean_git(monkeypatch) + repo = build.pop("repo", None) or _repository(tmp_path) + snapshot = tmp_path / "snapshot.json" + snapshot.write_text(json.dumps({"generation": {}}), encoding="utf-8") + web = build.pop("web", WEB) + httpd = serve.build_server(web, snapshot, repo, port=0, quiet=True, **build) + worker = threading.Thread(target=httpd.serve_forever, daemon=True) + worker.start() + return httpd, repo, worker + + +def _stop_plane(httpd, worker) -> None: + httpd.shutdown() + httpd.server_close() + worker.join(timeout=10) + + +def _assert_never_served(base, token: str, paths) -> None: + for path in paths: + for method in ("GET", "HEAD"): + status, headers, raw = _call(base, method, path) + assert status == 404, (method, path, status) + assert token.encode() not in raw, (method, path) + assert all(token not in str(v) for v in headers.values()), path + + +def test_a_source_root_retargeted_after_publication_never_serves_the_copy( + tmp_path, monkeypatch, standalone_profile) -> None: + """Copilot at openDox-code#84, r4178133842. A declared source root named + through a link is judged where the link leads when the copy is + published, but `/source` resolves it again on every request. Re-pointed + at the state directory afterwards, it used to serve the copy to anyone. + Every `/source` read is judged by the identity of the file it opened, so + the copy is never served, whatever the root leads to now.""" + from opendox import console_access, serve + + elsewhere = tmp_path / "elsewhere" + elsewhere.mkdir() + (elsewhere / "note.md").write_text("# a note\n", encoding="utf-8") + alias = tmp_path / "root-alias" + alias.symlink_to(elsewhere) + state = _state(tmp_path) + httpd, _repo, worker = _guarded_plane( + tmp_path, monkeypatch, repository="other", source_roots={"other": str(alias)}) + try: + base = httpd.server_address[:2] + copy = console_access.publish( + httpd, page_url=serve.server_url(httpd, "/index.html"), + env={"OPENDOX_STATE_DIR": str(state)}) + assert _call(base, "GET", "/source/note.md")[0] == 200 + alias.unlink() + alias.symlink_to(state) # retargeted after the check + name = copy.path.name + _assert_never_served(base, httpd.console_token, + (f"/source/console/{name}", + f"/source/other@main/console/{name}")) + finally: + _stop_plane(httpd, worker) + + +def test_a_snapshot_retargeted_after_publication_never_serves_the_copy( + tmp_path, monkeypatch, standalone_profile) -> None: + """The same for `/snapshot.json`, which reads a registered entry's file + directly: an entry whose snapshot is named through a link re-pointed at + the copy after publication is a 404, never the copy.""" + from opendox import console_access, default_registry, serve + + _clean_git(monkeypatch) + repo = _repository(tmp_path) + real = tmp_path / "other.snapshot.json" + real.write_text(json.dumps({"generation": {}}), encoding="utf-8") + alias = tmp_path / "snapshot-alias.json" + alias.symlink_to(real) + source = default_registry.SnapshotSource( + baked_snapshot=tmp_path / "snapshot.json", checkout_root=repo) + source.registry.register(default_registry.entry_from_snapshot_file( + alias, repository="other", ref="main")) + state = _state(tmp_path) + httpd, _repo, worker = _guarded_plane(tmp_path, monkeypatch, repo=repo, + snapshot_source=source) + try: + base = httpd.server_address[:2] + copy = console_access.publish( + httpd, page_url=serve.server_url(httpd, "/index.html"), + env={"OPENDOX_STATE_DIR": str(state)}) + path = "/snapshot.json?repository=other&ref=main" + assert _call(base, "GET", path)[0] == 200 + alias.unlink() + alias.symlink_to(copy.path) # retargeted after the check + _assert_never_served(base, httpd.console_token, (path,)) + finally: + _stop_plane(httpd, worker) + + +@pytest.mark.parametrize("where", ["the static bundle", "the served checkout"]) +def test_a_hard_link_to_the_copy_is_never_served( + tmp_path, monkeypatch, standalone_profile, where) -> None: + """A path cannot tell a hard link from the file itself: `web/x.html`, or + `checkout/x.md`, hard-linked to the copy, resolves to a name outside the + state directory. The file's own identity tells, so the static handler + and `/source` answer 404 and never send the copy.""" + import shutil as _shutil + + from opendox import console_access, serve + + web = tmp_path / "web" + _shutil.copytree(WEB, web) + state = _state(tmp_path) + httpd, repo, worker = _guarded_plane(tmp_path, monkeypatch, web=web) + try: + base = httpd.server_address[:2] + copy = console_access.publish( + httpd, page_url=serve.server_url(httpd, "/index.html"), + env={"OPENDOX_STATE_DIR": str(state)}) + if where == "the static bundle": + os.link(copy.path, web / "hard.html") + paths = ("/hard.html",) + else: + os.link(copy.path, repo / "hard.md") + paths = ("/source/hard.md",) + _assert_never_served(base, httpd.console_token, paths) + assert _call(base, "GET", "/index.html")[0] == 200 + finally: + _stop_plane(httpd, worker) + + +def _raw_get(base, path: str) -> bytes: + """Every byte the server sends for `GET path`, read until it closes: + a response cut short is read as far as it went, never raised.""" + with socket.create_connection(base, timeout=30) as conn: + conn.sendall(f"GET {path} HTTP/1.0\r\nHost: {base[0]}:{base[1]}\r\n\r\n" + .encode("ascii")) + received = b"" + while True: + chunk = conn.recv(65536) + if not chunk: + return received + received += chunk + + +def test_the_static_backstop_never_sends_a_copy_swapped_in_after_the_check( + tmp_path, monkeypatch, standalone_profile) -> None: + """The file the stdlib handler opens is judged after it opens, against a + link swapped in between the handler's own check and that open. Staged by + blinding the first check: the copy's bytes are still never sent.""" + import shutil as _shutil + + from opendox import console_access, serve + + web = tmp_path / "web" + _shutil.copytree(WEB, web) + state = _state(tmp_path) + httpd, _repo, worker = _guarded_plane(tmp_path, monkeypatch, web=web) + try: + base = httpd.server_address[:2] + copy = console_access.publish( + httpd, page_url=serve.server_url(httpd, "/index.html"), + env={"OPENDOX_STATE_DIR": str(state)}) + os.link(copy.path, web / "swapped.html") + monkeypatch.setattr(console_access, "opens_a_private_file", + lambda path, roots: False) # the race, won + received = _raw_get(base, "/swapped.html") + assert httpd.console_token.encode() not in received, received[:300] + assert b"opendox-console" not in received + assert b"index" in _raw_get(base, "/index.html").lower() + finally: + _stop_plane(httpd, worker) diff --git a/tests/test_source_core_arm.py b/tests/test_source_core_arm.py index 4d579781..a9c9d13f 100644 --- a/tests/test_source_core_arm.py +++ b/tests/test_source_core_arm.py @@ -387,7 +387,18 @@ def test_the_route_is_read_only(): body = _source_of(_method("_serve_source")) for writer in ("write_bytes", "write_text", "unlink", "mkdir", "rename"): assert writer not in body, f"_serve_source calls {writer}" - assert "read_bytes()" in body + # It READS through `read_unless_private` (plan 034 T104; Copilot at + # openDox-code#84, r4178133842), which judges the file it opened and + # never sends a console token's private copy. That reader is read-only + # too: it opens for reading, and writes nothing. + assert "read_unless_private(" in body + reader = next((_source_of(node) for node in SERVE_TREE.body + if isinstance(node, ast.FunctionDef) + and node.name == "read_unless_private"), "") + assert 'open(path, "rb")' in reader, "read_unless_private reads nothing" + for writer in ("write_bytes", "write_text", "unlink", "mkdir", "rename", + '"w', '"a'): + assert writer not in reader, f"read_unless_private calls {writer}" # --------------------------------------------------------------------------- From 9fe57dffa9d513930c22d687ceca62e39a50f358 Mon Sep 17 00:00:00 2001 From: Brett Heap <1513478+brettheap@users.noreply.github.com> Date: Sun, 4 Oct 2026 16:18:23 +0000 Subject: [PATCH 79/88] T104: a snapshot named at a copy not yet written refuses the start; the hold case judges every interrupt Mutant run 16 at fb8a1cc4 left M32 ("the snapshot file is not a served root") alive: every case that named the snapshot at a copy named one that already existed, so the registered entry's own snapshot path (M32b's line) refused it either way. A configured snapshot that does not exist yet is registered as no entry, and `/snapshot.json` falls back to reading that path. Named at `/console/.html`, the copy this start is about to write, only the configured snapshot's own served root refuses it. The deferred-termination case now catches every interrupt and judges it, so a stop raised where it should have been held fails the case instead of ending the pytest session. Arc: neutral-product-standalone-operability Lane: openxfactory-4 (openXfactory-4-openDox_extraction) Co-Authored-By: Claude Opus 5.5 (1M context) --- tests/test_console_token_delivery.py | 61 ++++++++++++++++++++++++---- 1 file changed, 52 insertions(+), 9 deletions(-) diff --git a/tests/test_console_token_delivery.py b/tests/test_console_token_delivery.py index 17ed72d0..d0069aee 100644 --- a/tests/test_console_token_delivery.py +++ b/tests/test_console_token_delivery.py @@ -1889,19 +1889,33 @@ def test_deferred_termination_holds_a_stop_until_the_block_ends() -> None: in hand, and removal, already a stop, lets it go.""" from opendox import console_access + # Every interrupt is CAUGHT here and judged, so a stop raised where it + # should have been held fails this case instead of ending the session. + reached = [] with console_access.terminate_as_interrupt(True): assert signal.getsignal(signal.SIGTERM) not in (signal.SIG_DFL, signal.SIG_IGN) - with pytest.raises(KeyboardInterrupt): + try: with console_access.deferred_termination(): + try: + os.kill(os.getpid(), signal.SIGTERM) + signal.pthread_sigmask(signal.SIG_BLOCK, []) # deliver now + except KeyboardInterrupt: + pytest.fail("the stop was raised inside the block, not held") + reached.append("held") + except KeyboardInterrupt: + reached.append("raised once the block was done") + assert reached == ["held", "raised once the block was done"], reached + try: + with console_access.deferred_termination(raise_pending=False): os.kill(os.getpid(), signal.SIGTERM) - signal.pthread_sigmask(signal.SIG_BLOCK, []) # deliver now - reached = True # not raised here - assert reached - with console_access.deferred_termination(raise_pending=False): - os.kill(os.getpid(), signal.SIGTERM) - signal.pthread_sigmask(signal.SIG_BLOCK, []) - with console_access.deferred_termination(): - pass # nothing left over + signal.pthread_sigmask(signal.SIG_BLOCK, []) + except KeyboardInterrupt: + pytest.fail("a stop held during a removal was raised") + try: + with console_access.deferred_termination(): + pass # nothing left over + except KeyboardInterrupt: + pytest.fail("a stop that was let go came back") @pytest.mark.parametrize("entry", ["serve", "generate-and-open"]) @@ -2475,3 +2489,32 @@ def test_the_static_backstop_never_sends_a_copy_swapped_in_after_the_check( assert b"index" in _raw_get(base, "/index.html").lower() finally: _stop_plane(httpd, worker) + + +def test_a_snapshot_named_at_a_copy_not_yet_written_refuses_the_start( + tmp_path, monkeypatch, capsys, standalone_profile) -> None: + """The configured snapshot is a served root even when its file does not + exist yet, and so is registered as no entry: `/snapshot.json` falls back + to reading that path. Named at the copy this start is about to write, + `/console/.html`, it refuses the start by name, and no copy + is written.""" + from opendox import console_access, serve + + _clean_git(monkeypatch) + repo = _repository(tmp_path) + state = _state(tmp_path) + monkeypatch.setenv("OPENDOX_STATE_DIR", str(state)) + monkeypatch.setattr(serve, "real_notebook_adapter", lambda *a, **k: None) + + def served(self, *args, **kwargs): + raise AssertionError("the server served") + + monkeypatch.setattr(socketserver.BaseServer, "serve_forever", served) + port = _free_port() + future = console_access.private_copy_path(state, port) + assert serve.main(["--snapshot", str(future), "--checkout-root", str(repo), + "--port", str(port)]) == 1 + err = capsys.readouterr().err + assert "serve refused:" in err and "OPENDOX_STATE_DIR" in err, err + assert not future.exists() + assert _port_is_free(port) From 45958bf3e9a358fd143305d39af07cff364a8513 Mon Sep 17 00:00:00 2001 From: Brett Heap <1513478+brettheap@users.noreply.github.com> Date: Sun, 4 Oct 2026 16:18:23 +0000 Subject: [PATCH 80/88] T104 fix round 9: every standalone plane keeps the boundary; identity judges a second spelling; a stale copy is swept (adversarial review) The holder's adversarial review of fb8a1cc4 and of round 8 (73df8bac), B1-B10, folded in with Copilot's two fb8a1cc4 findings (round 8). B3 waits on Brett's ruling; B7 is the body's correction only (holder's ruling). B1 (high). A standalone plane that minted no token (no git identity) wrote no copy, so it asked no boundary and marked no private root, and a root of its that held the shared state directory served a sibling plane's copy, token and all. The delivery is now the plane's, token or not (`serve. build_server`), and `publish` on every standalone plane asks the boundary and marks the copies' directory private (`guard_private_roots`); only the writing still needs a token. B2. A platform without the POSIX primitives (Windows) ended the start in an AttributeError traceback. `unsupported_platform()` names the gap, and the writer and the reader refuse by name first, as the bundle's own does (holder's ruling). B4. The served-root overlap and the static handler's guard compared spellings. On a case-insensitive filesystem a second spelling of the state directory, or of `console/`, passed both. Both now also compare the directories' `(st_dev, st_ino)` (`_identity`, `within_private_roots`), the copies' directory's own included. B5. A second stop that landed after the first had unwound the serve loop and before the cleanup's hold escaped the `finally` and left the copy. The first stop is latched; every later one is only recorded. B6. Three rules no case pinned: the walk's link-owner check, the reader's re-judging of the tree, and the fchmod under a umask that strips owner write. The reviewer's three cases are taken as written. B8. `--no-serve` opened a copy for a server it then closed. It publishes, opens and prints no copy now; the cases that read a copy serve once and stop at a Ctrl-C (`stopped_once_serving`) instead. B9. A SIGKILLed serve's copy stayed until a later serve took its port. A publication sweeps, under the console directory's lock, every copy whose reservation is free and every temporary or taken name a dead writer or remover left, and nothing that is not this user's own 0600 file named so. Red first: the 15 new cases of section 17 that pin B1, B2, B4, B5, B8 and B9 fail at the merged pre-fix tree 0b664557 (review9-red.txt). Arc: neutral-product-standalone-operability Lane: openxfactory-4 (openXfactory-4-openDox_extraction) Co-Authored-By: Claude Opus 5.5 (1M context) --- src/opendox/cli.py | 20 +- src/opendox/console_access.py | 276 +++++++++++-- src/opendox/serve.py | 33 +- tests/test_console_token_delivery.py | 595 ++++++++++++++++++++++++++- 4 files changed, 865 insertions(+), 59 deletions(-) diff --git a/src/opendox/cli.py b/src/opendox/cli.py index 9a40e2b3..fa7a8380 100644 --- a/src/opendox/cli.py +++ b/src/opendox/cli.py @@ -905,9 +905,10 @@ def _generate_and_serve(args: argparse.Namespace, run_dir: Path, *, # to `webbrowser.open` sits on a command line every user can read # (`/proc//cmdline`). The path is printed with or without # `--no-open`, and the token never is: opening that file again re-opens - # the page. `None` on a host's plane and where no token was minted, and - # then nothing changes. A copy that cannot be written safely refuses the - # run before it serves. + # the page. `None` on a host's plane, where no token was minted (a + # standalone plane still keeps every console's copy unserved then, + # `console_access.guard_private_roots`), and under `--no-serve`. A copy + # that cannot be written safely refuses the run before it serves. # # A plain `kill`, or a closed terminal, stops a standalone console the way # Ctrl-C does (`terminate_as_interrupt`), from BEFORE the copy is written @@ -915,12 +916,21 @@ def _generate_and_serve(args: argparse.Namespace, run_dir: Path, *, # stop that arrives while the copy is being written or removed is held # until that is done (`deferred_termination`), so no copy is ever left # half handled. A plane that writes no copy keeps the signals' defaults. + # + # `--no-serve` SERVES NOTHING, SO IT PUBLISHES NOTHING (adversarial review + # of openDox-code#84, B8). It closes the server as soon as it has printed + # the URL, so a copy written for it opened a console page nothing + # answered, and was deleted as the run returned. No copy is written, none + # is opened, and no console line is printed. console = None - with console_access.terminate_as_interrupt(console_access.needs_copy(httpd)): + serving = not args.no_serve + with console_access.terminate_as_interrupt( + serving and console_access.needs_copy(httpd)): try: try: with console_access.deferred_termination(): - console = console_access.publish(httpd, page_url=url) + if serving: + console = console_access.publish(httpd, page_url=url) print(f" serving {url}") print(f" snapshot {serve_mod.server_url(httpd, '/snapshot.json')}") if console is not None: diff --git a/src/opendox/console_access.py b/src/opendox/console_access.py index 0b154dcd..febe89e1 100644 --- a/src/opendox/console_access.py +++ b/src/opendox/console_access.py @@ -58,17 +58,29 @@ descriptor, opened without blocking: a regular file, this user's, exactly 0600, one link; * the state directory and every root the plane serves may not overlap in - either direction, by name and before any write, as T100's - served-repository boundary refuses its own (holder's rulings on - openxFactory#1220's review, Copilot `r4171166321`, and on batch N's, - `r4174345203`); + either direction, before any write, as T100's served-repository boundary + refuses its own (holder's rulings on openxFactory#1220's review, Copilot + `r4171166321`, and on batch N's, `r4174345203`), judged by name AND by + the directories' own identities, so a second spelling of one directory + (a case-insensitive filesystem's) is the same directory; + * EVERY standalone plane keeps that boundary and never serves a copy, the + planes that minted no token included (`publish`): a sibling plane of the + same user shares the state directory, and serves what another plane + wrote there unless it refuses it too; + * a publication sweeps the copies their servers left when they died + (`_sweep_stale_copies`): a copy whose reservation is free is no running + console's; * every refusal names its path, an operating-system one included, so an entry point refuses its start by name; and the copy is removed when the server stops, by Ctrl-C, SIGTERM or SIGHUP, or when its start is refused after it was written. A stop is read as Ctrl-C from before the copy is written to after it is removed, and held while a copy is being written or - removed (`deferred_termination`), and every writer and remover of - `console/` takes the directory's lock (`_lock`). + removed (`deferred_termination`); the first stop is the only one raised + (`_terminate_as_interrupt`), and every writer and remover of `console/` + takes the directory's lock (`_lock`); + * a platform without the POSIX primitives these rules rest on is named and + refused before anything is written or read (`unsupported_platform`), as + the bundle refuses its own. #1144 12.4a, as T007 batch N amends it (openxFactory#1222), is the normative text this module realizes. @@ -86,6 +98,7 @@ import re import signal import stat +import sys import urllib.parse from collections.abc import Iterable, Mapping from pathlib import Path @@ -102,10 +115,11 @@ "CONSOLE_DIRNAME", "ConsoleAccessRefused", "ConsoleTerminated", "DELIVERY_CAPABILITIES", "DELIVERY_OPENED_URL", "FRAGMENT_KEY", "PrivateCopy", "RECORD_ELEMENT_ID", - "RECORD_KIND", "deferred_termination", "delivery_for", "is_private_file", - "needs_copy", "opens_a_private_file", + "RECORD_KIND", "deferred_termination", "delivery_for", "guard_private_roots", + "is_private_file", "needs_copy", "opens_a_private_file", "opened_url", "private_copy_path", "publish", "read_private_copy", - "remove_private_copy", "terminate_as_interrupt", "write_private_copy", + "remove_private_copy", "terminate_as_interrupt", "unsupported_platform", + "within_private_roots", "write_private_copy", ] #: The token rides on `/capabilities`, as a host's plane has always read it. @@ -136,6 +150,54 @@ r'') _LOOPBACK_HOSTS = frozenset({"127.0.0.1", "::1", "localhost"}) +#: The names a publication may sweep when their servers are gone +#: (`_sweep_stale_copies`): a copy, a writer's temporary file, and a remover's +#: taken name. Each holds a token, and nothing else is ever touched. +_SWEEPABLE = re.compile(r"[0-9]+\.html|\.[0-9]+\.html\.opendox-[0-9]+" + r"|\.[0-9]+\.html\.removing-[0-9]+-[0-9a-f]{12}") + +#: Whether every call the copy's rules make relative to a directory's +#: descriptor takes one here, read ONCE at import, as `opendox.runtime.bundle` +#: reads its own: a case that stands a wrapper in for one of them must not +#: read as another platform. +_DIR_FD_CALLS = all(call in os.supports_dir_fd for call in ( + os.open, os.mkdir, os.stat, os.rename, os.unlink, os.link)) + + +def unsupported_platform() -> str | None: + """Why this platform cannot keep a console token's private copy, or `None`. + + The copy is a POSIX design, as openDox-code#69's bundle is, and every one + of its rules rests on a POSIX primitive: its directories and the file are + judged by owner (`os.getuid`), made and opened without following a link + (`O_DIRECTORY`, `O_NOFOLLOW`, calls relative to a directory's + descriptor), set to 0600 by descriptor (`fchmod`), and read without + blocking (`O_NONBLOCK`). Without them (Windows) the standalone start + ended in an `AttributeError` traceback (adversarial review of + openDox-code#84, B2). So the writer and the reader name the gap first, + and refuse, as `opendox.runtime.bundle.unsupported_platform` names its + own (holder's ruling).""" + missing = [name for name, present in ( + ("os.getuid", hasattr(os, "getuid")), + ("os.O_DIRECTORY", hasattr(os, "O_DIRECTORY")), + ("os.O_NOFOLLOW", hasattr(os, "O_NOFOLLOW")), + ("os.O_NONBLOCK", hasattr(os, "O_NONBLOCK")), + ("os.fchmod", hasattr(os, "fchmod")), + ("calls relative to a directory's descriptor", _DIR_FD_CALLS), + ) if not present] + if not missing: + return None + return (f"the console token's private copy needs a POSIX platform, and " + f"this one ({sys.platform}) lacks {', '.join(missing)}: the copy " + "and its directories are judged by owner and made without " + "following a link, so a standalone console cannot hand its token " + "to this user alone here, and is not started") + + +def _refuse_an_unsupported_platform() -> None: + reason = unsupported_platform() + if reason is not None: + raise ConsoleAccessRefused(reason) class ConsoleAccessRefused(Exception): @@ -523,9 +585,24 @@ def _opener_html(record: Mapping[str, Any]) -> str: "\n") +def _identity(path: Path | str) -> tuple[int, int] | None: + """`(st_dev, st_ino)` of what `path` names, or `None` where nothing + there can be asked.""" + try: + info = os.stat(path) + except (OSError, ValueError): + return None + return (info.st_dev, info.st_ino) + + +def _identities_above(path: Path) -> set[tuple[int, int]]: + """The identities of every directory above `path` that exists.""" + return {key for key in map(_identity, Path(path).parents) if key is not None} + + def _refuse_a_served_state_dir(state_dir: Path, served_roots: Iterable[Path | str], *, - port: int) -> None: + port: int | None) -> None: """The state directory and every root this plane serves may not overlap, in EITHER direction, or the copy is refused before anything is written. @@ -540,19 +617,33 @@ def _refuse_a_served_state_dir(state_dir: Path, it, served as a root, would serve the copy (Copilot at openDox-code#84, `r4173889265`, found the first of these). Asked of the RESOLVED paths, so a link counts as where it leads. `port` names the copy the refusal is - about.""" + about, and `None`, on a plane that writes none, names every console's. + + AND OF THE DIRECTORIES' OWN IDENTITIES (adversarial review of + openDox-code#84, B4). On a case-insensitive filesystem (macOS's default) + `/STATE` and `/state` are one directory, and resolving a + path keeps the case it was given, so names alone let the second + spelling of the state directory, or of a root, through. Every directory + that exists on either path is also compared by `(st_dev, st_ino)` + (`_identity`): the same directory is the same, however it is spelled.""" resolved = Path(state_dir).resolve() - copy = private_copy_path(resolved, port) + copy = (f"the copy {private_copy_path(resolved, port)}" if port is not None + else f"every console's private copy in {resolved / CONSOLE_DIRNAME}") + state_is = _identity(resolved) + above_state = _identities_above(resolved) for root in served_roots: served = Path(root).resolve() - if resolved == served: + served_is = _identity(served) + if resolved == served or (state_is is not None and state_is == served_is): where = f"is the served repository ({served})" - elif served in resolved.parents: + elif served in resolved.parents or (served_is is not None + and served_is in above_state): where = f"lies inside the served repository ({served})" - elif resolved in served.parents: + elif resolved in served.parents or (state_is is not None + and state_is in _identities_above(served)): where = (f"holds {served}, a root this plane serves, so the plane " - f"would serve what the state directory keeps, the copy " - f"{copy} among it") + f"would serve what the state directory keeps, {copy} " + "among it") else: continue raise ConsoleAccessRefused( @@ -593,7 +684,11 @@ def write_private_copy(state_dir: Path | str, *, page_url: str, port: int, THE WALK IS INSIDE THE CONVERSION TOO (Copilot at openDox-code#84, r4177975898): an overlong component (ENAMETOOLONG) or an unsearchable - parent (EACCES) on the way is a refusal by name, like any other.""" + parent (EACCES) on the way is a refusal by name, like any other. + + A PLATFORM WITHOUT THE POSIX PRIMITIVES is refused first, by name + (`unsupported_platform`).""" + _refuse_an_unsupported_platform() target = private_copy_path(state_dir, port) try: state = _walked(state_dir) @@ -643,6 +738,7 @@ def _write_the_copy(state: Path, target: Path, if reason is not None: raise _unsafe(target.parent, reason) _lock(directory) # until the copy is in place (`_lock`) + _sweep_stale_copies(directory, spare=target.name) try: present = os.stat(target.name, dir_fd=directory, follow_symlinks=False) @@ -735,6 +831,47 @@ def _refuse_a_running_consoles_copy(target: Path, directory: int, os.close(held) +def _sweep_stale_copies(directory: int, *, spare: str) -> None: + """Remove every copy in `console/` whose console is gone, `spare` aside. + + A SERVER THAT DIED LEFT ITS COPY (adversarial review of openDox-code#84, + B9). A SIGKILL, an out-of-memory kill or a power cut runs no cleanup, so + its copy, a token in it, stayed until a later serve happened to take the + same port. A publication now sweeps them. It runs with the console + directory's lock held (`_lock`), so no publication or removal is under + way: a copy whose lock is free belongs to no running console + (`_Reservation`), and a writer's temporary file or a remover's taken name + found then belongs to a process that died mid-way. Only this user's own + regular files of mode 0600 with one link, named as those are named + (`_SWEEPABLE`), are swept, each only while its name is still the file + whose lock was taken. A lock still held, or a filesystem that keeps no + locks, tells nothing, and the file stays. `spare` is the name this + publication judges itself (`_refuse_a_running_consoles_copy`).""" + if fcntl is None: + return + uid = os.getuid() + try: + names = os.listdir(directory) + except OSError: + return + for name in names: + if name == spare or not _SWEEPABLE.fullmatch(name): + continue + with contextlib.suppress(OSError): + handle = os.open(name, os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK, + dir_fd=directory) + try: + info = os.fstat(handle) + if _file_unsafe_because(info, uid=uid) is not None: + continue + fcntl.flock(handle, fcntl.LOCK_EX | fcntl.LOCK_NB) # held: stays + still = os.stat(name, dir_fd=directory, follow_symlinks=False) + if (still.st_dev, still.st_ino) == (info.st_dev, info.st_ino): + os.unlink(name, dir_fd=directory) + finally: + os.close(handle) + + def _writer_of(handle: int) -> str: """`" (pid N)"` from the copy's own record, or nothing.""" with contextlib.suppress(OSError, ValueError, AttributeError): @@ -749,7 +886,10 @@ def _writer_of(handle: int) -> str: def read_private_copy(path: Path | str) -> dict: """The record in the copy at `path`, or a refusal naming why: an operating-system error on the way (an overlong component, an unsearchable - parent) included (Copilot at openDox-code#84, r4177975898).""" + parent) included (Copilot at openDox-code#84, r4177975898). A platform + without the POSIX primitives is refused first, by name + (`unsupported_platform`).""" + _refuse_an_unsupported_platform() try: return _read_the_copy(path) except OSError as exc: @@ -913,16 +1053,27 @@ class ConsoleTerminated(KeyboardInterrupt): already stop on.""" -#: Whether a stop is being HELD (`deferred_termination`), and the one that -#: arrived meanwhile. Python runs signal handlers in the main thread only, as -#: the entry points publish and remove there, so plain module state serves. -_held = {"depth": 0, "pending": None} +#: Whether a stop is being HELD (`deferred_termination`), the one that +#: arrived meanwhile, and whether a stop has already been RAISED. Python runs +#: signal handlers in the main thread only, as the entry points publish and +#: remove there, so plain module state serves. +_held = {"depth": 0, "pending": None, "stopping": False} def _terminate_as_interrupt(signum, frame): - if _held["depth"]: + """A stop, raised once. + + THE FIRST STOP IS THE ONLY ONE RAISED (adversarial review of + openDox-code#84, B5). A double Ctrl-C, or a SIGTERM and then the SIGHUP + of a closing terminal, could land its second signal after the first had + unwound the serve loop and before the cleanup's `deferred_termination` + held anything, and that second interrupt escaped the `finally` and left + the copy. Once one stop is raised, every later one is only recorded, and + the cleanup runs to its end.""" + if _held["depth"] or _held["stopping"]: _held["pending"] = signum return + _held["stopping"] = True raise ConsoleTerminated @@ -944,7 +1095,8 @@ def deferred_termination(*, raise_pending: bool = True): _held["depth"] -= 1 if not _held["depth"]: pending, _held["pending"] = _held["pending"], None - if pending is not None and raise_pending: + if pending is not None and raise_pending and not _held["stopping"]: + _held["stopping"] = True # raised once (`_terminate_as_interrupt`) raise ConsoleTerminated @@ -989,11 +1141,13 @@ def terminate_as_interrupt(enabled: bool): signal.signal(signum, handler) yield return + _held.update(pending=None, stopping=False) try: yield finally: for signum, handler in previous.items(): signal.signal(signum, handler) + _held.update(pending=None, stopping=False) def is_private_file(handle: int, private_roots: Iterable[Path | str]) -> bool: @@ -1023,6 +1177,29 @@ def is_private_file(handle: int, private_roots: Iterable[Path | str]) -> bool: return False +def within_private_roots(path: Path | str, + private_roots: Iterable[Path | str]) -> bool: + """Whether `path`, where it leads, IS a private-copy directory or lies + inside one: by name, and by the identity of every directory on its way. + + A case-insensitive filesystem (macOS's default) has more than one + spelling for each directory, and resolving a path keeps the case it was + given, so `/state-alias/CONSOLE/` named the copies' directory under + a name no private root spells (adversarial review of openDox-code#84, + B4). So the resolved path is also judged by `(st_dev, st_ino)`: where it, + or any directory above it, is a private root by identity, it is that + root, however it is spelled. A private root that does not exist yet is + judged by name alone, as nothing can lie inside it.""" + target = Path(path).resolve() + roots = [Path(root) for root in private_roots] + if any(target == root or root in target.parents for root in roots): + return True + marked = {key for key in map(_identity, roots) if key is not None} + if not marked: + return False + return bool(marked & ({_identity(target)} | _identities_above(target))) + + def opens_a_private_file(path: Path | str, private_roots: Iterable[Path | str]) -> bool: """Whether what opens at `path` is a file in a private-copy directory @@ -1043,28 +1220,56 @@ def needs_copy(httpd: Any) -> bool: """Whether the plane `httpd` delivers its console token through a private copy: a standalone plane that minted one. The entry points ask it BEFORE `publish`, to read a stop as Ctrl-C from before the copy exists.""" - return bool(getattr(httpd, "console_token", None)) and getattr( - httpd, "console_token_delivery", None) == DELIVERY_OPENED_URL + return bool(getattr(httpd, "console_token", None)) and _standalone(httpd) + + +def _standalone(httpd: Any) -> bool: + return getattr(httpd, "console_token_delivery", None) == DELIVERY_OPENED_URL + + +def guard_private_roots(httpd: Any, state_dir: Path | str) -> None: + """Keep a standalone plane that WRITES NO COPY from serving another's. + + A SIBLING PLANE SERVED ANOTHER PLANE'S COPY (adversarial review of + openDox-code#84, B1). Two standalone planes of one user share the state + directory. One that minted no token (no git identity, so no session + verbs) wrote no copy, so it asked no boundary and marked no private + root, and a root it served that held the state directory served the + other plane's copy, token and all, to any local caller. The boundary is + the PLANE's, not the copy's: this one is asked of the state directory as + `write_private_copy` asks it, by name and by identity, and refuses the + start by name, and the copies' directory is marked private + (`httpd.private_roots`), so the static handler, `/source` and + `/snapshot.json` refuse every console's copy here too.""" + _refuse_a_served_state_dir(state_dir, tuple(getattr(httpd, "served_roots", ())), + port=None) + httpd.private_roots = (Path(state_dir).resolve() / CONSOLE_DIRNAME,) def publish(httpd: Any, *, page_url: str, env: Mapping[str, str] | None = None) -> PrivateCopy | None: - """What an ENTRY POINT does after `serve.build_server`: on a standalone + """What an ENTRY POINT does after `serve.build_server`. On a standalone plane that minted a token, write the private copy into the install's - state directory and return it; otherwise `None`, and nothing is written. + state directory and return it. On a standalone plane that minted none, + write nothing, and still keep the boundary and mark every console's copy + private (`guard_private_roots`), so the DELIVERY'S RULES do not depend on + the token. Otherwise, a host's plane, `None`, and nothing changes. A state directory that cannot be named, or a tree that is not this user's alone, refuses (`ConsoleAccessRefused`), and the entry point refuses with it: a console nobody can open is not served as if it could be.""" - if not needs_copy(httpd): + if not _standalone(httpd): return None - token = httpd.console_token try: state = runtime_config.state_dir(env) except runtime_config.ConfigurationError as exc: raise ConsoleAccessRefused( - f"the console token's private copy has no state directory: {exc}" - ) from None + "the console tokens' private copies have no state directory, so " + f"this standalone plane cannot keep them unserved: {exc}") from None + if not needs_copy(httpd): + guard_private_roots(httpd, state) + return None + token = httpd.console_token copy = write_private_copy(state, page_url=page_url, port=int(httpd.server_address[1]), token=token, served_roots=getattr(httpd, "served_roots", ())) @@ -1072,7 +1277,8 @@ def publish(httpd: Any, *, page_url: str, # r4173889294). It follows links inside `--web-dir` (a governed host's # composed web root is made of them), so a link out of the bundle into the # state directory would reach the copies. The handler refuses every static - # request whose resolved target is this directory or lies inside it - # (`serve.DashboardHandler.send_head`), every port's copy included. + # request whose resolved target is this directory or lies inside it, by + # name or by identity (`within_private_roots`), every port's copy included + # (`serve.DashboardHandler.send_head`). httpd.private_roots = (copy.path.parent.resolve(),) return copy diff --git a/src/opendox/serve.py b/src/opendox/serve.py index 50a3279a..74137e0e 100644 --- a/src/opendox/serve.py +++ b/src/opendox/serve.py @@ -822,8 +822,7 @@ def read_unless_private(path: Path | str, private_roots) -> bytes | None: resolved again on every request, and could be re-pointed at the state directory after the copy was published; a hard link reaches the copy by another name. The file actually opened is judged, so neither is served. - With no private root (a host's plane, or a plane that wrote no copy) it - reads as before.""" + With no private root (a host's plane) it reads as before.""" with open(path, "rb") as stream: if private_roots and console_access.is_private_file( stream.fileno(), private_roots): @@ -1362,9 +1361,16 @@ def send_head(self): serve the token's copy (plan 034 T104; Copilot at openDox-code#84, r4173889294): a target whose RESOLVED path is a directory the entry point marked private (`console_access.publish` sets - `private_roots` on the server), or lies inside one, is a 404, for GET - and HEAD, files and listings alike. A server with no copy marks - nothing, and serves exactly as before. + `private_roots` on the server, on every standalone plane, whether it + wrote a copy or not), or lies inside one, is a 404, for GET and HEAD, + files and listings alike. A host's plane marks nothing, and serves + exactly as before. + + BY NAME AND BY IDENTITY (adversarial review of openDox-code#84, B4): + a case-insensitive filesystem spells the copies' directory more than + one way, so where the resolved target, or a directory above it, is a + private root by `(st_dev, st_ino)`, it is that root + (`console_access.within_private_roots`). A DIRECTORY REQUEST IS JUDGED BY WHAT IT SERVES (Copilot at openDox-code#84, r4174674625). For `/sub/` the stdlib handler serves @@ -1391,9 +1397,7 @@ def send_head(self): judged.append(path / name) break for candidate in judged: - target = candidate.resolve() - if (any(target == root or root in target.parents - for root in private) + if (console_access.within_private_roots(candidate, private) or console_access.opens_a_private_file(candidate, private)): self.send_error(404, "File not found") return None @@ -2352,10 +2356,15 @@ def build_server( # entry point writes it into a 0600 private copy instead and opens the page # with it in the URL's fragment (`console_access.publish`). The routes that # require it require it exactly as before; only the delivery differs. + # + # THE DELIVERY IS THE PLANE'S, TOKEN OR NOT (adversarial review of + # openDox-code#84, B1): a standalone plane that minted none still keeps + # the state directory's boundary and never serves another plane's copy + # (`console_access.guard_private_roots`), so it is named a standalone + # plane's whether or not a token was minted. console_token = (mint_console_token() if capabilities["actions"]["session"] else None) - console_delivery = (console_access.delivery_for(domain_profile.current()) - if console_token else None) + console_delivery = console_access.delivery_for(domain_profile.current()) if console_token and console_delivery == console_access.DELIVERY_CAPABILITIES: capabilities[CONSOLE_TOKEN_FIELD] = console_token # THE ONE REPOSITORY THIS SERVE CAN WRITE TO. A plane reaching several @@ -2538,8 +2547,8 @@ def build_server( factory = functools.partial(bound, directory=str(web_dir)) httpd = _server_class_for(host)((host, port), factory) # FOR THE ENTRY POINT, which delivers the token where `/capabilities` does - # not (`console_access.publish`): the token, and which delivery this plane - # uses. Both `None` where no token was minted. + # not (`console_access.publish`): the token (`None` where none was + # minted), and which delivery this plane uses, whether or not it was. httpd.console_token = console_token httpd.console_token_delivery = console_delivery # ...and EVERY root this plane serves files from, which the token's copy diff --git a/tests/test_console_token_delivery.py b/tests/test_console_token_delivery.py index d0069aee..1f35aa25 100644 --- a/tests/test_console_token_delivery.py +++ b/tests/test_console_token_delivery.py @@ -364,15 +364,34 @@ def test_the_record_cannot_break_out_of_its_script_element(tmp_path) -> None: def _generate_and_open_args(tmp_path: Path, repo: Path, *extra: str) -> argparse.Namespace: + """A run that SERVES: a `--no-serve` run publishes no copy (adversarial + review of openDox-code#84, B8), so every case built from these takes + `stopped_once_serving` too, and never blocks in a serve loop.""" from opendox import cli return cli.build_parser().parse_args([ "generate-and-open", "--repo-root", str(repo), "--repository", "fixture", "--run-dir", str(tmp_path / "run"), "--port", "0", "--no-validate", - "--no-serve", *extra]) + *extra]) + + +@pytest.fixture() +def stopped_once_serving(monkeypatch): + """The serve loop, stopped as Ctrl-C stops it the moment it starts: what + it lists is every server that reached it, so a refused start can show it + never served.""" + reached: list[object] = [] + + def serve_forever(self, *args, **kwargs): + reached.append(self) + raise KeyboardInterrupt + + monkeypatch.setattr(socketserver.BaseServer, "serve_forever", serve_forever) + return reached def test_generate_and_open_hands_the_browser_a_file_and_prints_no_token( - tmp_path, monkeypatch, capsys, standalone_profile) -> None: + tmp_path, monkeypatch, capsys, standalone_profile, + stopped_once_serving) -> None: """What the entry point gives the browser is the copy's `file://` path: a URL handed to `webbrowser.open` sits on a command line every user can read. The copy it names opens the page with the token in the fragment. @@ -403,10 +422,12 @@ def opener(url): assert record["page_url"] in out.splitlines(), out assert not list((state / console_access.CONSOLE_DIRNAME).iterdir()), \ "the copy outlived the run" + assert len(stopped_once_serving) == 1, "the run never served" def test_no_open_prints_the_copy_path_and_opens_nothing( - tmp_path, monkeypatch, capsys, standalone_profile) -> None: + tmp_path, monkeypatch, capsys, standalone_profile, + stopped_once_serving) -> None: """`--no-open`: no browser, and the way back to the page is still printed, the copy's path, while the token is still never printed.""" from opendox import cli @@ -424,10 +445,12 @@ def test_no_open_prints_the_copy_path_and_opens_nothing( if _CONSOLE.match(line)) assert match.group(1).startswith("file://"), out assert "console_token=" not in out, out + assert len(stopped_once_serving) == 1, "the run never served" def test_generate_and_open_refuses_where_no_safe_copy_can_be_written( - tmp_path, monkeypatch, capsys, standalone_profile) -> None: + tmp_path, monkeypatch, capsys, standalone_profile, + stopped_once_serving) -> None: """A state directory another user could change refuses the run before it serves, naming the directory; a console nobody can safely be handed is not served as if it could be.""" @@ -444,6 +467,7 @@ def test_generate_and_open_refuses_where_no_safe_copy_can_be_written( captured = capsys.readouterr() assert opened == [] assert "generate-and-open refused:" in captured.err + assert stopped_once_serving == [], "the refused run served" assert str(state) in captured.err and "writable by its group" in captured.err @@ -785,7 +809,8 @@ def test_a_state_directory_reached_through_a_link_into_the_served_root_is_refuse def test_generate_and_open_refuses_a_state_directory_inside_the_served_repository( - tmp_path, monkeypatch, capsys, standalone_profile) -> None: + tmp_path, monkeypatch, capsys, standalone_profile, + stopped_once_serving) -> None: """Through the entry point: `OPENDOX_STATE_DIR` inside the repository it serves refuses the run, naming the setting, and the repository gains no file.""" @@ -801,6 +826,7 @@ def test_generate_and_open_refuses_a_state_directory_inside_the_served_repositor err = capsys.readouterr().err assert opened == [] assert "generate-and-open refused:" in err and "OPENDOX_STATE_DIR" in err + assert stopped_once_serving == [], "the refused run served" assert "lies inside the served repository" in err, err assert _tree(repo) == before @@ -1342,7 +1368,8 @@ def foreign(path, *args, **kwargs): @pytest.mark.parametrize("kind", sorted(_PLANTED)) def test_generate_and_open_refuses_by_name_what_was_planted_at_the_copy( - tmp_path, monkeypatch, capsys, standalone_profile, kind) -> None: + tmp_path, monkeypatch, capsys, standalone_profile, + stopped_once_serving, kind) -> None: """12.4a, through the entry point: anything at the copy's path but an earlier copy of this user's refuses the START by name. Nothing is printed that serves, no browser is opened, the planted thing is untouched, and the @@ -1363,6 +1390,7 @@ def test_generate_and_open_refuses_by_name_what_was_planted_at_the_copy( out, err = capsys.readouterr() assert opened == [] assert "generate-and-open refused:" in err, err + assert stopped_once_serving == [], "the refused run served" assert str(planted) in err and _PLANTED[kind] in err, err assert " console " not in out and f":{port}/" not in out, out assert _fingerprint(planted) == before @@ -1479,7 +1507,8 @@ def test_a_console_directory_that_is_not_0700_is_refused(tmp_path, mode) -> None @pytest.mark.skipif(os.geteuid() == 0, reason="root can write any directory") def test_a_copy_that_cannot_be_written_refuses_the_start_by_name( - tmp_path, monkeypatch, capsys, standalone_profile) -> None: + tmp_path, monkeypatch, capsys, standalone_profile, + stopped_once_serving) -> None: """The lifecycle self-pass: a state directory its parent will not let this user make (an operating-system refusal, not a rule of this module) used to escape as a raw `PermissionError`, a traceback and no refusal. @@ -1504,6 +1533,7 @@ def test_a_copy_that_cannot_be_written_refuses_the_start_by_name( err = capsys.readouterr().err assert opened == [] assert "generate-and-open refused:" in err and "cannot be written" in err, err + assert stopped_once_serving == [], "the refused run served" assert not state.exists() finally: locked.chmod(0o700) @@ -2518,3 +2548,554 @@ def served(self, *args, **kwargs): assert "serve refused:" in err and "OPENDOX_STATE_DIR" in err, err assert not future.exists() assert _port_is_free(port) + + +# --------------------------------------------------------------------------- +# 17 — the holder's adversarial review of fb8a1cc4 and 73df8bac +# (openDox-code#84; B1, B2, B4, B5, B6, B8, B9). The cases named +# `test_b*` are the reviewer's own, kept as they were written. +# --------------------------------------------------------------------------- + +def _adv_env(state: Path | None = None) -> dict: + env = {k: v for k, v in os.environ.items() + if not k.startswith(("GIT_", "XF_", "OPENDOX_"))} + env.update(GIT_CONFIG_GLOBAL=os.devnull, GIT_CONFIG_SYSTEM=os.devnull, + LANG="C.UTF-8", PYTHONUNBUFFERED="1", PYTHONPATH=str(ROOT / "src")) + if state is not None: + env["OPENDOX_STATE_DIR"] = str(state) + return env + + +def _adv_repo(where: Path, *, identity: bool = True) -> Path: + """A checkout; with no `identity`, its plane resolves no actor, so it has + no session verbs and mints no token.""" + import subprocess + + def run(*args: str) -> None: + subprocess.run(["git", *args], cwd=where, env=_adv_env(), check=True, + capture_output=True) + + where.mkdir(parents=True, exist_ok=True) + run("init", "-q", "-b", "main") + if identity: + run("config", "user.name", "fixture") + run("config", "user.email", "fixture@example.invalid") + (where / "README.md").write_text("# fixture\n") + run("add", ".") + run("-c", "user.name=x", "-c", "user.email=x@example.invalid", + "commit", "-qm", "init") + return where + + +def _adv_serve(tmp: Path, repo: Path, state: Path, port: int, name: str, + code: str | None = None): + """`python -m opendox.serve` as a user starts it, or `code` and then + `serve.main`, waited for until it serves or ends.""" + import subprocess + import sys + import time + + snapshot = tmp / f"{name}.json" + snapshot.write_text(json.dumps({"generation": {}})) + out = tmp / f"{name}.out" + argv = ["--snapshot", str(snapshot), "--checkout-root", str(repo), + "--port", str(port)] + if code is None: + cmd = [sys.executable, "-m", "opendox.serve", *argv] + else: + cmd = [sys.executable, "-c", code + f"\nsys.exit(serve.main({argv!r}))"] + proc = subprocess.Popen(cmd, cwd=tmp, env=_adv_env(state), + stdout=out.open("w"), stderr=subprocess.STDOUT, + preexec_fn=_default_stops) + for _ in range(300): + if "serving ideation dashboard" in out.read_text() or proc.poll() is not None: + break + time.sleep(0.1) + return proc, out + + +def _default_stops() -> None: + """A child started from a background job inherits SIGINT ignored, and + one under `nohup` SIGHUP: give it a terminal's, so its stops are read.""" + signal.signal(signal.SIGINT, signal.default_int_handler) + signal.signal(signal.SIGHUP, signal.SIG_DFL) + + +def _adv_stop(proc) -> int: + if proc.poll() is None: + proc.send_signal(signal.SIGTERM) + return proc.wait(30) + + +def test_b1_a_tokenless_sibling_plane_never_serves_another_planes_copy( + tmp_path) -> None: + """B1, as the reviewer staged it, with two real planes. Plane A (a git + identity, so a token) publishes into the shared state directory. Plane B + (no identity, so no token) serves a checkout that HOLDS that directory. + B used to ask no boundary and mark no private root, so its `/source` + served A's copy, token and all. It is the same plane's boundary now, + token or not: B refuses its start by name, and serves nothing.""" + from opendox import console_access + + repo_a = _adv_repo(tmp_path / "a") + outer = _adv_repo(tmp_path / "outer", identity=False) + state = _state(outer) + port_a, port_b = _free_port(), _free_port() + a, _out_a = _adv_serve(tmp_path, repo_a, state, port_a, "a") + try: + assert a.poll() is None, _out_a.read_text() + token = console_access.read_private_copy( + console_access.private_copy_path(state, port_a))["console_token"] + b, out_b = _adv_serve(tmp_path, outer, state, port_b, "b") + rc = b.wait(60) + text = out_b.read_text() + assert rc == 1 and "serve refused:" in text, text + assert "OPENDOX_STATE_DIR" in text and str(outer.resolve()) in text, text + assert "Traceback" not in text and token not in text, text + assert _port_is_free(port_b), "the refused plane kept its socket" + assert a.poll() is None, "plane A went down with B's refusal" + finally: + _adv_stop(a) + + +def test_a_tokenless_standalone_plane_keeps_the_boundary( + tmp_path, monkeypatch, standalone_profile) -> None: + """B1 in the process: a standalone plane that minted no token is still a + standalone plane (its delivery does not depend on the token), and its + publication still asks the boundary, refusing by name a state directory + inside its checkout. It writes nothing, and the sibling's copy there is + left as it was.""" + from opendox import console_access, serve + + _clean_git(monkeypatch) + outer = fresh_repository(PLAIN, tmp_path / "b") + state = _state(outer) + other = _write(state, port=9) # a sibling plane's copy + before = (other.path.read_bytes(), _fingerprint(other.path)) + snapshot = tmp_path / "snapshot.json" + snapshot.write_text(json.dumps({"generation": {}}), encoding="utf-8") + httpd = serve.build_server(WEB, snapshot, outer, port=0, quiet=True) + try: + assert httpd.console_token is None, "the case is vacuous: a token was minted" + assert httpd.console_token_delivery == console_access.DELIVERY_OPENED_URL + assert not console_access.needs_copy(httpd) + with pytest.raises(console_access.ConsoleAccessRefused, + match="OPENDOX_STATE_DIR") as refused: + console_access.publish( + httpd, page_url=serve.server_url(httpd, "/index.html"), + env={"OPENDOX_STATE_DIR": str(state)}) + assert "lies inside the served repository" in str(refused.value) + assert str(outer.resolve()) in str(refused.value) + finally: + httpd.server_close() + assert sorted(p.name for p in (state / console_access.CONSOLE_DIRNAME).iterdir()) \ + == [other.path.name] + assert (other.path.read_bytes(), _fingerprint(other.path)) == before + console_access.remove_private_copy(other) + + +def test_a_tokenless_standalone_plane_never_serves_a_siblings_copy( + tmp_path, monkeypatch, standalone_profile) -> None: + """B1, the reviewer's `--web-dir` link variant: no root of the tokenless + plane holds the state directory, but a link inside its static bundle + leads there, and a hard link in its checkout is the copy by another name. + The copies' directory is marked private on this plane too, so the + sibling's copy and the directory's listing are 404, through the static + handler and `/source` alike, and the bundle still answers.""" + import shutil as _shutil + + from opendox import console_access, serve + + web = tmp_path / "web" + _shutil.copytree(WEB, web) + state = _state(tmp_path) + (web / "state-alias").symlink_to(state) + outer = fresh_repository(PLAIN, tmp_path / "b") + httpd, _repo, worker = _guarded_plane(tmp_path, monkeypatch, repo=outer, web=web) + try: + base = httpd.server_address[:2] + assert httpd.console_token is None, "the case is vacuous: a token was minted" + other = _write(state, port=9) # a sibling plane's copy + token = console_access.read_private_copy(other.path)["console_token"] + assert console_access.publish( + httpd, page_url=serve.server_url(httpd, "/index.html"), + env={"OPENDOX_STATE_DIR": str(state)}) is None + assert sorted(p.name for p in other.path.parent.iterdir()) == [other.path.name] + os.link(other.path, outer / "hard.md") + _assert_never_served(base, token, ( + f"/state-alias/console/{other.path.name}", "/state-alias/console/", + "/source/hard.md")) + status, _headers, raw = _call(base, "GET", "/state-alias/console/") + assert other.path.name.encode() not in raw + assert _call(base, "GET", "/index.html")[0] == 200 + finally: + _stop_plane(httpd, worker) + + +def test_b2_a_platform_without_the_posix_primitives_refuses_by_name( + tmp_path) -> None: + """B2, as the reviewer staged it: on Windows `os.getuid`, `O_NOFOLLOW` and + `O_DIRECTORY` do not exist. The standalone start is a refusal by name, as + `bundle.unsupported_platform` names the same gap, and never an + `AttributeError` traceback.""" + import textwrap + + repo = _adv_repo(tmp_path / "r") + state = _state(tmp_path) + code = textwrap.dedent(""" + import os, sys + from opendox import serve + for name in ('getuid', 'O_NOFOLLOW', 'O_DIRECTORY'): + delattr(os, name) + """) + proc, out = _adv_serve(tmp_path, repo, state, _free_port(), "w", code=code) + rc = proc.wait(60) + text = out.read_text() + assert "Traceback" not in text, text + assert rc == 1 and "serve refused:" in text, text + assert "needs a POSIX platform" in text and "os.getuid" in text, text + assert not (state / "console").exists() + + +@pytest.mark.parametrize("gap", ["os.O_NOFOLLOW", "a directory descriptor"]) +def test_the_writer_and_the_reader_refuse_a_platform_without_the_primitives( + tmp_path, monkeypatch, gap) -> None: + """B2, by part: the writer refuses before it writes anything, the reader + before it reads, each naming the gap (`unsupported_platform`).""" + from opendox import console_access + + state = _state(tmp_path) + copy = _write(state) # while the primitives exist + if gap == "os.O_NOFOLLOW": + monkeypatch.delattr(os, "O_NOFOLLOW") + named = "os.O_NOFOLLOW" + else: + monkeypatch.setattr(console_access, "_DIR_FD_CALLS", False) + named = "calls relative to a directory's descriptor" + try: + assert named in (console_access.unsupported_platform() or "") + with pytest.raises(console_access.ConsoleAccessRefused, + match="needs a POSIX platform") as refused: + _write(state, port=9) + assert named in str(refused.value) + assert not console_access.private_copy_path(state, 9).exists() + with pytest.raises(console_access.ConsoleAccessRefused, + match="needs a POSIX platform"): + console_access.read_private_copy(copy.path) + finally: + monkeypatch.undo() + assert console_access.unsupported_platform() is None + console_access.remove_private_copy(copy) + + +def _two_spellings(monkeypatch, real: Path, alias: Path) -> None: + """A case-insensitive filesystem, as far as `os.stat` and `os.listdir` + can tell: `alias`, and every name under it, is `real`. Linux cannot spell + one directory two ways (a bind mount needs root), so the two calls a + second spelling reaches are told so; `os.lstat`, and so resolving, still + sees `alias` as a name that does not exist, as macOS's resolving keeps + the case it was given.""" + real_stat, real_listdir = os.stat, os.listdir + + def mapped(path): + if isinstance(path, (str, os.PathLike)): + text = os.fspath(path) + if isinstance(text, str) and (text == str(alias) + or text.startswith(str(alias) + os.sep)): + return str(real) + text[len(str(alias)):] + return path + + monkeypatch.setattr(os, "stat", lambda path, *a, **k: real_stat(mapped(path), *a, **k)) + monkeypatch.setattr(os, "listdir", + lambda path=".", *a, **k: real_listdir(mapped(path), *a, **k)) + + +@pytest.mark.parametrize("served", ["the state directory", "a root inside it", + "a root holding it"]) +def test_the_boundary_knows_a_second_spelling_by_its_identity( + tmp_path, monkeypatch, served) -> None: + """B4, the overlap: on a case-insensitive filesystem `/OUTER/state` + IS `/outer/state`, though no name says so. The boundary compares the + directories' identities too, so the second spelling of the state + directory, of a root inside it, or of a root holding it, refuses by name + before anything is written.""" + from opendox import console_access + + outer = tmp_path / "outer" + outer.mkdir() + state = _state(outer) + alias = tmp_path / "OUTER" + _two_spellings(monkeypatch, outer, alias) + root = {"the state directory": alias / "state", + "a root inside it": alias / "state" / "inner", + "a root holding it": alias}[served] + with pytest.raises(console_access.ConsoleAccessRefused, + match="OPENDOX_STATE_DIR") as refused: + console_access.write_private_copy( + state, page_url="http://127.0.0.1:8080/index.html", port=8080, + token=_token(), served_roots=(root,)) + assert str(root) in str(refused.value) + assert not (state / console_access.CONSOLE_DIRNAME).exists() + + +def test_a_second_spelling_of_the_copies_directory_is_never_listed( + tmp_path, monkeypatch, standalone_profile) -> None: + """B4, the static guard, in the reviewer's r9 layout: `web/state-alias` + leads to the state directory, and on a case-insensitive filesystem + `/state-alias/CONSOLE/` lists the copies' directory under a name no + private root spells. The guard knows the directory by its identity, so + that listing is a 404 like the plain spelling's.""" + import shutil as _shutil + + from opendox import console_access, serve + + web = tmp_path / "web" + _shutil.copytree(WEB, web) + state = _state(tmp_path) + (web / "state-alias").symlink_to(state) + httpd, _repo, worker = _guarded_plane(tmp_path, monkeypatch, web=web) + try: + base = httpd.server_address[:2] + copy = console_access.publish( + httpd, page_url=serve.server_url(httpd, "/index.html"), + env={"OPENDOX_STATE_DIR": str(state)}) + _two_spellings(monkeypatch, copy.path.parent, state / "CONSOLE") + assert console_access.within_private_roots( + web / "state-alias" / "CONSOLE", httpd.private_roots) + for path in ("/state-alias/CONSOLE/", "/state-alias/console/"): + for method in ("GET", "HEAD"): + status, _headers, raw = _call(base, method, path) + assert status == 404, (method, path, status) + assert copy.path.name.encode() not in raw, (method, path) + assert _call(base, "GET", "/index.html")[0] == 200 + finally: + _stop_plane(httpd, worker) + + +def test_b3_a_second_stop_before_the_cleanup_hold_leaves_no_copy(tmp_path) -> None: + """B5 (the reviewer's `test_b3`): a double Ctrl-C, or SIGTERM and then the + SIGHUP of a closing terminal. The second stop lands after the first + unwound the serve loop and before the cleanup's + `deferred_termination(raise_pending=False)` holds anything; it is + delivered here at exactly that point. Only the first stop is raised, so + the cleanup runs to its end: no copy, no traceback, exit 0.""" + import textwrap + + repo = _adv_repo(tmp_path / "r") + state = _state(tmp_path) + port = _free_port() + code = textwrap.dedent(""" + import os, signal, sys + from opendox import console_access as ca, serve + real = ca.deferred_termination + def window(*, raise_pending=True): + if not raise_pending: + os.kill(os.getpid(), signal.SIGINT) + for _ in range(1000): + pass + return real(raise_pending=raise_pending) + ca.deferred_termination = window + """) + proc, out = _adv_serve(tmp_path, repo, state, port, "s", code=code) + copy = state / "console" / f"{port}.html" + assert copy.exists(), out.read_text() + rc = _adv_stop(proc) + assert not copy.exists(), "the copy outlived the stop" + assert "Traceback" not in out.read_text() and rc == 0, out.read_text() + + +def test_only_the_first_stop_is_raised() -> None: + """B5 in the process: once a stop has been raised, a later one is only + recorded, held or not; and a console that starts again raises its own + first stop.""" + from opendox import console_access + + def stop() -> None: + os.kill(os.getpid(), signal.SIGTERM) + signal.pthread_sigmask(signal.SIG_BLOCK, []) # deliver now + + for _attempt in range(2): + raised: list[str] = [] + with console_access.terminate_as_interrupt(True): + try: + stop() + except KeyboardInterrupt: + raised.append("first") + try: + stop() + for _ in range(1000): + pass + except KeyboardInterrupt: + pytest.fail("a second stop was raised") + try: + with console_access.deferred_termination(): + stop() + except KeyboardInterrupt: + pytest.fail("a stop held after the first was raised") + assert raised == ["first"], raised + + +def test_b6a_a_link_another_user_owns_on_the_state_path_is_refused( + tmp_path, monkeypatch) -> None: + """B6(a): `_walked`'s link-owner rule, the only guard against a link + another user owns (and can re-point) on OPENDOX_STATE_DIR. Simulated by + reporting the link's owner as another uid.""" + from opendox import console_access + + private = _state(tmp_path / "private") + alias = tmp_path / "alias" + alias.symlink_to(private) + real_lstat = os.lstat + + def lstat(path, *a, **k): + info = real_lstat(path, *a, **k) + if Path(path) == alias: + fields = list(info) + fields[stat.ST_UID] = os.getuid() + 1 + return os.stat_result(fields) + return info + + monkeypatch.setattr(console_access.os, "lstat", lstat) + with pytest.raises(console_access.ConsoleAccessRefused, + match="symbolic link owned by uid"): + console_access.write_private_copy( + alias, page_url="http://127.0.0.1:8080/index.html", port=8080, + token="t" * 43, served_roots=()) + assert not (private / "console").exists() + + +def test_b6b_the_reader_judges_the_state_directory_again(tmp_path) -> None: + """B6(b): `read_private_copy` asks all of it again: a state directory + loosened to 1777 after the copy was written (sticky, so the walk's rule + for the directories above lets it pass) is refused by the reader.""" + from opendox import console_access + + state = _state(tmp_path / "state") + copy = console_access.write_private_copy( + state, page_url="http://127.0.0.1:8080/index.html", port=8080, + token="t" * 43, served_roots=()) + state.chmod(0o1777) + try: + with pytest.raises(console_access.ConsoleAccessRefused, + match="writable by every user"): + console_access.read_private_copy(copy.path) + finally: + state.chmod(0o700) + console_access.remove_private_copy(copy) + + +def test_b6c_the_copy_is_0600_under_a_umask_that_strips_owner_write(tmp_path) -> None: + """B6(c): `os.fchmod(handle, 0o600)` is what makes the copy 0600 where the + umask removes an owner bit. The umask case of section 3 uses umask 0, + where `os.open`'s mode alone gives 0600.""" + from opendox import console_access + + state = _state(tmp_path / "state") + previous = os.umask(0o277) + try: + copy = console_access.write_private_copy( + state, page_url="http://127.0.0.1:8080/index.html", port=8080, + token="t" * 43, served_roots=()) + finally: + os.umask(previous) + assert stat.S_IMODE(os.lstat(copy.path).st_mode) == 0o600 + console_access.remove_private_copy(copy) + + +def test_a_no_serve_run_publishes_opens_and_prints_no_copy( + tmp_path, monkeypatch, capsys, standalone_profile) -> None: + """B8: `--no-serve` closes the server once it has printed the URL, so a + copy written for it opened a console page nothing answered, and was gone + as the run returned. No copy is written, none is opened, and no console + line is printed; the page's URL is still printed and opened, as it was + before T104, and carries no token.""" + from opendox import cli, console_access + + _clean_git(monkeypatch) + repo = _repository(tmp_path) + state = _state(tmp_path) + monkeypatch.setenv("OPENDOX_STATE_DIR", str(state)) + written: list[object] = [] + real = console_access.write_private_copy + + def recording(*args, **kwargs): + written.append(args) + return real(*args, **kwargs) + + def served(self, *args, **kwargs): + raise AssertionError("a --no-serve run served") + + monkeypatch.setattr(console_access, "write_private_copy", recording) + monkeypatch.setattr(socketserver.BaseServer, "serve_forever", served) + opened: list[str] = [] + assert cli._generate_and_open( + _generate_and_open_args(tmp_path, repo, "--no-serve"), + opener=opened.append) == 0 + out = capsys.readouterr().out + assert written == [], "a --no-serve run wrote a copy" + assert not (state / console_access.CONSOLE_DIRNAME).exists() + assert " console " not in out and "console_token" not in out, out + (url,), = [opened] + assert url.startswith("http://") and url.endswith("/index.html"), url + assert url in out.splitlines(), out + + +def _own_file(directory: Path, name: str, mode: int = 0o600) -> Path: + path = directory / name + path.write_text("left behind\n", encoding="utf-8") + path.chmod(mode) + return path + + +def test_a_publication_sweeps_the_copies_whose_consoles_died(tmp_path) -> None: + """B9: a serve that died (SIGKILL, an out-of-memory kill) left its copy, + a token in it, until a later serve took the same port. A publication now + sweeps every copy whose reservation is free, and the temporary and + taken names a writer or a remover left mid-way. A running console's copy + is never swept, and neither is anything that is not this user's own + copy-shaped file of mode 0600: a loosened copy (refused by name, never + replaced), a link, a file of another name.""" + from opendox import console_access + + state = _state(tmp_path) + dead = _write(state, port=9) + _abandon(dead) # its server died + alive = _write(state, port=10) # its server still runs + console = alive.path.parent + left = [_own_file(console, ".11.html.opendox-424242"), + _own_file(console, ".12.html.removing-1-0123456789ab")] + kept = [_own_file(console, "13.html", mode=0o644), + _own_file(console, "notes.txt")] + (console / "14.html").symlink_to(tmp_path / "elsewhere.html") + fresh = _write(state, port=8080) + names = sorted(p.name for p in console.iterdir()) + assert dead.path.name not in names, "a dead console's copy was not swept" + assert not any(p.name in names for p in left), names + assert sorted([alive.path.name, fresh.path.name, "13.html", "14.html", + "notes.txt"]) == names + assert console_access.read_private_copy(alive.path)["console_token"] + for copy in (alive, fresh): + console_access.remove_private_copy(copy) + + +def test_a_sigkilled_serves_copy_is_swept_by_the_next_serve(tmp_path) -> None: + """B9, as the reviewer staged it: one serve is killed with SIGKILL, which + runs no cleanup, and another starts on another port. The killed serve's + copy is swept when the second publishes, and the second's goes when it + stops: nothing is left.""" + repo = _adv_repo(tmp_path / "r") + state = _state(tmp_path) + first_port = _free_port() + first, _out = _adv_serve(tmp_path, repo, state, first_port, "first") + assert (state / "console" / f"{first_port}.html").exists(), _out.read_text() + first.send_signal(signal.SIGKILL) + first.wait(20) + assert (state / "console" / f"{first_port}.html").exists() + second, out = _adv_serve(tmp_path, repo, state, _free_port(), "second") + try: + assert second.poll() is None, out.read_text() + assert not (state / "console" / f"{first_port}.html").exists(), \ + "the killed serve's copy was not swept" + finally: + assert _adv_stop(second) == 0 + assert list((state / "console").iterdir()) == [] From c4e6c01ba24f579bec3e43c0cc474d3a0b692fef Mon Sep 17 00:00:00 2001 From: Brett Heap <1513478+brettheap@users.noreply.github.com> Date: Sun, 4 Oct 2026 17:33:59 +0000 Subject: [PATCH 81/88] T104: the start prints the ruled hint line for a browser that cannot open the copy (B3) The adversarial review's B3: Ubuntu's default snap browser, and Flatpak browsers, cannot read a file under a hidden directory such as ~/.local/state, and a Windows browser under WSL may not open a Linux path. The token is never printed, so there was no way past it. RULED by Brett (2026-10-04, "Hint line, accepted limit (Recommended)"): beside the copy's path, both entry points print ONE line with no token, saying to set OPENDOX_STATE_DIR to a folder that is not hidden and start again (console_access.UNOPENABLE_HINT). The README's side is openDox#17's (T076). Case: test_the_start_prints_the_unopenable_hint_and_never_the_token, for serve and generate-and-open: the line is printed once, right after the copy's path, and no line printed carries the token. A --no-serve run prints no hint. Arc: neutral-product-standalone-operability Lane: openxfactory-4 (openXfactory-4-openDox_extraction) Co-Authored-By: Claude Opus 5.5 (1M context) --- src/opendox/cli.py | 3 ++ src/opendox/console_access.py | 19 +++++++++-- src/opendox/serve.py | 3 ++ tests/test_console_token_delivery.py | 48 ++++++++++++++++++++++++++++ 4 files changed, 71 insertions(+), 2 deletions(-) diff --git a/src/opendox/cli.py b/src/opendox/cli.py index fa7a8380..e84c81f1 100644 --- a/src/opendox/cli.py +++ b/src/opendox/cli.py @@ -937,6 +937,9 @@ def _generate_and_serve(args: argparse.Namespace, run_dir: Path, *, print(f" console {console.file_url} (this user's private " "copy, mode 0600: open it to open the console page " "again)") + # A browser that cannot open it is told the way past it, + # in one line with no token (RULED, B3). + print(f" {console_access.UNOPENABLE_HINT}") # The URL is ALWAYS printed on its own line, AND FLUSHED (plan # 034 T056). Where standard output is a pipe or a file, Python # buffers it by block, and the process is about to block in diff --git a/src/opendox/console_access.py b/src/opendox/console_access.py index febe89e1..f501fd36 100644 --- a/src/opendox/console_access.py +++ b/src/opendox/console_access.py @@ -33,7 +33,10 @@ the copy's `file://` path. That is Jupyter's own redirect file, and for the same reason. The start prints the copy's PATH, never the token, with or without `--no-open`, and opening that file again is how a user re-opens the -page while the server runs. The copy is removed when the server stops. +page while the server runs. The copy is removed when the server stops. Beside +the path, one line with no token tells a user whose browser cannot open that +file (a snap or Flatpak browser, a Windows browser under WSL) to move the +state directory (`UNOPENABLE_HINT`, RULED as an accepted limit). THE COPY IS CHECKED THE WAY openDox-code#69's BUNDLE CHECKS ITS TREE (`opendox.runtime.bundle`: `refuse_an_unsafe_tree`, `_make_private_directories`, @@ -115,7 +118,8 @@ "CONSOLE_DIRNAME", "ConsoleAccessRefused", "ConsoleTerminated", "DELIVERY_CAPABILITIES", "DELIVERY_OPENED_URL", "FRAGMENT_KEY", "PrivateCopy", "RECORD_ELEMENT_ID", - "RECORD_KIND", "deferred_termination", "delivery_for", "guard_private_roots", + "RECORD_KIND", "UNOPENABLE_HINT", "deferred_termination", "delivery_for", + "guard_private_roots", "is_private_file", "needs_copy", "opens_a_private_file", "opened_url", "private_copy_path", "publish", "read_private_copy", "remove_private_copy", "terminate_as_interrupt", "unsupported_platform", @@ -141,6 +145,17 @@ #: The one mode the copies' directory, `console/`, may have (#1144 12.4a: the #: copy is mode 0600 "in a directory of mode 0700"). CONSOLE_DIR_MODE = 0o700 +#: The one line a start prints beside the copy's path, and it carries no token. +#: Some browsers cannot open the copy where it is: a snap or Flatpak browser +#: is kept out of a hidden directory such as `~/.local/state`, and a Windows +#: browser under WSL may not open a Linux path at all. The token is never +#: printed, so this line is the way past it (RULED by Brett on the adversarial +#: review of openDox-code#84, B3, 2026-10-04: "Hint line, accepted limit"). +UNOPENABLE_HINT = ( + "if your browser cannot open this file (a snap or Flatpak browser, or a " + "Windows browser under WSL), set " + f"{runtime_config.PREFIX}STATE_DIR to a folder that is not hidden and " + "start again") #: A copy is a few hundred bytes; a read stops well past that. _READ_LIMIT = 64 * 1024 #: `secrets.token_urlsafe` spells a token in these characters only, so a token diff --git a/src/opendox/serve.py b/src/opendox/serve.py index 74137e0e..dd1d7f15 100644 --- a/src/opendox/serve.py +++ b/src/opendox/serve.py @@ -2703,6 +2703,9 @@ def serve( if console is not None: print(f"console {console.file_url} (this user's private " "copy, mode 0600: open it to open the console page)") + # A browser that cannot open it is told the way past it, + # in one line with no token (RULED, B3). + print(console_access.UNOPENABLE_HINT) # FLUSHED before the process blocks (plan 034 T056): where # standard output is a pipe or a file it is block-buffered, so # an unflushed line never reaches a wrapper while the server diff --git a/tests/test_console_token_delivery.py b/tests/test_console_token_delivery.py index 1f35aa25..b19e3a6a 100644 --- a/tests/test_console_token_delivery.py +++ b/tests/test_console_token_delivery.py @@ -3035,6 +3035,7 @@ def served(self, *args, **kwargs): assert written == [], "a --no-serve run wrote a copy" assert not (state / console_access.CONSOLE_DIRNAME).exists() assert " console " not in out and "console_token" not in out, out + assert console_access.UNOPENABLE_HINT not in out, out (url,), = [opened] assert url.startswith("http://") and url.endswith("/index.html"), url assert url in out.splitlines(), out @@ -3099,3 +3100,50 @@ def test_a_sigkilled_serves_copy_is_swept_by_the_next_serve(tmp_path) -> None: finally: assert _adv_stop(second) == 0 assert list((state / "console").iterdir()) == [] + + +@pytest.mark.parametrize("entry", ["serve", "generate-and-open"]) +def test_the_start_prints_the_unopenable_hint_and_never_the_token( + tmp_path, monkeypatch, capsys, standalone_profile, entry) -> None: + """B3, RULED by Brett ("Hint line, accepted limit", 2026-10-04): a snap + or Flatpak browser cannot open a file under a hidden directory such as + `~/.local/state`, and a Windows browser under WSL may not open a Linux + path at all. The token is never printed, so beside the copy's path the + start prints ONE line saying how to move the state directory, and no + line it prints, that one included, carries the token.""" + from opendox import cli, console_access, serve + + _clean_git(monkeypatch) + repo = _repository(tmp_path) + state = _state(tmp_path) + monkeypatch.setenv("OPENDOX_STATE_DIR", str(state)) + monkeypatch.setattr(serve, "real_notebook_adapter", lambda *a, **k: None) + tokens: list[str] = [] + + def serve_forever(self, *args, **kwargs): + copy = console_access.private_copy_path(state, self.server_address[1]) + tokens.append(console_access.read_private_copy(copy)["console_token"]) + raise KeyboardInterrupt + + monkeypatch.setattr(socketserver.BaseServer, "serve_forever", serve_forever) + if entry == "serve": + snapshot = tmp_path / "snapshot.json" + snapshot.write_text(json.dumps({"generation": {}}), encoding="utf-8") + assert serve.main(["--snapshot", str(snapshot), "--checkout-root", str(repo), + "--port", "0"]) == 0 + prefix = "console file://" + else: + assert cli._generate_and_open( + _generate_and_open_args(tmp_path, repo, "--no-open"), + opener=lambda url: None) == 0 + prefix = " console file://" + out, err = capsys.readouterr() + (token,) = tokens + lines = out.splitlines() + at = next(i for i, line in enumerate(lines) if line.startswith(prefix)) + hint = lines[at + 1] + assert hint.strip() == console_access.UNOPENABLE_HINT, lines + assert "OPENDOX_STATE_DIR" in hint and "not hidden" in hint, hint + assert sum(console_access.UNOPENABLE_HINT in line for line in lines) == 1, lines + assert all(token not in line for line in (out + err).splitlines()), \ + "a line the start printed carries the token" From 0539f8c01df8a6b1212def8e9b68dc3e58aa1718 Mon Sep 17 00:00:00 2001 From: Brett Heap <1513478+brettheap@users.noreply.github.com> Date: Sun, 4 Oct 2026 18:34:50 +0000 Subject: [PATCH 82/88] T104: a removal releases the copy's reservation, and a copy's repr never carries its token Mutant run 18 (71 mutants at 45958bf3) left one alive: M36c, where the removal never closes the descriptor that reserved the copy. Nothing asked for that descriptor after a removal, so a stopped console kept its unlinked copy's file open until garbage collection. test_a_removal_releases_the_copys_reservation asks: after a removal, and where the directory is gone already, no descriptor of this process is the copy's file, and a second release is harmless. Under M36c the removal case fails (review10-m36c-red.txt). That failure's own message printed the copy's repr, and with it the opened URL and the token. A log line, a traceback or a failed assertion that prints a PrivateCopy must never carry its token, so `opened_url` is kept out of the repr (test_a_copys_repr_never_carries_its_token, which fails at c4e6c01b). Mutants M36d (the early return keeps the reservation) and M51 (the repr carries the token) join the run. Arc: neutral-product-standalone-operability Lane: openxfactory-4 (openXfactory-4-openDox_extraction) Co-Authored-By: Claude Opus 5.5 (1M context) --- src/opendox/console_access.py | 5 ++- tests/test_console_token_delivery.py | 59 ++++++++++++++++++++++++++++ 2 files changed, 63 insertions(+), 1 deletion(-) diff --git a/src/opendox/console_access.py b/src/opendox/console_access.py index f501fd36..c35fad57 100644 --- a/src/opendox/console_access.py +++ b/src/opendox/console_access.py @@ -225,7 +225,10 @@ class PrivateCopy: path: Path page_url: str - opened_url: str + #: The URL with the token in its fragment. Kept out of the copy's `repr`, + #: so a log line, a traceback or a failed assertion that prints a copy + #: never prints its token. + opened_url: str = dataclasses.field(repr=False) #: `(st_dev, st_ino)` of the file this process wrote, so a removal at #: shutdown removes that file and never one written after it. identity: tuple[int, int] diff --git a/tests/test_console_token_delivery.py b/tests/test_console_token_delivery.py index b19e3a6a..9c470510 100644 --- a/tests/test_console_token_delivery.py +++ b/tests/test_console_token_delivery.py @@ -3147,3 +3147,62 @@ def serve_forever(self, *args, **kwargs): assert sum(console_access.UNOPENABLE_HINT in line for line in lines) == 1, lines assert all(token not in line for line in (out + err).splitlines()), \ "a line the start printed carries the token" + + +def _held_open(copy) -> int: + """The descriptor that reserves `copy`, checked to be the copy's own.""" + fd = copy.reservation._fd + assert fd is not None, "the copy was written unreserved" + info = os.fstat(fd) + assert (info.st_dev, info.st_ino) == copy.identity + return fd + + +def _assert_released(fd: int, identity: tuple[int, int]) -> None: + """No descriptor `fd` of this process is still the copy's file: it is + closed, or the number has gone to another file since.""" + try: + info = os.fstat(fd) + except OSError as exc: + assert exc.errno == errno.EBADF, exc + return + assert (info.st_dev, info.st_ino) != identity, \ + "the removal kept the copy's reservation open" + + +@pytest.mark.parametrize("how", ["removed", "its directory gone"]) +def test_a_removal_releases_the_copys_reservation(tmp_path, how) -> None: + """The reservation goes with the copy (mutant run 18's M36c). Removing a + copy closes the descriptor that reserved it, so a stopped console holds + no file of its own open, and the token's file, unlinked, is not kept + alive by it. The same holds where the directory is gone already and there + is nothing to remove. A second release is harmless.""" + from opendox import console_access + + state = _state(tmp_path) + copy = _write(state) + fd = _held_open(copy) + if how == "its directory gone": + os.rename(state, tmp_path / "moved") + console_access.remove_private_copy(copy) + _assert_released(fd, copy.identity) + copy.reservation.close() + console_access.remove_private_copy(copy) + if how == "removed": + assert not copy.path.exists() + + +def test_a_copys_repr_never_carries_its_token(tmp_path) -> None: + """A log line, a traceback or a failed assertion that prints a + `PrivateCopy` prints its path, its page and its identity, and never the + token: the opened URL is kept out of its `repr`.""" + from opendox import console_access + + token = _token() + copy = _write(_state(tmp_path), token=token) + try: + assert token in copy.opened_url + assert token not in repr(copy) and token not in str(copy) + assert str(copy.path) in repr(copy) + finally: + console_access.remove_private_copy(copy) From af2a2efb3509ad94704205f6114371e23b906e12 Mon Sep 17 00:00:00 2001 From: Brett Heap <1513478+brettheap@users.noreply.github.com> Date: Sun, 4 Oct 2026 20:02:10 +0000 Subject: [PATCH 83/88] T104 fix round 10: the tokenless guard walks the state directory once and refuses an unsupported platform first (Copilot review) Copilot's review at 0539f8c0 (5407901398), two threads, both on round 9's tokenless guard (`guard_private_roots`). Each case failed first at 0539f8c0 (review10-red.txt): - r4179091592. The boundary check and the marking each resolved the configured state path for themselves, so a link on it re-pointed between the two left the boundary judging the real state directory and the marking naming a decoy, and an outward static link served a sibling plane's copy. The guard now walks the state directory once (`_walked`), judging every directory and link on the way as the writer does, and the boundary and the marking both use that walk's path. An operating-system error on the way is a refusal by name. Cases: - test_a_tokenless_planes_state_link_retargeted_mid_guard_marks_the_real_directory, Copilot's layout: the link is re-pointed at a decoy between the two, and the sibling's copy and the listing stay 404; - test_a_tokenless_planes_unsafe_state_path_refuses_its_start: a world-writable, non-sticky directory on the way refuses the tokenless start by name, and nothing is marked. - r4179091624. A tokenless plane skipped the platform check, started, and marked a private root that its handlers then judged with the missing O_NONBLOCK. `publish` and the guard now refuse an unsupported platform by name before anything else, token or not. Cases: - test_a_tokenless_plane_refuses_a_platform_without_the_primitives, in the process, without O_NONBLOCK; - test_a_tokenless_start_without_the_posix_primitives_refuses_by_name, the no-identity variant of the reviewer's B2 child. Mutants M52 (the marking resolves the path again), M52b (the guard does not walk) and M53 (the tokenless plane skips the platform check) join the run; M26, M34, M44c and M44d now point at the walked guard's lines. Arc: neutral-product-standalone-operability Lane: openxfactory-4 (openXfactory-4-openDox_extraction) Co-Authored-By: Claude Opus 5.5 (1M context) --- src/opendox/console_access.py | 31 +++++- tests/test_console_token_delivery.py | 145 +++++++++++++++++++++++++++ 2 files changed, 172 insertions(+), 4 deletions(-) diff --git a/src/opendox/console_access.py b/src/opendox/console_access.py index c35fad57..3ed59265 100644 --- a/src/opendox/console_access.py +++ b/src/opendox/console_access.py @@ -1258,10 +1258,32 @@ def guard_private_roots(httpd: Any, state_dir: Path | str) -> None: `write_private_copy` asks it, by name and by identity, and refuses the start by name, and the copies' directory is marked private (`httpd.private_roots`), so the static handler, `/source` and - `/snapshot.json` refuse every console's copy here too.""" - _refuse_a_served_state_dir(state_dir, tuple(getattr(httpd, "served_roots", ())), - port=None) - httpd.private_roots = (Path(state_dir).resolve() / CONSOLE_DIRNAME,) + `/snapshot.json` refuse every console's copy here too. + + ONE WALK, AS THE WRITER WALKS (Copilot at openDox-code#84, + r4179091592). The boundary and the marking each resolved the configured + path for themselves, so a link on it re-pointed between the two left the + boundary judging the real state directory and the marking naming a + decoy. The state directory is walked once (`_walked`), every directory + and link on the way judged as the writer judges them, and the boundary + and the marking both use the path that walk returned. An operating-system + refusal on the way is a refusal by name, as it is for the writer. + + AND THE PLATFORM FIRST (Copilot at openDox-code#84, r4179091624): this + plane's handlers judge files by the same POSIX primitives, so a platform + without them is refused by name here too (`unsupported_platform`).""" + _refuse_an_unsupported_platform() + try: + state = _walked(state_dir) + _refuse_a_served_state_dir(state, tuple(getattr(httpd, "served_roots", ())), + port=None) + except OSError as exc: + raise ConsoleAccessRefused( + f"{private_copy_path(state_dir, 0).parent} cannot be judged ({exc}), " + "so this standalone plane cannot keep the console tokens' private " + "copies unserved, and the start is refused. Use a state directory " + f"this user can reach ({runtime_config.PREFIX}STATE_DIR)") from None + httpd.private_roots = (state / CONSOLE_DIRNAME,) def publish(httpd: Any, *, page_url: str, @@ -1278,6 +1300,7 @@ def publish(httpd: Any, *, page_url: str, it: a console nobody can open is not served as if it could be.""" if not _standalone(httpd): return None + _refuse_an_unsupported_platform() # token or not (r4179091624) try: state = runtime_config.state_dir(env) except runtime_config.ConfigurationError as exc: diff --git a/tests/test_console_token_delivery.py b/tests/test_console_token_delivery.py index 9c470510..2c3dd53d 100644 --- a/tests/test_console_token_delivery.py +++ b/tests/test_console_token_delivery.py @@ -3206,3 +3206,148 @@ def test_a_copys_repr_never_carries_its_token(tmp_path) -> None: assert str(copy.path) in repr(copy) finally: console_access.remove_private_copy(copy) + + +# --------------------------------------------------------------------------- +# 18 — Copilot's review at 0539f8c0: the tokenless plane walks the state +# directory once (r4179091592) and refuses an unsupported platform +# first (r4179091624) +# --------------------------------------------------------------------------- + +def test_a_tokenless_planes_state_link_retargeted_mid_guard_marks_the_real_directory( + tmp_path, monkeypatch, standalone_profile) -> None: + """r4179091592, Copilot's layout. `OPENDOX_STATE_DIR` names a link to the + real state directory, and the tokenless plane's `--web-dir` holds an + outward link to it, where a sibling plane's copy lies. The link is + re-pointed at a decoy between the boundary check and the marking. The + guard walks once and marks what that walk reached, so the sibling's copy + stays a 404; the marking used to resolve the link again and name the + decoy, and the copy was served.""" + import shutil as _shutil + + from opendox import console_access, serve + + web = tmp_path / "web" + _shutil.copytree(WEB, web) + real = _state(tmp_path) + decoy = tmp_path / "decoy" + decoy.mkdir(mode=0o700) + alias = tmp_path / "state-link" + alias.symlink_to(real) + (web / "state-alias").symlink_to(real) + outer = fresh_repository(PLAIN, tmp_path / "b") + httpd, _repo, worker = _guarded_plane(tmp_path, monkeypatch, repo=outer, web=web) + real_boundary = console_access._refuse_a_served_state_dir + + def then_retarget(*args, **kwargs): + real_boundary(*args, **kwargs) + alias.unlink() + alias.symlink_to(decoy) # re-pointed between the two + + try: + base = httpd.server_address[:2] + assert httpd.console_token is None, "the case is vacuous: a token was minted" + other = _write(real, port=9) # a sibling plane's copy + token = console_access.read_private_copy(other.path)["console_token"] + monkeypatch.setattr(console_access, "_refuse_a_served_state_dir", then_retarget) + assert console_access.publish( + httpd, page_url=serve.server_url(httpd, "/index.html"), + env={"OPENDOX_STATE_DIR": str(alias)}) is None + assert alias.resolve() == decoy.resolve(), "the retarget was never staged" + (marked,) = httpd.private_roots + assert marked.resolve() == (real / console_access.CONSOLE_DIRNAME).resolve() + _assert_never_served(base, token, ( + f"/state-alias/console/{other.path.name}", "/state-alias/console/")) + assert _call(base, "GET", "/index.html")[0] == 200 + console_access.remove_private_copy(other) + finally: + _stop_plane(httpd, worker) + + +def test_a_tokenless_planes_unsafe_state_path_refuses_its_start( + tmp_path, monkeypatch, standalone_profile) -> None: + """r4179091592: the guard judges the state directory's path as the writer + does, so a directory on the way that another user could change (here a + world-writable, non-sticky one) refuses the tokenless start by name too, + and nothing is marked.""" + from opendox import console_access, serve + + shared = tmp_path / "shared" + shared.mkdir() + shared.chmod(0o777) + try: + state = _state(shared / "inner") + outer = fresh_repository(PLAIN, tmp_path / "b") + _clean_git(monkeypatch) + snapshot = tmp_path / "snapshot.json" + snapshot.write_text(json.dumps({"generation": {}}), encoding="utf-8") + httpd = serve.build_server(WEB, snapshot, outer, port=0, quiet=True) + try: + assert httpd.console_token is None, "the case is vacuous: a token was minted" + with pytest.raises(console_access.ConsoleAccessRefused, + match="not sticky") as refused: + console_access.publish( + httpd, page_url=serve.server_url(httpd, "/index.html"), + env={"OPENDOX_STATE_DIR": str(state)}) + assert str(shared) in str(refused.value) + assert not getattr(httpd, "private_roots", ()) + finally: + httpd.server_close() + finally: + shared.chmod(0o700) + + +def test_a_tokenless_plane_refuses_a_platform_without_the_primitives( + tmp_path, monkeypatch, standalone_profile) -> None: + """r4179091624, in the process: a tokenless standalone plane used to skip + the platform check, start, and mark a private root its handlers then + judged with the missing `O_NONBLOCK`, dropping every static request. It + refuses by name now, before it marks anything.""" + from opendox import console_access, serve + + _clean_git(monkeypatch) + outer = fresh_repository(PLAIN, tmp_path / "b") + state = _state(tmp_path) + snapshot = tmp_path / "snapshot.json" + snapshot.write_text(json.dumps({"generation": {}}), encoding="utf-8") + httpd = serve.build_server(WEB, snapshot, outer, port=0, quiet=True) + try: + assert httpd.console_token is None, "the case is vacuous: a token was minted" + monkeypatch.delattr(os, "O_NONBLOCK") + with pytest.raises(console_access.ConsoleAccessRefused, + match="needs a POSIX platform") as refused: + console_access.publish( + httpd, page_url=serve.server_url(httpd, "/index.html"), + env={"OPENDOX_STATE_DIR": str(state)}) + assert "os.O_NONBLOCK" in str(refused.value) + assert not getattr(httpd, "private_roots", ()) + finally: + monkeypatch.undo() + httpd.server_close() + + +def test_a_tokenless_start_without_the_posix_primitives_refuses_by_name( + tmp_path) -> None: + """r4179091624, as a user starts it: the no-identity variant of the + reviewer's B2 child. With `os.getuid`, `O_NOFOLLOW`, `O_DIRECTORY` and + `O_NONBLOCK` gone, as on Windows, a checkout with no git identity (so no + token) refuses its start by name, and never serves.""" + import textwrap + + repo = _adv_repo(tmp_path / "r", identity=False) + state = _state(tmp_path) + code = textwrap.dedent(""" + import os, sys + from opendox import serve + for name in ('getuid', 'O_NOFOLLOW', 'O_DIRECTORY', 'O_NONBLOCK'): + delattr(os, name) + """) + proc, out = _adv_serve(tmp_path, repo, state, _free_port(), "w", code=code) + if proc.poll() is None: + _adv_stop(proc) + pytest.fail("the tokenless plane started serving: " + out.read_text()) + rc = proc.wait(60) + text = out.read_text() + assert "Traceback" not in text, text + assert rc == 1 and "serve refused:" in text, text + assert "needs a POSIX platform" in text and "os.O_NONBLOCK" in text, text From ed6e4769876f93e36656a74c87dc8c3fba3bb143 Mon Sep 17 00:00:00 2001 From: Brett Heap <1513478+brettheap@users.noreply.github.com> Date: Sun, 4 Oct 2026 20:49:19 +0000 Subject: [PATCH 84/88] T104 fix round 11: a copy is known by what it holds, a check that cannot be made denies, and an entry's payload comes first (Copilot review) Copilot's review at af2a2efb (5408101006): three threads, and one finding "previously missed" in the review's body. Each case failed first at af2a2efb (review11-red.txt): - r4179239380, another state directory. Two standalone planes of one user can have different OPENDOX_STATE_DIR values, and the guard knew only its own plane's copies. If plane A's web root linked to plane B's state directory, A served B's copy. `is_private_file` now first judges the file it was handed by what that file holds: a regular file whose head carries a console record (`_carries_a_console_record`, read with pread from the open descriptor) is a copy, wherever it lies. Case: test_another_state_directorys_copy_is_never_served, through a static link and a hard link under /source. - r4179239411, a removal mid-read. A copy removed after a read opened it, and before the scan could stat its name, matched nothing. The open file still holds its record, so it is refused. Case: test_a_copy_removed_during_the_scan_is_never_served. - r4179239424, a scan that fails. A private directory that exists but cannot be listed (EMFILE, EACCES), or a name in it whose status cannot be read for any reason but its removal, used to let the file through. Both deny now, and so does a regular file whose head cannot be read. A private directory that does not exist still holds no copy. Cases: test_a_private_directory_that_cannot_be_scanned_denies_the_read (EMFILE simulated, and EACCES), test_a_name_whose_status_cannot_be_read_denies_the_read, test_a_file_whose_head_cannot_be_read_is_denied. - Previously missed, serve.py `_entry_bytes`: on a standalone plane an entry with both an in-memory payload and a snapshot path served the file, and a 404 where the file was missing, against SnapshotEntry.read_bytes' payload-first contract. The payload comes first again, and only the file fallback is guarded. Case: test_an_entrys_payload_comes_before_its_guarded_file, with the file missing, present, and a private copy. Mutants M54, M54b, M55, M55b and M56 join the run; M38 now points at the identity match's new indentation. Arc: neutral-product-standalone-operability Lane: openxfactory-4 (openXfactory-4-openDox_extraction) Co-Authored-By: Claude Opus 5.5 (1M context) --- src/opendox/console_access.py | 61 ++++++++- src/opendox/serve.py | 9 +- tests/test_console_token_delivery.py | 194 +++++++++++++++++++++++++++ 3 files changed, 256 insertions(+), 8 deletions(-) diff --git a/src/opendox/console_access.py b/src/opendox/console_access.py index 3ed59265..92c3e49c 100644 --- a/src/opendox/console_access.py +++ b/src/opendox/console_access.py @@ -1168,8 +1168,33 @@ def terminate_as_interrupt(enabled: bool): _held.update(pending=None, stopping=False) +def _carries_a_console_record(handle: int, info: os.stat_result) -> bool: + """Whether the regular file open on `handle` IS a console token's private + copy by what it holds: a console record (`RECORD_KIND`) in the element a + copy keeps it in, wherever the file lies and whatever its name. + + Read with `pread`, from the descriptor already open, so it needs no new + descriptor and does not move the offset the caller then reads from. A + regular file whose head cannot be read is judged to be a copy: a check + that cannot be made denies, never allows.""" + if not stat.S_ISREG(info.st_mode): + return False + try: + head = os.pread(handle, _READ_LIMIT, 0) + except OSError: + return True + found = _RECORD_PATTERN.search(head.decode("utf-8", "replace")) + if found is None: + return False + try: + record = json.loads(found.group("record")) + except ValueError: + return False + return isinstance(record, dict) and record.get("kind") == RECORD_KIND + + def is_private_file(handle: int, private_roots: Iterable[Path | str]) -> bool: - """Whether the file open on `handle` lives in a private-copy directory. + """Whether the file open on `handle` is a console token's private copy. Judged by the FILE'S OWN IDENTITY, `(st_dev, st_ino)`, against every name in each directory `publish` marked private (Copilot at openDox-code#84, @@ -1178,20 +1203,42 @@ def is_private_file(handle: int, private_roots: Iterable[Path | str]) -> bool: made anywhere under a served root all reach the copy by a name that looks like something else. The identity of what was OPENED cannot be swapped afterwards. Every name in the directory counts, every port's copy and a - temporary name included. A directory that cannot be listed counts for - nothing, as no copy can be published in it either.""" + temporary name included. + + AND BY WHAT THE FILE HOLDS (Copilot at openDox-code#84, r4179239380 and + r4179239411). A copy in ANOTHER state directory (a second standalone + plane of the same user, with its own `OPENDOX_STATE_DIR`) is in no + directory this plane marked, and a copy removed between this open and + the directory's scan has no name left there to match. Both are still a + file that holds a console record, so the file open on `handle` is judged + by its own bytes first (`_carries_a_console_record`), and by its + identity after. + + A SCAN THAT FAILS DENIES (Copilot at openDox-code#84, r4179239424). A + private directory that does not exist holds no copy, and counts for + nothing. One that exists and cannot be listed (out of descriptors, say) + cannot clear the file, so the file is judged to be a copy; so is a name + in it whose status cannot be read for any reason but its removal.""" info = os.fstat(handle) + if _carries_a_console_record(handle, info): + return True key = (info.st_dev, info.st_ino) for root in private_roots: try: with os.scandir(root) as entries: for entry in entries: - with contextlib.suppress(OSError): + try: found = entry.stat(follow_symlinks=False) - if (found.st_dev, found.st_ino) == key: - return True + except FileNotFoundError: + continue # removed meanwhile + except OSError: + return True # cannot be cleared: denied + if (found.st_dev, found.st_ino) == key: + return True + except FileNotFoundError: + continue # no directory: no copy in it except OSError: - continue + return True # cannot be listed: denied return False diff --git a/src/opendox/serve.py b/src/opendox/serve.py index dd1d7f15..f20e076a 100644 --- a/src/opendox/serve.py +++ b/src/opendox/serve.py @@ -1460,9 +1460,16 @@ def _read_snapshot(self, entry=None) -> bytes | None: def _entry_bytes(self, entry) -> bytes | None: """A registered entry's snapshot bytes, read as `read_unless_private` reads, where this plane marked a private-copy directory and the entry - names its file. Otherwise the entry reads itself, as before.""" + names its file. Otherwise the entry reads itself, as before. + + THE IN-MEMORY PAYLOAD STILL COMES FIRST (Copilot at openDox-code#84, + at af2a2efb): `SnapshotEntry.read_bytes` serves an entry's payload + before its file, and only the FILE is guarded, so an entry with both + serves its payload whether or not the file exists.""" private = getattr(self.server, "private_roots", ()) path = getattr(entry, "snapshot_path", None) + if getattr(entry, "payload", None) is not None: + return entry.read_bytes() if private and path is not None: try: return read_unless_private(path, private) diff --git a/tests/test_console_token_delivery.py b/tests/test_console_token_delivery.py index 2c3dd53d..10bfd01d 100644 --- a/tests/test_console_token_delivery.py +++ b/tests/test_console_token_delivery.py @@ -3351,3 +3351,197 @@ def test_a_tokenless_start_without_the_posix_primitives_refuses_by_name( assert "Traceback" not in text, text assert rc == 1 and "serve refused:" in text, text assert "needs a POSIX platform" in text and "os.O_NONBLOCK" in text, text + + +# --------------------------------------------------------------------------- +# 19 — Copilot's review at af2a2efb: a copy is known by what it holds, in any +# state directory (r4179239380) and mid-removal (r4179239411); a scan +# that fails denies (r4179239424); an entry's payload comes first +# --------------------------------------------------------------------------- + +def test_another_state_directorys_copy_is_never_served( + tmp_path, monkeypatch, standalone_profile) -> None: + """r4179239380, Copilot's layout: two standalone planes of one user with + DIFFERENT state directories. Plane A's `--web-dir` links to plane B's + state directory, and A's checkout holds a hard link to B's copy. B's + copy lies in no directory A marked, but it holds a console record, so A + answers 404 for it through the static handler and `/source`, for GET and + HEAD, while its bundle still answers.""" + import shutil as _shutil + + from opendox import console_access, serve + + web = tmp_path / "web" + _shutil.copytree(WEB, web) + state_a = _state(tmp_path / "a") + state_b = _state(tmp_path / "b") + (web / "state-b").symlink_to(state_b) + httpd, repo, worker = _guarded_plane(tmp_path, monkeypatch, web=web) + try: + base = httpd.server_address[:2] + own = console_access.publish( + httpd, page_url=serve.server_url(httpd, "/index.html"), + env={"OPENDOX_STATE_DIR": str(state_a)}) + assert own is not None + theirs = _write(state_b, port=9) # plane B's copy + token = console_access.read_private_copy(theirs.path)["console_token"] + os.link(theirs.path, repo / "theirs.md") + _assert_never_served(base, token, ( + f"/state-b/console/{theirs.path.name}", "/source/theirs.md")) + assert _call(base, "GET", "/index.html")[0] == 200 + console_access.remove_private_copy(theirs) + console_access.remove_private_copy(own) + finally: + _stop_plane(httpd, worker) + + +def test_a_copy_removed_during_the_scan_is_never_served( + tmp_path, monkeypatch) -> None: + """r4179239411, the interleaving: a read opens the copy, and the copy is + removed (taken under another name, then unlinked) before the private + directory's scan can stat its name. Nothing in the directory matches + then, but the file already open is still the copy, and is judged by what + it holds: the read is refused.""" + from opendox import console_access, serve + + state = _state(tmp_path) + copy = _write(state) + roots = (copy.path.parent,) + real_open = open + staged: list[str] = [] + + def opened_then_removed(path, mode="r", *args, **kwargs): + stream = real_open(path, mode, *args, **kwargs) + if not staged: # removed after the open + staged.append(str(path)) + console_access.remove_private_copy(copy) + return stream + + monkeypatch.setattr("builtins.open", opened_then_removed) + try: + assert serve.read_unless_private(copy.path, roots) is None + finally: + monkeypatch.undo() + assert staged, "the removal was never staged" + assert not copy.path.exists() + assert list(copy.path.parent.iterdir()) == [], "the removal left a name" + + +@pytest.mark.parametrize("failure", [errno.EMFILE, errno.EACCES], ids=["EMFILE", "EACCES"]) +def test_a_private_directory_that_cannot_be_scanned_denies_the_read( + tmp_path, monkeypatch, failure) -> None: + """r4179239424: a private directory that exists and cannot be listed + (out of descriptors, EMFILE, simulated here; or refused) cannot clear the + file being read, so the read is denied, an ordinary file's as well as a + copy's. A private directory that does not exist holds no copy, and the + ordinary file is read as before.""" + from opendox import console_access, serve + + state = _state(tmp_path) + copy = _write(state) + ordinary = tmp_path / "ordinary.md" + ordinary.write_text("# plain\n", encoding="utf-8") + roots = (copy.path.parent,) + + def cannot_list(path): + raise OSError(failure, os.strerror(failure), str(path)) + + monkeypatch.setattr(console_access.os, "scandir", cannot_list) + assert serve.read_unless_private(ordinary, roots) is None + assert serve.read_unless_private(copy.path, roots) is None + monkeypatch.undo() + absent = (tmp_path / "no-such-state" / console_access.CONSOLE_DIRNAME,) + assert serve.read_unless_private(ordinary, absent) == b"# plain\n" + assert serve.read_unless_private(ordinary, roots) == b"# plain\n" + console_access.remove_private_copy(copy) + + +def test_a_name_whose_status_cannot_be_read_denies_the_read( + tmp_path, monkeypatch) -> None: + """r4179239424, per name: a name in the private directory whose status + cannot be read, for any reason but its removal, cannot be told apart + from the file being read, so the read is denied. A name removed + meanwhile is skipped.""" + from opendox import console_access, serve + + state = _state(tmp_path) + copy = _write(state) + ordinary = tmp_path / "ordinary.md" + ordinary.write_text("# plain\n", encoding="utf-8") + roots = (copy.path.parent,) + real_scandir = os.scandir + + class _Entry: + def __init__(self, raised): + self.raised = raised + + def stat(self, follow_symlinks=True): + raise self.raised + + def entries_failing(raised): + def scandir(path): + listing = [*real_scandir(path), _Entry(raised)] + return contextlib.nullcontext(iter(listing)) + return scandir + + monkeypatch.setattr(console_access.os, "scandir", + entries_failing(PermissionError(errno.EACCES, "denied"))) + assert serve.read_unless_private(ordinary, roots) is None + monkeypatch.setattr(console_access.os, "scandir", + entries_failing(FileNotFoundError(errno.ENOENT, "gone"))) + assert serve.read_unless_private(ordinary, roots) == b"# plain\n" + monkeypatch.undo() + console_access.remove_private_copy(copy) + + +@pytest.mark.parametrize("file", ["missing", "present", "a copy"]) +def test_an_entrys_payload_comes_before_its_guarded_file( + tmp_path, file) -> None: + """Copilot's review at af2a2efb ("previously missed"): an entry with an + in-memory payload serves that payload first, as `SnapshotEntry. + read_bytes` does, whether its file is missing, present, or even a + private copy; only the file fallback is guarded. An entry with no + payload reads its file through the guard.""" + import types + + from opendox import console_access, default_registry, serve + + state = _state(tmp_path) + copy = _write(state) + path = {"missing": tmp_path / "missing.json", + "present": tmp_path / "present.json", + "a copy": copy.path}[file] + if file == "present": + path.write_bytes(b'{"from": "file"}') + handler = types.SimpleNamespace( + server=types.SimpleNamespace(private_roots=(copy.path.parent,))) + with_payload = default_registry.SnapshotEntry( + repository="fixture", snapshot_path=path, payload=b'{"from": "payload"}') + assert serve.DashboardHandler._entry_bytes(handler, with_payload) == \ + b'{"from": "payload"}' + without = default_registry.SnapshotEntry(repository="fixture", snapshot_path=path) + expected = {"missing": None, "present": b'{"from": "file"}', "a copy": None}[file] + assert serve.DashboardHandler._entry_bytes(handler, without) == expected + console_access.remove_private_copy(copy) + + +def test_a_file_whose_head_cannot_be_read_is_denied(tmp_path, monkeypatch) -> None: + """r4179239424, by content: a regular file whose head cannot be read + cannot be cleared of holding a console record, so the read is denied. + A file that holds none is read as before.""" + from opendox import console_access, serve + + state = _state(tmp_path) + copy = _write(state) + ordinary = tmp_path / "ordinary.md" + ordinary.write_text("# plain\n", encoding="utf-8") + roots = (copy.path.parent,) + assert serve.read_unless_private(ordinary, roots) == b"# plain\n" + + def unreadable(*args, **kwargs): + raise OSError(errno.EIO, os.strerror(errno.EIO)) + + monkeypatch.setattr(console_access.os, "pread", unreadable) + assert serve.read_unless_private(ordinary, roots) is None + monkeypatch.undo() + console_access.remove_private_copy(copy) From d466c1d28dab2b6595892d227d99028e90a6f88b Mon Sep 17 00:00:00 2001 From: Brett Heap <1513478+brettheap@users.noreply.github.com> Date: Sun, 4 Oct 2026 21:52:17 +0000 Subject: [PATCH 85/88] T104: no test server outlives its case or its run The holder found orphaned `python -m opendox.serve` children of the T104 cases on the machine, hours old, with PPID 1. Each was plane B of test_b1_a_tokenless_sibling_plane_never_serves_another_planes_copy, left by a mutant run: where a mutant made B serve instead of refusing, the case failed on `b.wait(60)`, and its `finally` stopped plane A only. A run killed outside pytest's control left its children too. Every child `_adv_serve` starts now: - runs in its own session, so its process group is its own (`start_new_session`); - is registered, and an autouse fixture reaps every registered child at teardown, whether the case passed, failed or raised: SIGTERM to the group, a bounded wait, then SIGKILL (`_reap`); - on Linux, gets SIGTERM from the kernel if the test process dies first (`PR_SET_PDEATHSIG`, `_child_setup`), since a killed run runs no teardown. Cases: - test_a_server_left_running_is_reaped_with_its_group: a serving child is in its own group and is reaped with its copy removed and its port freed; a child that ignores SIGTERM is killed after the bounded wait; - test_a_server_outlives_no_killed_run: an intermediate process starts a child the way `_adv_serve` does and is SIGKILLed, and the child ends with it. Checked against mutant M44 in a throwaway worktree (orphan-check.txt): the B1 case fails, as it must, and no server from that run survives it. Arc: neutral-product-standalone-operability Lane: openxfactory-4 (openXfactory-4-openDox_extraction) Co-Authored-By: Claude Opus 5.5 (1M context) --- tests/test_console_token_delivery.py | 144 ++++++++++++++++++++++++++- 1 file changed, 139 insertions(+), 5 deletions(-) diff --git a/tests/test_console_token_delivery.py b/tests/test_console_token_delivery.py index 10bfd01d..0bf1149a 100644 --- a/tests/test_console_token_delivery.py +++ b/tests/test_console_token_delivery.py @@ -38,6 +38,7 @@ import socket import socketserver import stat +import sys import threading import urllib.parse from pathlib import Path @@ -2604,9 +2605,12 @@ def _adv_serve(tmp: Path, repo: Path, state: Path, port: int, name: str, cmd = [sys.executable, "-m", "opendox.serve", *argv] else: cmd = [sys.executable, "-c", code + f"\nsys.exit(serve.main({argv!r}))"] + parent = os.getpid() proc = subprocess.Popen(cmd, cwd=tmp, env=_adv_env(state), stdout=out.open("w"), stderr=subprocess.STDOUT, - preexec_fn=_default_stops) + start_new_session=True, + preexec_fn=lambda: _child_setup(parent)) + _SPAWNED.append(proc) for _ in range(300): if "serving ideation dashboard" in out.read_text() or proc.poll() is not None: break @@ -2614,11 +2618,31 @@ def _adv_serve(tmp: Path, repo: Path, state: Path, port: int, name: str, return proc, out -def _default_stops() -> None: - """A child started from a background job inherits SIGINT ignored, and - one under `nohup` SIGHUP: give it a terminal's, so its stops are read.""" +#: Every server child `_adv_serve` started, reaped after each case +#: (`_reap_spawned_servers`), whether the case passed, failed or raised. +_SPAWNED: list = [] +#: `prctl(2)`, loaded in the test process, before any fork, on Linux only. +_PRCTL = None +if sys.platform.startswith("linux"): + import ctypes as _ctypes + + with contextlib.suppress(OSError, AttributeError): + _PRCTL = _ctypes.CDLL(None, use_errno=True).prctl +_PR_SET_PDEATHSIG = 1 + + +def _child_setup(parent: int) -> None: + """In the child, before it runs. A child started from a background job + inherits SIGINT ignored, and one under `nohup` SIGHUP: give it a + terminal's, so its stops are read. And on Linux, have the kernel send it + SIGTERM if the test process dies first (a killed run runs no teardown), + so no server outlives the run that started it.""" signal.signal(signal.SIGINT, signal.default_int_handler) signal.signal(signal.SIGHUP, signal.SIG_DFL) + if _PRCTL is not None: + _PRCTL(_PR_SET_PDEATHSIG, int(signal.SIGTERM), 0, 0, 0) + if os.getppid() != parent: # the parent died before prctl + os._exit(1) def _adv_stop(proc) -> int: @@ -2627,6 +2651,34 @@ def _adv_stop(proc) -> int: return proc.wait(30) +def _reap(proc, wait: float = 15) -> None: + """Stop `proc` and its process group (its own session, so nothing else + is in it): SIGTERM, a bounded wait, then SIGKILL. A child that already + ended is only collected.""" + import subprocess + + if proc.poll() is not None: + return + for sent in (signal.SIGTERM, signal.SIGKILL): + with contextlib.suppress(ProcessLookupError, PermissionError): + os.killpg(proc.pid, sent) + try: + proc.wait(wait) + return + except subprocess.TimeoutExpired: + continue + + +@pytest.fixture(autouse=True) +def _reap_spawned_servers(): + """No server child outlives its case: every one `_adv_serve` started is + stopped with its process group at teardown, the failing cases' included + (a refusal that did not come, say, leaves a plane serving).""" + yield + while _SPAWNED: + _reap(_SPAWNED.pop()) + + def test_b1_a_tokenless_sibling_plane_never_serves_another_planes_copy( tmp_path) -> None: """B1, as the reviewer staged it, with two real planes. Plane A (a git @@ -2655,7 +2707,7 @@ def test_b1_a_tokenless_sibling_plane_never_serves_another_planes_copy( assert _port_is_free(port_b), "the refused plane kept its socket" assert a.poll() is None, "plane A went down with B's refusal" finally: - _adv_stop(a) + _adv_stop(a) # B, if it never refused, is reaped at teardown def test_a_tokenless_standalone_plane_keeps_the_boundary( @@ -3545,3 +3597,85 @@ def unreadable(*args, **kwargs): assert serve.read_unless_private(ordinary, roots) is None monkeypatch.undo() console_access.remove_private_copy(copy) + + +# --------------------------------------------------------------------------- +# 20 — no test server outlives its case, or its run +# --------------------------------------------------------------------------- + +def test_a_server_left_running_is_reaped_with_its_group(tmp_path) -> None: + """The teardown's reaper: a server child still serving is stopped by + its process group, SIGTERM first, so it removes its copy and frees its + port. A child that ignores SIGTERM is killed after the bounded wait.""" + import subprocess + + repo = _adv_repo(tmp_path / "r") + state = _state(tmp_path) + port = _free_port() + proc, out = _adv_serve(tmp_path, repo, state, port, "left") + assert proc.poll() is None, out.read_text() + assert os.getpgid(proc.pid) == proc.pid, "the server is not in its own group" + assert (state / "console" / f"{port}.html").exists() + _reap(proc) + assert proc.returncode == 0, out.read_text() + assert not (state / "console" / f"{port}.html").exists() + assert _port_is_free(port) + stubborn = subprocess.Popen( + [sys.executable, "-c", + "import signal, time; signal.signal(signal.SIGTERM, signal.SIG_IGN); " + "print('ready', flush=True); time.sleep(120)"], + stdout=subprocess.PIPE, start_new_session=True) + try: + assert stubborn.stdout.readline().strip() == b"ready" + _reap(stubborn, wait=2) + assert stubborn.returncode == -signal.SIGKILL + finally: + if stubborn.poll() is None: + stubborn.kill() + stubborn.wait(10) + stubborn.stdout.close() + + +def test_a_server_outlives_no_killed_run(tmp_path) -> None: + """A run that is killed runs no teardown. On Linux the kernel stops a + server child when the process that started it dies (`_child_setup`'s + parent-death signal): here an intermediate process starts a child the + way `_adv_serve` does and is SIGKILLed, and the child ends with it. + Elsewhere the teardown alone answers for it.""" + import subprocess + import time + + if _PRCTL is None: + return # no parent-death signal on this platform + script = ( + "import os, subprocess, sys, time\n" + f"sys.path.insert(0, {str(Path(__file__).resolve().parent)!r})\n" + "import test_console_token_delivery as t\n" + "parent = os.getpid()\n" + "child = subprocess.Popen([sys.executable, '-c', 'import time; time.sleep(120)'],\n" + " start_new_session=True, preexec_fn=lambda: t._child_setup(parent))\n" + "print(child.pid, flush=True)\n" + "time.sleep(120)\n") + middle = subprocess.Popen([sys.executable, "-c", script], + stdout=subprocess.PIPE, cwd=tmp_path, + env=_adv_env()) + try: + child = int(middle.stdout.readline()) + middle.kill() + middle.wait(10) + for _ in range(150): + try: + state = Path(f"/proc/{child}/stat").read_text().rsplit(")", 1)[1].split()[0] + except FileNotFoundError: + break + if state == "Z": + break + time.sleep(0.1) + else: + os.kill(child, signal.SIGKILL) + pytest.fail("the child outlived the killed run that started it") + finally: + if middle.poll() is None: + middle.kill() + middle.wait(10) + middle.stdout.close() From 1e114a195ccee3fb0734e798876a9d2191a3e832 Mon Sep 17 00:00:00 2001 From: Brett Heap <1513478+brettheap@users.noreply.github.com> Date: Sun, 4 Oct 2026 23:20:08 +0000 Subject: [PATCH 86/88] T104: a partly written copy is refused by its place Mutant run 23 (83 mutants at d466c1d2) killed 82; M38 lived, where the identity match against the copies' directory finds nothing. Since round 11 judges every file by what it holds first, every whole copy was already refused by its record, and nothing asked for the identity match on its own. It answers for a file in the copies' directory that holds no whole record: a copy caught part way through its write has the token and not yet the end of its element. test_a_partly_written_copy_is_refused_by_its_place hard-links such a file under a served root and checks that it is never read out, and that its content alone would not have refused it. Arc: neutral-product-standalone-operability Lane: openxfactory-4 (openXfactory-4-openDox_extraction) Co-Authored-By: Claude Opus 5.5 (1M context) --- tests/test_console_token_delivery.py | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/tests/test_console_token_delivery.py b/tests/test_console_token_delivery.py index 0bf1149a..070a8489 100644 --- a/tests/test_console_token_delivery.py +++ b/tests/test_console_token_delivery.py @@ -3679,3 +3679,28 @@ def test_a_server_outlives_no_killed_run(tmp_path) -> None: middle.kill() middle.wait(10) middle.stdout.close() + + +def test_a_partly_written_copy_is_refused_by_its_place(tmp_path) -> None: + """A file in the copies' directory is refused by its identity even when + it holds no whole record: a copy caught part way through its write has + the token and not yet the end of its element, so it cannot be known by + what it holds, only by where it is. Hard-linked under a served root, it + is still never read out.""" + from opendox import console_access, serve + + state = _state(tmp_path) + copy = _write(state) + text = copy.path.read_text(encoding="utf-8") + token = console_access.read_private_copy(copy.path)["console_token"] + partial = copy.path.parent / f".{copy.path.name}.opendox-424242" + partial.write_text(text[:text.index(token) + len(token)], encoding="utf-8") + partial.chmod(0o600) + served = tmp_path / "served" + served.mkdir() + os.link(partial, served / "partial.md") + with open(served / "partial.md", "rb") as stream: + assert not console_access._carries_a_console_record( + stream.fileno(), os.fstat(stream.fileno())), "the case is vacuous" + assert serve.read_unless_private(served / "partial.md", (copy.path.parent,)) is None + console_access.remove_private_copy(copy) From a2e3665280d618a4c60e479387878c7164a39da2 Mon Sep 17 00:00:00 2001 From: Brett Heap <1513478+brettheap@users.noreply.github.com> Date: Sun, 4 Oct 2026 23:48:23 +0000 Subject: [PATCH 87/88] T104 fix round 12: a copy is written from a marker, and what is sent is judged as it is read (Copilot review) Copilot's review at 1e114a19 (5408767954), r4179793524. Round 11 knew a copy in another state directory by its whole record, so another plane's temporary file, part written with the token in its meta refresh and no record yet, passed the static handler, /source and /snapshot.json. A file that grew after it was judged passed as well, since the stdlib copies a static file to its end as it is when read. - The writer now begins every copy with COPY_MARKER, an HTML comment, before any byte of the token. A copy is written from its start, so any part of one that holds a token byte holds the whole marker first. `is_copy_bytes` knows a copy by the marker or by its whole record; fewer bytes than the marker hold no token. - `read_unless_private` reads first and judges what it read (`is_copy_bytes`) as well as the file by its descriptor, so a file that holds no token when judged cannot hand one out after. - The static handler's `copyfile` sends no more than the file's length when it was judged, and judges the body's first bytes, read before anything is sent: a copy's are never sent, and the connection is closed, as the backstop closes it. The judged length is reset for every request. Cases (review12-red.txt): - test_a_copy_starts_with_its_marker_before_any_token_byte; - test_another_state_directorys_partial_copy_is_never_served: Copilot's two-state-directory layout, through the static handler, /source and /snapshot.json, GET and HEAD; - test_a_file_that_grows_after_its_static_check_never_sends_a_token: the file grows right after the backstop's check; - test_a_file_that_grows_after_its_read_check_never_returns_a_token; - test_a_copy_replaced_after_its_read_is_never_returned, which pins the read-first design (at 1e114a19 it fails only because that reader judged before it read, so the staged replacement never happened). Each failed first at 1e114a19. The identity case for a file in the copies' directory is recast as defence in depth (test_a_file_in_the_copies_directory_is_refused_by_its_place): with the marker, a partly written copy is known by its bytes, so the case now holds a token-bearing file the marker cannot recognize. It still kills M38. serve.py imports os, for the judged length. Mutants M57, M57b, M58, M58b and M59 join the run (88). Arc: neutral-product-standalone-operability Lane: openxfactory-4 (openXfactory-4-openDox_extraction) Co-Authored-By: Claude Opus 5.5 (1M context) --- src/opendox/console_access.py | 51 ++++-- src/opendox/serve.py | 65 ++++++- tests/test_console_token_delivery.py | 253 +++++++++++++++++++++++++-- 3 files changed, 336 insertions(+), 33 deletions(-) diff --git a/src/opendox/console_access.py b/src/opendox/console_access.py index 92c3e49c..c5721447 100644 --- a/src/opendox/console_access.py +++ b/src/opendox/console_access.py @@ -118,8 +118,8 @@ "CONSOLE_DIRNAME", "ConsoleAccessRefused", "ConsoleTerminated", "DELIVERY_CAPABILITIES", "DELIVERY_OPENED_URL", "FRAGMENT_KEY", "PrivateCopy", "RECORD_ELEMENT_ID", - "RECORD_KIND", "UNOPENABLE_HINT", "deferred_termination", "delivery_for", - "guard_private_roots", + "COPY_MARKER", "RECORD_KIND", "UNOPENABLE_HINT", "deferred_termination", + "delivery_for", "guard_private_roots", "is_copy_bytes", "is_private_file", "needs_copy", "opens_a_private_file", "opened_url", "private_copy_path", "publish", "read_private_copy", "remove_private_copy", "terminate_as_interrupt", "unsupported_platform", @@ -140,6 +140,14 @@ RECORD_KIND = "opendox-console-access" RECORD_SCHEMA_VERSION = 1 RECORD_ELEMENT_ID = "opendox-console" +#: The copy's FIRST bytes, before any byte of the token (Copilot at +#: openDox-code#84, r4179793524). A copy is written from its start, so any +#: part of one that holds a byte of the token holds this whole line first: a +#: file that begins with it is a copy, written in full or caught part way +#: (`is_copy_bytes`), and a file shorter than it holds no token yet. An HTML +#: comment, which a browser reads before the doctype without effect. +COPY_MARKER = (b"\n") #: The one mode a private copy may have. PRIVATE_MODE = 0o600 #: The one mode the copies' directory, `console/`, may have (#1144 12.4a: the @@ -585,7 +593,8 @@ def _record_json(record: Mapping[str, Any]) -> str: def _opener_html(record: Mapping[str, Any]) -> str: target = html.escape(str(record["opened_url"]), quote=True) return ( - "\n" + COPY_MARKER.decode("ascii") + + "\n" '\n' "\n" '\n' @@ -1168,29 +1177,43 @@ def terminate_as_interrupt(enabled: bool): _held.update(pending=None, stopping=False) +def is_copy_bytes(data: bytes) -> bool: + """Whether `data`, a file's bytes from its start, are a console token's + private copy's: they begin with its `COPY_MARKER`, written in full or + caught part way through its write (Copilot at openDox-code#84, + r4179793524), or they carry a whole console record (`RECORD_KIND`) in + the element a copy keeps it in. Fewer bytes than the marker hold no + token, and are not a copy's.""" + if data.startswith(COPY_MARKER): + return True + found = _RECORD_PATTERN.search(data[:_READ_LIMIT].decode("utf-8", "replace")) + if found is None: + return False + try: + record = json.loads(found.group("record")) + except ValueError: + return False + return isinstance(record, dict) and record.get("kind") == RECORD_KIND + + def _carries_a_console_record(handle: int, info: os.stat_result) -> bool: """Whether the regular file open on `handle` IS a console token's private - copy by what it holds: a console record (`RECORD_KIND`) in the element a - copy keeps it in, wherever the file lies and whatever its name. + copy by what it holds (`is_copy_bytes`), wherever the file lies and + whatever its name, a copy still being written included. Read with `pread`, from the descriptor already open, so it needs no new descriptor and does not move the offset the caller then reads from. A regular file whose head cannot be read is judged to be a copy: a check - that cannot be made denies, never allows.""" + that cannot be made denies, never allows. What the caller then SENDS is + judged again by its own bytes (`serve.read_unless_private`, + `serve.DashboardHandler.copyfile`), since a file can grow after this.""" if not stat.S_ISREG(info.st_mode): return False try: head = os.pread(handle, _READ_LIMIT, 0) except OSError: return True - found = _RECORD_PATTERN.search(head.decode("utf-8", "replace")) - if found is None: - return False - try: - record = json.loads(found.group("record")) - except ValueError: - return False - return isinstance(record, dict) and record.get("kind") == RECORD_KIND + return is_copy_bytes(head) def is_private_file(handle: int, private_roots: Iterable[Path | str]) -> bool: diff --git a/src/opendox/serve.py b/src/opendox/serve.py index f20e076a..fae584f0 100644 --- a/src/opendox/serve.py +++ b/src/opendox/serve.py @@ -105,6 +105,7 @@ import functools import http.server import json +import os import secrets import socket import subprocess @@ -822,12 +823,21 @@ def read_unless_private(path: Path | str, private_roots) -> bytes | None: resolved again on every request, and could be re-pointed at the state directory after the copy was published; a hard link reaches the copy by another name. The file actually opened is judged, so neither is served. - With no private root (a host's plane) it reads as before.""" + With no private root (a host's plane) it reads as before. + + THE BYTES READ ARE JUDGED, NOT ONLY THE FILE (Copilot at + openDox-code#84, r4179793524). A copy being written in another state + directory grows: judged before the read, it could hold no token yet, and + hold one by the time it was read. So the file is read first, and what + was read is judged by its own bytes (`console_access.is_copy_bytes`) as + well as the file by its identity.""" with open(path, "rb") as stream: - if private_roots and console_access.is_private_file( - stream.fileno(), private_roots): + data = stream.read() + if private_roots and ( + console_access.is_copy_bytes(data) + or console_access.is_private_file(stream.fileno(), private_roots)): return None - return stream.read() + return data def resolve_source_path(checkout_root: Path, url_tail: str) -> Path | None: @@ -1388,6 +1398,9 @@ def send_head(self): are already sent by then, so it is closed unread and its bytes are never written.""" private = getattr(self.server, "private_roots", ()) + # Judged afresh for every request: a length judged for an earlier one + # never bounds this one's body, should a handler ever serve several. + self._judged_length = None if private: path = Path(self.translate_path(self.path)) judged = [path] @@ -1411,8 +1424,52 @@ def send_head(self): stream.close() self.close_connection = True # its promised body never comes return None + if handle is not None: + # What `copyfile` may send: the file as long as it was when + # judged, and its own first bytes judged again as they are + # sent (`copyfile`). + self._judged_length = os.fstat(handle).st_size return stream + def copyfile(self, source, outputfile): + """The static body, as `SimpleHTTPRequestHandler` copies it, except + on a plane that marked a private-copy directory. + + A FILE CAN GROW AFTER IT WAS JUDGED (Copilot at openDox-code#84, + r4179793524). A copy being written in another state directory holds + no token in its first bytes, and the stdlib copies to the end of the + file as it is when it reads, not as it was when `send_head` judged it. + So no more than the judged length is sent, and the body's first bytes, + read before anything is sent, are judged by what they are: a copy's + (`console_access.is_copy_bytes`) are never sent, and the connection + is closed, as the backstop closes it. Bytes shorter than a copy's + marker hold no token, and only they are sent.""" + limit = getattr(self, "_judged_length", None) + self._judged_length = None + if limit is None: + return super().copyfile(source, outputfile) + need = min(limit, len(console_access.COPY_MARKER)) + head = b"" + while len(head) < need: + chunk = source.read(need - len(head)) + if not chunk: + break + head += chunk + if console_access.is_copy_bytes(head): + self.close_connection = True + return None + outputfile.write(head) + remaining = limit - len(head) + if len(head) < need: + return None + while remaining > 0: + chunk = source.read(min(64 * 1024, remaining)) + if not chunk: + break + outputfile.write(chunk) + remaining -= len(chunk) + return None + def do_GET(self): # noqa: N802 if not self._route(head_only=False): super().do_GET() diff --git a/tests/test_console_token_delivery.py b/tests/test_console_token_delivery.py index 070a8489..a834e471 100644 --- a/tests/test_console_token_delivery.py +++ b/tests/test_console_token_delivery.py @@ -3681,26 +3681,249 @@ def test_a_server_outlives_no_killed_run(tmp_path) -> None: middle.stdout.close() -def test_a_partly_written_copy_is_refused_by_its_place(tmp_path) -> None: - """A file in the copies' directory is refused by its identity even when - it holds no whole record: a copy caught part way through its write has - the token and not yet the end of its element, so it cannot be known by - what it holds, only by where it is. Hard-linked under a served root, it - is still never read out.""" +def test_a_file_in_the_copies_directory_is_refused_by_its_place(tmp_path) -> None: + """Defence in depth: a file in the copies' directory is refused by its + identity whatever it holds, even where its bytes are not recognized as a + copy's (here a token-bearing line with no marker and no record). + Hard-linked under a served root, it is still never read out.""" from opendox import console_access, serve state = _state(tmp_path) copy = _write(state) - text = copy.path.read_text(encoding="utf-8") token = console_access.read_private_copy(copy.path)["console_token"] - partial = copy.path.parent / f".{copy.path.name}.opendox-424242" - partial.write_text(text[:text.index(token) + len(token)], encoding="utf-8") - partial.chmod(0o600) + stray = copy.path.parent / f".{copy.path.name}.opendox-424242" + stray.write_text(f"\n", encoding="utf-8") + stray.chmod(0o600) + assert token in stray.read_text(encoding="utf-8") served = tmp_path / "served" served.mkdir() - os.link(partial, served / "partial.md") - with open(served / "partial.md", "rb") as stream: - assert not console_access._carries_a_console_record( - stream.fileno(), os.fstat(stream.fileno())), "the case is vacuous" - assert serve.read_unless_private(served / "partial.md", (copy.path.parent,)) is None + os.link(stray, served / "stray.md") + with open(served / "stray.md", "rb") as stream: + assert not console_access.is_copy_bytes(stream.read()), "the case is vacuous" + assert serve.read_unless_private(served / "stray.md", (copy.path.parent,)) is None + console_access.remove_private_copy(copy) + + +# --------------------------------------------------------------------------- +# 21 — Copilot's review at 1e114a19 (r4179793524): a copy caught part way +# through its write, in another state directory, and a file that grows +# after it was judged +# --------------------------------------------------------------------------- + +def _partial_copy_bytes(tmp_path: Path) -> tuple[bytes, str]: + """A real copy's bytes, cut just after the token's first appearance (the + meta refresh), before the record's element is written: what another + plane's writer leaves for a moment in its temporary file.""" + from opendox import console_access + + scratch = _state(tmp_path / "scratch") + copy = _write(scratch, port=7) + data = copy.path.read_bytes() + token = console_access.read_private_copy(copy.path)["console_token"] + console_access.remove_private_copy(copy) + cut = data.index(token.encode()) + len(token) + assert b"" not in data[:cut], "the cut is not part way" + return data[:cut], token + + +def test_a_copy_starts_with_its_marker_before_any_token_byte(tmp_path) -> None: + """r4179793524: every copy is written from `COPY_MARKER`, so any part of + one that holds a byte of the token holds the whole marker first, and is + known for a copy (`is_copy_bytes`); fewer bytes than the marker hold no + token and are not.""" + from opendox import console_access + + copy = _write(_state(tmp_path)) + data = copy.path.read_bytes() + token = console_access.read_private_copy(copy.path)["console_token"].encode() + assert data.startswith(console_access.COPY_MARKER) + assert data.index(token) >= len(console_access.COPY_MARKER) + for cut in range(len(console_access.COPY_MARKER), len(data) + 1, 37): + assert console_access.is_copy_bytes(data[:cut]), cut + for cut in range(len(console_access.COPY_MARKER)): + assert token not in data[:cut] + assert not console_access.is_copy_bytes(data[:cut]), cut + assert not console_access.is_copy_bytes(b"# a document\n") console_access.remove_private_copy(copy) + + +def test_another_state_directorys_partial_copy_is_never_served( + tmp_path, monkeypatch, standalone_profile) -> None: + """r4179793524, Copilot's layout: plane B's writer has its temporary file + part written, the token in its meta refresh and no record yet, in B's own + state directory. Plane A's `--web-dir` links there, A's checkout holds a + hard link to it, and A's snapshot file is one too. The static handler, + `/source` and `/snapshot.json` all refuse it, for GET and HEAD.""" + import shutil as _shutil + + from opendox import console_access, serve + + partial, token = _partial_copy_bytes(tmp_path) + web = tmp_path / "web" + _shutil.copytree(WEB, web) + state_a = _state(tmp_path / "a") + state_b = _state(tmp_path / "b") + console_b = state_b / console_access.CONSOLE_DIRNAME + console_b.mkdir(mode=0o700) + temporary = console_b / ".9.html.opendox-4242" + temporary.write_bytes(partial) + temporary.chmod(0o600) + (web / "state-b").symlink_to(state_b) + httpd, repo, worker = _guarded_plane(tmp_path, monkeypatch, web=web) + try: + base = httpd.server_address[:2] + own = console_access.publish( + httpd, page_url=serve.server_url(httpd, "/index.html"), + env={"OPENDOX_STATE_DIR": str(state_a)}) + os.link(temporary, repo / "partial.md") + snapshot = tmp_path / "snapshot.json" + snapshot.unlink() + os.link(temporary, snapshot) + for path in (f"/state-b/console/{temporary.name}", "/source/partial.md", + "/snapshot.json"): + for method in ("GET", "HEAD"): + status, headers, raw = _call(base, method, path) + assert token.encode() not in raw, (method, path, status) + assert all(token not in str(v) for v in headers.values()), path + assert status != 200, (method, path, status) + assert token.encode() not in _raw_get(base, f"/state-b/console/{temporary.name}") + assert _call(base, "GET", "/index.html")[0] == 200 + console_access.remove_private_copy(own) + finally: + _stop_plane(httpd, worker) + + +def test_a_file_that_grows_after_its_static_check_never_sends_a_token( + tmp_path, monkeypatch, standalone_profile) -> None: + """r4179793524, a file that grows: when the static handler judges it, both + before the stdlib opens it and after, the other plane's temporary file + holds only the start of the marker, and no token; it grows to hold one + right after the second judgment, before the body is copied. The body + sent is judged again as it is read, and never carries the token, whatever + reaches the wire.""" + import shutil as _shutil + + from opendox import console_access + + partial, token = _partial_copy_bytes(tmp_path) + first = partial[:10] + web = tmp_path / "web" + _shutil.copytree(WEB, web) + state_b = _state(tmp_path / "b") + console_b = state_b / console_access.CONSOLE_DIRNAME + console_b.mkdir(mode=0o700) + growing = console_b / ".9.html.opendox-4242" + growing.write_bytes(first) + growing.chmod(0o600) + (web / "state-b").symlink_to(state_b) + real = console_access.is_private_file + judged: list[bool] = [] + grown: list[int] = [] + + def then_grow(handle, roots): + answer = real(handle, roots) + info = os.fstat(handle) + if (info.st_dev, info.st_ino) == (growing.stat().st_dev, + growing.stat().st_ino): + judged.append(answer) + if len(judged) == 2 and not grown: # after the backstop + grown.append(1) + with open(growing, "ab") as more: + more.write(partial[len(first):]) + return answer + + httpd, _repo, worker = _guarded_plane(tmp_path, monkeypatch, web=web) + try: + base = httpd.server_address[:2] + own = console_access.publish( + httpd, page_url="http://127.0.0.1:%d/index.html" % base[1], + env={"OPENDOX_STATE_DIR": str(_state(tmp_path / "a"))}) + monkeypatch.setattr(console_access, "is_private_file", then_grow) + raw = _raw_get(base, f"/state-b/console/{growing.name}") + assert grown and judged == [False, False], ("the growth was never " + "staged after both checks", judged) + assert token.encode() not in raw + console_access.remove_private_copy(own) + finally: + _stop_plane(httpd, worker) + + +def test_a_copy_replaced_after_its_read_is_never_returned( + tmp_path, monkeypatch) -> None: + """r4179793524, the other way round: the bytes read are a copy's, and the + file is emptied right after the read, before anything judges it by its + descriptor. What was read is judged too, so the copy's bytes are never + returned.""" + from opendox import console_access, serve + + own = _write(_state(tmp_path / "a")) + other = _write(_state(tmp_path / "b"), port=9) + token = console_access.read_private_copy(other.path)["console_token"] + served = tmp_path / "served.json" + os.link(other.path, served) + real_open = open + emptied: list[int] = [] + + class _EmptiedAfterRead: + def __init__(self, stream): + self.stream = stream + + def __enter__(self): + return self + + def __exit__(self, *exc): + self.stream.close() + + def fileno(self): + return self.stream.fileno() + + def read(self, *args): + data = self.stream.read(*args) + os.truncate(served, 0) # emptied right after the read + emptied.append(1) + return data + + def opening(path, mode="r", *args, **kwargs): + stream = real_open(path, mode, *args, **kwargs) + return _EmptiedAfterRead(stream) if str(path) == str(served) else stream + + monkeypatch.setattr("builtins.open", opening) + try: + data = serve.read_unless_private(served, (own.path.parent,)) + finally: + monkeypatch.undo() + assert emptied, "the replacement was never staged" + assert data is None or token.encode() not in data + console_access.remove_private_copy(own) + console_access.remove_private_copy(other) + + +def test_a_file_that_grows_after_its_read_check_never_returns_a_token( + tmp_path, monkeypatch) -> None: + """r4179793524, for `/source` and `/snapshot.json`'s reader: the file is + read first and the bytes read are judged, so a file that holds no token + when it is judged cannot hand one out after.""" + from opendox import console_access, serve + + partial, token = _partial_copy_bytes(tmp_path) + own = _write(_state(tmp_path / "a")) + growing = tmp_path / "growing.json" + growing.write_bytes(partial[:10]) + real = console_access.is_private_file + grown: list[int] = [] + + def then_grow(handle, roots): + answer = real(handle, roots) + if not grown: + grown.append(1) + with open(growing, "ab") as more: + more.write(partial[10:]) + return answer + + monkeypatch.setattr(console_access, "is_private_file", then_grow) + data = serve.read_unless_private(growing, (own.path.parent,)) + assert grown, "the growth was never staged" + assert data is None or token.encode() not in data + monkeypatch.undo() + assert serve.read_unless_private(growing, (own.path.parent,)) is None + console_access.remove_private_copy(own) From c5fcdfa4eba3ee661e9cc2557e25a1843aaab778 Mon Sep 17 00:00:00 2001 From: Brett Heap <1513478+brettheap@users.noreply.github.com> Date: Mon, 5 Oct 2026 01:23:27 +0000 Subject: [PATCH 88/88] T104 fix round 13: the console page's URL is judged as a browser reads it (Copilot review) Copilot's review at a2e36652 (5409152074), r4180089809. `urlsplit` reads `http://evil.example\@127.0.0.1:8080/index.html` as user information at 127.0.0.1, but a browser takes the backslash for a slash and navigates to evil.example, whose page could read the token's fragment. The built-in entry points build their own URLs, but write_private_copy and opened_url are public, and their loopback rule did not hold its contract. `_refuse_page_url` now requires the authority to be exactly a loopback host, spelled as `serve.server_url` spells it, and an optional port of at most 65535 (`_LOOPBACK_AUTHORITY`): no user information and no second port. A backslash or a control character anywhere in the URL, which a browser rewrites or strips, is refused. Cases (review13-red.txt; 7 of the 8 bad URLs failed first at a2e36652, the backslash after the port was already refused): - test_a_page_url_a_browser_reads_as_another_host_is_refused: Copilot's backslash-userinfo URL first, through opened_url and write_private_copy, with nothing written; - test_every_loopback_page_url_a_plane_announces_is_accepted: 127.0.0.1, [::1] and localhost, with and without a port. Mutants M61 (the authority is not judged exactly) and M61b (a backslash or a control character passes) join the run (90). Arc: neutral-product-standalone-operability Lane: openxfactory-4 (openXfactory-4-openDox_extraction) Co-Authored-By: Claude Opus 5.5 (1M context) --- src/opendox/console_access.py | 24 ++++++++++++- tests/test_console_token_delivery.py | 50 ++++++++++++++++++++++++++++ 2 files changed, 73 insertions(+), 1 deletion(-) diff --git a/src/opendox/console_access.py b/src/opendox/console_access.py index c5721447..694f5dfa 100644 --- a/src/opendox/console_access.py +++ b/src/opendox/console_access.py @@ -173,6 +173,11 @@ r'') _LOOPBACK_HOSTS = frozenset({"127.0.0.1", "::1", "localhost"}) +#: The one shape a console page's AUTHORITY may have: a loopback host, +#: spelled as `server_url` spells it, and an optional port. No user +#: information, and nothing a browser reads as the authority's end (`\\`). +_LOOPBACK_AUTHORITY = re.compile( + r"(?:127\.0\.0\.1|localhost|\[::1\])(?::(?P[0-9]{1,5}))?") #: The names a publication may sweep when their servers are gone #: (`_sweep_stale_copies`): a copy, a writer's temporary file, and a remover's #: taken name. Each holds a token, and nothing else is ever touched. @@ -289,8 +294,25 @@ def delivery_for(profile: Any) -> str: def _refuse_page_url(page_url: str) -> None: + """The console page must be this machine's own plane, as a BROWSER reads + the URL, not only as `urlsplit` does (Copilot at openDox-code#84, + r4180089809). A browser takes `\\` for `/`, so in + `http://evil.example\\@127.0.0.1:8080/` it sees the host `evil.example` + where `urlsplit` sees user information and `127.0.0.1`, and the token's + fragment would be handed to the remote page. So the authority must be a + loopback host and an optional port, exactly (`_LOOPBACK_AUTHORITY`), and + a backslash or a control character anywhere, which a browser rewrites or + strips, is refused.""" + if "\\" in page_url or any(ord(c) < 0x20 or ord(c) == 0x7F for c in page_url): + raise ConsoleAccessRefused( + f"the console page {page_url!r} holds a backslash or a control " + "character, which a browser reads differently, so it is not " + "certainly this machine's own plane") parts = urllib.parse.urlsplit(page_url) - if parts.scheme != "http" or parts.hostname not in _LOOPBACK_HOSTS: + authority = _LOOPBACK_AUTHORITY.fullmatch(parts.netloc) + if (parts.scheme != "http" or parts.hostname not in _LOOPBACK_HOSTS + or authority is None + or int(authority.group("port") or 80) > 65535): raise ConsoleAccessRefused( f"the console page {page_url!r} is not a loopback http URL, and a " "console token is only ever opened on this machine's own plane") diff --git a/tests/test_console_token_delivery.py b/tests/test_console_token_delivery.py index a834e471..2c22d3b9 100644 --- a/tests/test_console_token_delivery.py +++ b/tests/test_console_token_delivery.py @@ -3927,3 +3927,53 @@ def then_grow(handle, roots): monkeypatch.undo() assert serve.read_unless_private(growing, (own.path.parent,)) is None console_access.remove_private_copy(own) + + +# --------------------------------------------------------------------------- +# 22 — Copilot's review at a2e36652 (r4180089809): the page URL is judged +# as a browser reads it +# --------------------------------------------------------------------------- + +@pytest.mark.parametrize("bad", [ + "http://evil.example\\@127.0.0.1:8080/index.html", + "http://127.0.0.1:8080\\@evil.example/index.html", + "http://evil.example@127.0.0.1:8080/index.html", + "http://user:pass@127.0.0.1:8080/index.html", + "http://127.0.0.1:8080/\\\\evil.example/index.html", + "http://127.0.0.1:8080/\tindex.html", + "http://127.0.0.1:99999/index.html", + "http://127.0.0.1:8080:9/index.html", +], ids=["backslash-userinfo", "backslash-after-port", "userinfo", "user-and-password", + "backslash-in-path", "control-character", "port-out-of-range", "two-ports"]) +def test_a_page_url_a_browser_reads_as_another_host_is_refused( + tmp_path, bad) -> None: + """r4180089809, Copilot's case first: `urlsplit` reads + `http://evil.example\\@127.0.0.1:8080/` as user information at + `127.0.0.1`, and a browser, taking the backslash for a slash, navigates + to `evil.example`, whose page could read the token's fragment. The + authority must be a loopback host and an optional port, exactly, and a + backslash or a control character anywhere is refused. Nothing is + written.""" + from opendox import console_access + + token = _token() + with pytest.raises(console_access.ConsoleAccessRefused): + console_access.opened_url(bad, token) + state = _state(tmp_path) + with pytest.raises(console_access.ConsoleAccessRefused) as refused: + console_access.write_private_copy( + state, page_url=bad, port=8080, token=token, served_roots=()) + assert token not in str(refused.value) + assert not (state / console_access.CONSOLE_DIRNAME).exists() + + +@pytest.mark.parametrize("good", [ + "http://127.0.0.1:8080/index.html", "http://[::1]:8080/index.html", + "http://localhost:8080/index.html", "http://127.0.0.1/index.html"]) +def test_every_loopback_page_url_a_plane_announces_is_accepted(good) -> None: + """The spellings `serve.server_url` announces a plane at, with and + without a port, are still accepted.""" + from opendox import console_access + + token = _token() + assert console_access.opened_url(good, token).startswith(good + "#")