From e406a006f7334c86c1ff84b3256c0a32d982ead4 Mon Sep 17 00:00:00 2001 From: Brett Heap <1513478+brettheap@users.noreply.github.com> Date: Mon, 28 Sep 2026 18:50:14 +0000 Subject: [PATCH 01/39] =?UTF-8?q?T071:=2013.2=20and=2013.3=20=E2=80=94=20l?= =?UTF-8?q?oad=5Fsettings=20refuses=20a=20non-PostgreSQL=20DSN=20and=20a?= =?UTF-8?q?=20collapsed=20DSN=20pair=20(plan=20034)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Realizes #1144 13.2 and 13.3, falsifier F13.1's `load_settings` block. - 13.2: `_refuse_non_postgresql_dsn` refuses either DSN (`OPENDOX_DATABASE_URL` or `OPENDOX_MIGRATION_DATABASE_URL`) whose URI scheme is not `postgresql://` or `postgres://`, naming the setting and the dialect kept. The keyword/value conninfo form (`host=h dbname=d …`) names no dialect at all and is unaffected — that syntax is libpq's own grammar, and no other driver reads it. - 13.3: `OPENDOX_MIGRATION_DATABASE_URL` stops being optional in `load_settings` (the `Setting` row's `required` flag, `_require` in place of `_optional`, and `RuntimeSettings.migration_database_url`'s type). A new `_refuse_the_same_dsn_in_both_settings` refuses the two DSNs being the exact same STRING, naming `OPENDOX_MIGRATION_DATABASE_URL`, once they are already known to agree on where they land (`_refuse_two_dsns_that_select_different_schemas`, unchanged, now called first): two DIFFERENT secrets for one role still pass, as the existing "single-role install" case documents. - Explicitly NOT in this task: 13.4-13.6 (`OPENDOX_INSTALL_MODE`, T070). Nothing here reads or names that setting, and `load_settings`'s only new required input is the migration DSN itself. Every existing call site that built an environment without `OPENDOX_MIGRATION_DATABASE_URL` needed one once it became required: `tests_runtime/conftest.py` gains a `migration_dsn` fixture (a `postgres_dsn` distinguished by a URI fragment, invisible to every DSN reader this module has); `test_api_endpoints.py`, `test_migrations_apply.py`, `test_runtime_cli.py` and `test_runtime_surface.py` thread it or a literal peer through. `test_two_dsns_that_select_different_schemas_are_refused`'s "a migration DSN that is simply absent" case is rewritten from accepted to refused, which is the behavior 13.3 changes. Two new tests (`test_a_non_postgresql_dsn_is_refused_naming_the_dialect_kept`, `test_the_same_dsn_in_both_settings_is_refused_naming_the_migration_one`) cover the two new refusals directly. Measured locally against this change (own Postgres container, bridge IP — this sandbox's host-mapped loopback ports are unreachable): `python -m pytest -q` reports 2469 passed, 11 skipped, 1 failed — the one failure is `tests/test_model_provider_broker.py::test_the_broker_child_inherits_no_ credential_shaped_environment`, already red against unmodified `main` (2d116415) in the same environment (an `LC_CTYPE` ambient in this sandbox, unrelated to runtime/config.py). Against `main`'s own reading (2479 selected / 2468 passed / 11 skipped, matching this repo's last recorded CI triple), this change is +2/+2/+0 for the two new tests — `validate.yml`'s `Pin the triple` floors (`MIN_SELECTED=2476`, `MIN_PASSED=2465`, `EXPECT_SKIPPED=11`) permit the rise unchanged. Arc: neutral-product-standalone-operability Lane: openxfactory-4 (openXfactory-4-openDox_extraction) Co-Authored-By: Claude Opus 5.5 (1M context) --- src/opendox/runtime/config.py | 104 ++++++++++++++++++--- tests_runtime/conftest.py | 31 ++++++- tests_runtime/test_api_endpoints.py | 18 +++- tests_runtime/test_migrations_apply.py | 7 ++ tests_runtime/test_runtime_cli.py | 124 ++++++++++++++++++++++++- tests_runtime/test_runtime_surface.py | 2 + 6 files changed, 264 insertions(+), 22 deletions(-) diff --git a/src/opendox/runtime/config.py b/src/opendox/runtime/config.py index 5a832e12..046a46ab 100644 --- a/src/opendox/runtime/config.py +++ b/src/opendox/runtime/config.py @@ -81,7 +81,7 @@ class Setting: "it must not be the identity migrations run as", ), Setting( - PREFIX + "MIGRATION_DATABASE_URL", None, False, True, + PREFIX + "MIGRATION_DATABASE_URL", None, True, True, "the PRIVILEGED DSN ordered-SQL migrations are applied with, used by " "`opendox runtime migrate` alone and never by the served application", ), @@ -180,10 +180,14 @@ class RuntimeSettings: """The resolved configuration of one runtime process. Construct with :func:`load_settings`; the fields are in `SETTINGS` order. + + `migration_database_url` is never `None` from either loader (plan 034, + 13.3): `load_settings` requires it exactly as it requires `database_url`, + and `load_migration_settings` already refused to load without one. """ database_url: str - migration_database_url: str | None + migration_database_url: str oidc_issuer: str oidc_audience: str oidc_jwks_url: str | None @@ -1288,6 +1292,63 @@ def effective_schema(dsn: str) -> str | None: return user_named_by(dsn) +#: THE ONLY DIALECT THIS RUNTIME KEEPS (plan 034, 13.2). `psycopg` is the one +#: driver `runtime` depends on and it speaks PostgreSQL alone, but a DSN is a +#: string and nothing stopped an operator writing `sqlite:///…` into either +#: setting and discovering the mismatch however far the code got before the +#: driver refused it. RULING Q1 keeps this database DOCUMENT-FREE, which is +#: why a second dialect is refused HERE rather than supported: it would double +#: every migration and every schema test forever, for a database that holds no +#: document. `postgres://` is accepted beside `postgresql://` because libpq +#: treats the two as one scheme. +POSTGRESQL_SCHEMES = frozenset({"postgresql", "postgres"}) + + +def _refuse_non_postgresql_dsn(name: str, dsn: str) -> None: + """`name`'s DSN is refused unless it selects a PostgreSQL scheme. + + A DSN in the keyword/value form (`host=h dbname=d …`) names NO DIALECT AT + ALL — that syntax is libpq's own conninfo grammar, and no other driver + reads it — so only the URI form is checked: `urlsplit` reports an EMPTY + scheme for the keyword/value form (there is no `://` to split on), and an + empty scheme is read as "says nothing" here, exactly as `schema_selected_by` + reads a DSN that names no schema as `None` rather than as a refusal. + """ + scheme = urllib.parse.urlsplit(dsn).scheme + if scheme and scheme not in POSTGRESQL_SCHEMES: + raise ConfigurationError( + f"{name} names the {scheme!r} dialect. PostgreSQL " + "(`postgresql://` or `postgres://`) is the only dialect this " + "runtime keeps: a second one would double every migration and " + "every schema test forever, for a database that holds no " + "document (RULING Q1)") + + +def _refuse_the_same_dsn_in_both_settings(served: str, migration: str) -> None: + """One credential pasted into both settings is refused (plan 034, 13.3). + + `OPENDOX_DATABASE_URL` is the least-privileged identity the API serves + with; `OPENDOX_MIGRATION_DATABASE_URL` is the privileged one ordered-SQL + migrations run as — the whole point of keeping two settings. A + single-user install is not a reason to collapse them into one: this is + the two settings simply BEING each other, which is different from + `_refuse_two_dsns_that_select_different_schemas` below, where they + DISAGREE about where they land. It is different too from the accepted + "single-role install" (`test_a_dsn_that_names_no_database_still_reaches_ + one`): two DSNs for the same ROLE with two DIFFERENT secrets are two + credentials, not one pasted twice, and this checks the value actually + given, not the identity it happens to resolve to. + """ + if served == migration: + raise ConfigurationError( + f"{PREFIX}MIGRATION_DATABASE_URL is the same value as " + f"{PREFIX}DATABASE_URL. The identity migrations run as must not " + "also be the identity the API serves with; give the migration " + "credential its own DSN, even where both reach the same " + "database (the values are not repeated: a DSN carries a " + "password)") + + def _refuse_two_dsns_that_select_different_schemas( served: str, migration: str | None) -> None: """Both DSNs must land in one schema, or neither answer means anything. @@ -1400,21 +1461,42 @@ def load_settings(env: Mapping[str, str] | None = None) -> RuntimeSettings: accept `HS256` would verify a token signed with the public key anybody can fetch from the broker's JWKS, and discovering that on the first request means it is already serving. + + BOTH DSNs ARE REQUIRED (plan 034, 13.3): `OPENDOX_MIGRATION_DATABASE_URL` + used to default to `None`, so the collapsed, single-role shape this + refuses (below) was reachable only by accident, through a caller who + happened to set both to the same value — an install that left the + migration credential unset was never asked the question at all. Naming it + explicitly, even at the same database a served DSN already names, is what + keeps the two identities two DECISIONS rather than one remembered twice. """ env = os.environ if env is None else env algorithms = _algorithms(env) - # AND THE TWO DSNs LAND IN ONE SCHEMA. See - # `_refuse_two_dsns_that_select_different_schemas`: this is the half of - # that invariant a string can answer, and it is asked here because this is - # the one loader that holds BOTH values. - _refuse_two_dsns_that_select_different_schemas( - _require(env, _by_name(PREFIX + "DATABASE_URL")), - _optional(env, _by_name(PREFIX + "MIGRATION_DATABASE_URL"))) + served = _require(env, _by_name(PREFIX + "DATABASE_URL")) + migration = _require(env, _by_name(PREFIX + "MIGRATION_DATABASE_URL")) + # THE DIALECT FIRST: a scheme this module cannot parse as PostgreSQL is not + # yet a DSN worth comparing at all. + _refuse_non_postgresql_dsn(PREFIX + "DATABASE_URL", served) + _refuse_non_postgresql_dsn(PREFIX + "MIGRATION_DATABASE_URL", migration) + # THEN WHETHER THEY DISAGREE. See `_refuse_two_dsns_that_select_different_ + # schemas`: this is the half of that invariant a string can answer, and it + # is asked here because this is the one loader that holds BOTH values. A + # DSN compared against ITSELF can never disagree, so this step passes + # silently on exactly the pair the next one exists to catch. + _refuse_two_dsns_that_select_different_schemas(served, migration) + # AND, LAST, WHETHER THEY ARE SIMPLY EACH OTHER. Two DSNs that agree on + # where they land are ordinarily two credentials for the one database + # (`test_a_dsn_that_names_no_database_still_reaches_one`'s "single-role + # install" is exactly that, two DIFFERENT secrets for one role) — but + # agreement bought by pasting the SAME value into both settings is not a + # second decision at all, and this is the check the one before it cannot + # make. + _refuse_the_same_dsn_in_both_settings(served, migration) return RuntimeSettings( - database_url=_require(env, _by_name(PREFIX + "DATABASE_URL")), - migration_database_url=_optional(env, _by_name(PREFIX + "MIGRATION_DATABASE_URL")), + database_url=served, + migration_database_url=migration, oidc_issuer=_broker_url(env, _by_name(PREFIX + "OIDC_ISSUER"), required=True, is_a_base_url=True) or "", oidc_audience=_require(env, _by_name(PREFIX + "OIDC_AUDIENCE")), diff --git a/tests_runtime/conftest.py b/tests_runtime/conftest.py index 2c6e0b7b..93f53a4d 100644 --- a/tests_runtime/conftest.py +++ b/tests_runtime/conftest.py @@ -219,6 +219,29 @@ def postgres_dsn() -> str: return dsn +@pytest.fixture(scope="session") +def migration_dsn(postgres_dsn: str) -> str: + """A DSN for `OPENDOX_MIGRATION_DATABASE_URL`, distinct from `postgres_dsn`. + + Plan 034, 13.3: `load_settings` now REQUIRES this setting and refuses a + value that is simply `postgres_dsn` repeated (13.3's own collapse + refusal), so every fixture built from the pair below needs a second, + genuinely different string — not a second database. None of this + module's fixtures apply migrations through this identity: `database` + applies them directly with `MigrationRunner`, and the served app under + test is never asked to `migrate`. So `load_settings` is the only reader + that cares about this value at all, and what it asks of a DSN is a + PostgreSQL scheme (13.2), a STRING distinct from `postgres_dsn` (13.3), + and a database and schema that agree with it + (`_refuse_two_dsns_that_select_different_schemas`). A URI FRAGMENT is + invisible to `database_named_by`, `user_named_by` and `schema_selected_by` + alike — none of the three inspects `urlsplit(...).fragment` — so + appending one changes the STRING without moving the identity those + functions compare. + """ + return postgres_dsn + "#opendox-test-migration-identity" + + @pytest.fixture def database(postgres_dsn: str) -> Iterator[object]: """A `Database` on a throwaway schema, with the migrations already applied.""" @@ -357,7 +380,7 @@ def verifier(jwks_path: str): @pytest.fixture() -def client(database, postgres_dsn: str, verifier): +def client(database, postgres_dsn: str, migration_dsn: str, verifier): """A `TestClient` over the REAL application, on this test's own schema. The application is given its OWN `Database` on the same schema rather than @@ -376,6 +399,7 @@ def client(database, postgres_dsn: str, verifier): settings = load_settings({ PREFIX + "DATABASE_URL": postgres_dsn, + PREFIX + "MIGRATION_DATABASE_URL": migration_dsn, PREFIX + "OIDC_ISSUER": TEST_ISSUER, PREFIX + "OIDC_AUDIENCE": TEST_AUDIENCE, }) @@ -399,8 +423,8 @@ def project_repository_root(tmp_path): @pytest.fixture() -def client_with_repositories(database, postgres_dsn: str, verifier, - project_repository_root): +def client_with_repositories(database, postgres_dsn: str, migration_dsn: str, + verifier, project_repository_root): """`client`, with the repository root pointed at this test's own directory.""" fastapi_testclient = _import_fastapi_testclient() from opendox.runtime.app import create_app @@ -409,6 +433,7 @@ def client_with_repositories(database, postgres_dsn: str, verifier, settings = load_settings({ PREFIX + "DATABASE_URL": postgres_dsn, + PREFIX + "MIGRATION_DATABASE_URL": migration_dsn, PREFIX + "OIDC_ISSUER": TEST_ISSUER, PREFIX + "OIDC_AUDIENCE": TEST_AUDIENCE, PREFIX + "PROJECT_REPOSITORY_ROOT": str(project_repository_root), diff --git a/tests_runtime/test_api_endpoints.py b/tests_runtime/test_api_endpoints.py index 21897203..84caccfe 100644 --- a/tests_runtime/test_api_endpoints.py +++ b/tests_runtime/test_api_endpoints.py @@ -425,7 +425,7 @@ def test_the_unauthenticated_surface_is_exactly_the_two_probes( def test_the_schema_viewers_appear_only_when_the_install_says_so( - database, postgres_dsn: str, verifier) -> None: + database, postgres_dsn: str, migration_dsn: str, verifier) -> None: from fastapi.testclient import TestClient from opendox.runtime.app import create_app @@ -435,6 +435,7 @@ def test_the_schema_viewers_appear_only_when_the_install_says_so( settings = load_settings({ PREFIX + "DATABASE_URL": postgres_dsn, + PREFIX + "MIGRATION_DATABASE_URL": migration_dsn, PREFIX + "OIDC_ISSUER": TEST_ISSUER, PREFIX + "OIDC_AUDIENCE": TEST_AUDIENCE, PREFIX + "PUBLISH_OPENAPI": "true", @@ -448,7 +449,7 @@ def test_the_schema_viewers_appear_only_when_the_install_says_so( def test_readiness_refuses_an_unmigrated_database_by_name( - postgres_dsn: str, verifier) -> None: + postgres_dsn: str, migration_dsn: str, verifier) -> None: """`select 1` succeeds against a schema with no tables in it at all. Readiness without a migration check therefore turns a fresh install READY @@ -471,6 +472,7 @@ def test_readiness_refuses_an_unmigrated_database_by_name( try: settings = load_settings({ PREFIX + "DATABASE_URL": postgres_dsn, + PREFIX + "MIGRATION_DATABASE_URL": migration_dsn, PREFIX + "OIDC_ISSUER": TEST_ISSUER, PREFIX + "OIDC_AUDIENCE": TEST_AUDIENCE, }) @@ -588,7 +590,8 @@ def test_draft_pagination_returns_the_principals_drafts_not_an_empty_page( def test_readiness_refuses_a_database_whose_migration_file_has_changed( - database, postgres_dsn: str, verifier, tmp_path) -> None: + database, postgres_dsn: str, migration_dsn: str, verifier, + tmp_path) -> None: """Nothing pending is not the same as matching this tree.""" import shutil @@ -609,6 +612,7 @@ def test_readiness_refuses_a_database_whose_migration_file_has_changed( settings = load_settings({ PREFIX + "DATABASE_URL": postgres_dsn, + PREFIX + "MIGRATION_DATABASE_URL": migration_dsn, PREFIX + "OIDC_ISSUER": TEST_ISSUER, PREFIX + "OIDC_AUDIENCE": TEST_AUDIENCE, PREFIX + "MIGRATIONS_DIR": str(tmp_path), @@ -852,7 +856,8 @@ def test_a_hidden_user_and_a_missing_one_answer_byte_for_byte_the_same( def test_readiness_refuses_a_migrations_directory_without_the_pinned_0001( - database, postgres_dsn: str, verifier, tmp_path) -> None: + database, postgres_dsn: str, migration_dsn: str, verifier, + tmp_path) -> None: """An EMPTY directory made `plan()` and `drift()` both empty. `discover()` returns `[]` for a directory that exists and holds no @@ -874,6 +879,7 @@ def test_readiness_refuses_a_migrations_directory_without_the_pinned_0001( empty.mkdir() settings = load_settings({ PREFIX + "DATABASE_URL": postgres_dsn, + PREFIX + "MIGRATION_DATABASE_URL": migration_dsn, PREFIX + "OIDC_ISSUER": TEST_ISSUER, PREFIX + "OIDC_AUDIENCE": TEST_AUDIENCE, PREFIX + "MIGRATIONS_DIR": str(empty), @@ -1552,7 +1558,8 @@ def transaction(): def test_an_anonymous_token_naming_a_key_of_the_wrong_type_is_401_not_500( - database, postgres_dsn: str, rsa_key_pair, tmp_path: Path) -> None: + database, postgres_dsn: str, migration_dsn: str, rsa_key_pair, + tmp_path: Path) -> None: """The end of A25-2, measured where it was reported: at the HTTP boundary. A realm publishing an RSA and an EC signing key — Keycloak, the moment a @@ -1593,6 +1600,7 @@ def test_an_anonymous_token_naming_a_key_of_the_wrong_type_is_401_not_500( settings = load_settings({ PREFIX + "DATABASE_URL": postgres_dsn, + PREFIX + "MIGRATION_DATABASE_URL": migration_dsn, PREFIX + "OIDC_ISSUER": TEST_ISSUER, PREFIX + "OIDC_AUDIENCE": TEST_AUDIENCE, }) diff --git a/tests_runtime/test_migrations_apply.py b/tests_runtime/test_migrations_apply.py index 2afc0fe1..1b657628 100644 --- a/tests_runtime/test_migrations_apply.py +++ b/tests_runtime/test_migrations_apply.py @@ -345,6 +345,11 @@ def test_status_reports_a_reachable_database_and_its_applied_migrations( # libpq startup parameter `Database(schema=…)` sets for the harness. scoped = (f"{postgres_dsn}?options=-c%20search_path%3D{database.schema}") monkeypatch.setenv(PREFIX + "DATABASE_URL", scoped) + # A DISTINCT STRING (13.3's collapse refusal) THAT SELECTS THE SAME SCHEMA + # (`_refuse_two_dsns_that_select_different_schemas`): a URI FRAGMENT moves + # neither, since neither reader inspects one. + monkeypatch.setenv(PREFIX + "MIGRATION_DATABASE_URL", + scoped + "#opendox-test-migration-identity") monkeypatch.setenv(PREFIX + "OIDC_ISSUER", "https://broker.test/realms/x") monkeypatch.setenv(PREFIX + "OIDC_AUDIENCE", "opendox-runtime") monkeypatch.setenv(PREFIX + "MIGRATIONS_DIR", str(ROOT / "migrations")) @@ -720,6 +725,8 @@ def test_status_calls_an_unmigrated_database_unhealthy_and_blames_the_tree( conn.execute(f"create schema {schema}") try: monkeypatch.setenv(PREFIX + "DATABASE_URL", scoped) + monkeypatch.setenv(PREFIX + "MIGRATION_DATABASE_URL", + scoped + "#opendox-test-migration-identity") monkeypatch.setenv(PREFIX + "OIDC_ISSUER", "https://broker/realms/x") monkeypatch.setenv(PREFIX + "OIDC_AUDIENCE", "opendox-runtime") monkeypatch.setenv(PREFIX + "MIGRATIONS_DIR", str(ROOT / "migrations")) diff --git a/tests_runtime/test_runtime_cli.py b/tests_runtime/test_runtime_cli.py index 4a6d9d13..363caced 100644 --- a/tests_runtime/test_runtime_cli.py +++ b/tests_runtime/test_runtime_cli.py @@ -211,6 +211,8 @@ def test_init_creates_the_project_repository_root_and_touches_no_database( root = tmp_path / "projects" monkeypatch.setenv(PREFIX + "DATABASE_URL", "postgresql://nobody@127.0.0.1:1/none") + monkeypatch.setenv(PREFIX + "MIGRATION_DATABASE_URL", + "postgresql://migrate@127.0.0.1:1/none") monkeypatch.setenv(PREFIX + "OIDC_ISSUER", "https://broker/realms/x") monkeypatch.setenv(PREFIX + "OIDC_AUDIENCE", "opendox-runtime") monkeypatch.setenv(PREFIX + "PROJECT_REPOSITORY_ROOT", str(root)) @@ -362,6 +364,8 @@ def test_status_reports_every_declared_setting_and_none_as_null( from opendox.runtime.config import SETTING_NAMES monkeypatch.setenv(PREFIX + "DATABASE_URL", "postgresql://u:pw@127.0.0.1:1/x") + monkeypatch.setenv(PREFIX + "MIGRATION_DATABASE_URL", + "postgresql://m:pw@127.0.0.1:1/x") monkeypatch.setenv(PREFIX + "OIDC_ISSUER", "https://broker/realms/x") monkeypatch.setenv(PREFIX + "OIDC_AUDIENCE", "opendox-runtime") monkeypatch.setenv(PREFIX + "RUNTIME_PG_ROLE", "opendox_runtime") @@ -432,6 +436,8 @@ def test_init_refuses_a_repository_root_that_is_not_a_directory( root.write_text("not a directory", encoding="utf-8") monkeypatch.setenv(PREFIX + "DATABASE_URL", "postgresql://nobody@127.0.0.1:1/none") + monkeypatch.setenv(PREFIX + "MIGRATION_DATABASE_URL", + "postgresql://migrate@127.0.0.1:1/none") monkeypatch.setenv(PREFIX + "OIDC_ISSUER", "https://broker/realms/x") monkeypatch.setenv(PREFIX + "OIDC_AUDIENCE", "opendox-runtime") monkeypatch.setenv(PREFIX + "PROJECT_REPOSITORY_ROOT", str(root)) @@ -496,6 +502,8 @@ def run(self) -> None: monkeypatch.setitem(sys.modules, "opendox.runtime.app", app_stub) monkeypatch.setenv(PREFIX + "DATABASE_URL", "postgresql://nobody@127.0.0.1:1/none") + monkeypatch.setenv(PREFIX + "MIGRATION_DATABASE_URL", + "postgresql://migrate@127.0.0.1:1/none") monkeypatch.setenv(PREFIX + "OIDC_ISSUER", "https://broker/realms/x") monkeypatch.setenv(PREFIX + "OIDC_AUDIENCE", "opendox-runtime") code, evidence = _run(cli.build_parser().parse_args(["runtime", "serve"])) @@ -541,6 +549,8 @@ def run(self) -> None: monkeypatch.setitem(sys.modules, "opendox.runtime.app", app_stub) monkeypatch.setenv(PREFIX + "DATABASE_URL", "postgresql://nobody@127.0.0.1:1/none") + monkeypatch.setenv(PREFIX + "MIGRATION_DATABASE_URL", + "postgresql://migrate@127.0.0.1:1/none") monkeypatch.setenv(PREFIX + "OIDC_ISSUER", "https://broker/realms/x") monkeypatch.setenv(PREFIX + "OIDC_AUDIENCE", "opendox-runtime") code, evidence = _run(cli.build_parser().parse_args(["runtime", "serve"])) @@ -624,6 +634,8 @@ def __exit__(self, *exc: object) -> None: db_stub.Database = _Exploding # type: ignore[attr-defined] monkeypatch.setitem(sys.modules, "opendox.runtime.db", db_stub) monkeypatch.setenv(PREFIX + "DATABASE_URL", dsn) + monkeypatch.setenv(PREFIX + "MIGRATION_DATABASE_URL", + "postgresql://migrate@db.internal:5432/opendox") monkeypatch.setenv(PREFIX + "OIDC_ISSUER", "https://broker/realms/x") monkeypatch.setenv(PREFIX + "OIDC_AUDIENCE", "opendox-runtime") @@ -677,6 +689,8 @@ def __init__(self, config: object) -> None: monkeypatch.setitem(sys.modules, "opendox.runtime.app", app_stub) monkeypatch.setenv(PREFIX + "DATABASE_URL", "postgresql://nobody:hunter2@127.0.0.1:1/none") + monkeypatch.setenv(PREFIX + "MIGRATION_DATABASE_URL", + "postgresql://migrate@127.0.0.1:1/none") monkeypatch.setenv(PREFIX + "OIDC_ISSUER", "https://broker/realms/x") monkeypatch.setenv(PREFIX + "OIDC_AUDIENCE", "opendox-runtime") @@ -794,6 +808,8 @@ def connection(self): monkeypatch.setitem(sys.modules, "opendox.runtime.db", db_stub) monkeypatch.setenv(PREFIX + "DATABASE_URL", "postgresql://runtime:hunter2@127.0.0.1:5432/opendox") + monkeypatch.setenv(PREFIX + "MIGRATION_DATABASE_URL", + "postgresql://migrate@127.0.0.1:5432/opendox") monkeypatch.setenv(PREFIX + "OIDC_ISSUER", "https://broker/realms/x") monkeypatch.setenv(PREFIX + "OIDC_AUDIENCE", "opendox-runtime") monkeypatch.setenv(PREFIX + "MIGRATIONS_DIR", @@ -862,6 +878,8 @@ def transaction(self): monkeypatch.setitem(sys.modules, "opendox.runtime.db", db_stub) monkeypatch.setenv(PREFIX + "DATABASE_URL", "postgresql://runtime@127.0.0.1:5432/opendox") + monkeypatch.setenv(PREFIX + "MIGRATION_DATABASE_URL", + "postgresql://migrate@127.0.0.1:5432/opendox") monkeypatch.setenv(PREFIX + "OIDC_ISSUER", "https://broker/realms/x") monkeypatch.setenv(PREFIX + "OIDC_AUDIENCE", "opendox-runtime") monkeypatch.setenv(PREFIX + "PROJECT_REPOSITORY_ROOT", str(tmp_path)) @@ -1148,6 +1166,7 @@ def test_no_broker_url_this_runtime_prints_can_carry_a_credential() -> None: redacted_url) base = {PREFIX + "DATABASE_URL": "postgresql://u:p@h/db", + PREFIX + "MIGRATION_DATABASE_URL": "postgresql://m:p@h/db", PREFIX + "OIDC_AUDIENCE": "opendox-runtime"} for name in ("OIDC_ISSUER", "OIDC_JWKS_URL"): env = dict(base, **{PREFIX + "OIDC_ISSUER": "https://broker/realms/x"}) @@ -1195,6 +1214,7 @@ def test_a_credential_in_a_broker_urls_query_is_refused_like_one_in_its_userinfo redacted_url) base = {PREFIX + "DATABASE_URL": "postgresql://u:p@h/db", + PREFIX + "MIGRATION_DATABASE_URL": "postgresql://m:p@h/db", PREFIX + "OIDC_AUDIENCE": "opendox-runtime", PREFIX + "OIDC_ISSUER": "https://broker/realms/x"} for name, value, why in ( @@ -1243,6 +1263,7 @@ def test_the_broker_url_must_be_https_because_it_is_the_trust_anchor() -> None: from opendox.runtime.config import ConfigurationError, load_settings base = {PREFIX + "DATABASE_URL": "postgresql://u:p@h/db", + PREFIX + "MIGRATION_DATABASE_URL": "postgresql://m:p@h/db", PREFIX + "OIDC_AUDIENCE": "opendox-runtime"} for issuer in ("https://broker/realms/x", "http://localhost:8080/realms/x", "http://127.0.0.1:8080/realms/x", "http://[::1]:8080/x"): @@ -1424,6 +1445,7 @@ def test_a_broker_url_that_names_no_host_is_refused_at_the_door() -> None: from opendox.runtime.config import ConfigurationError, load_settings base = {PREFIX + "DATABASE_URL": "postgresql://u:p@h/db", + PREFIX + "MIGRATION_DATABASE_URL": "postgresql://m:p@h/db", PREFIX + "OIDC_AUDIENCE": "opendox-runtime"} for issuer in ("https:///realms/x", "https://", "https:///", "broker/realms/x", "https://user:hunter2@/realms/x"): @@ -1509,6 +1531,7 @@ def test_a_broker_url_whose_port_is_not_a_number_is_refused_at_the_door( from opendox.runtime.config import ConfigurationError, load_settings base = {PREFIX + "DATABASE_URL": "postgresql://u:p@h/db", + PREFIX + "MIGRATION_DATABASE_URL": "postgresql://m:p@h/db", PREFIX + "OIDC_AUDIENCE": "opendox-runtime"} for issuer in ("https://broker:not-a-port/realm", "https://broker:99999/realm", @@ -1554,6 +1577,7 @@ def test_the_issuer_carries_no_query_or_fragment_because_paths_are_appended( from opendox.runtime.config import ConfigurationError, load_settings base = {PREFIX + "DATABASE_URL": "postgresql://u:p@h/db", + PREFIX + "MIGRATION_DATABASE_URL": "postgresql://m:p@h/db", PREFIX + "OIDC_AUDIENCE": "opendox-runtime"} for issuer, component in (("https://broker/realms/x?tenant=a", "query"), ("https://broker/realms/x#frag", "fragment"), @@ -1599,6 +1623,7 @@ def test_the_loopback_exception_is_for_http_and_not_for_every_other_scheme( from opendox.runtime.config import ConfigurationError, load_settings base = {PREFIX + "DATABASE_URL": "postgresql://u:p@h/db", + PREFIX + "MIGRATION_DATABASE_URL": "postgresql://m:p@h/db", PREFIX + "OIDC_AUDIENCE": "opendox-runtime"} for issuer in ("ftp://localhost/realms/x", "file://127.0.0.1/realms/x", "ws://localhost:8080/realms/x", "ftp://[::1]/realms/x"): @@ -1910,6 +1935,7 @@ def test_a_malformed_broker_url_never_prints_its_own_password() -> None: issuer = "https://svc:hunter2@broker\u2100evil.example/realms/x" env = {PREFIX + "DATABASE_URL": "postgresql://u:p@h/db", + PREFIX + "MIGRATION_DATABASE_URL": "postgresql://m:p@h/db", PREFIX + "OIDC_AUDIENCE": "opendox", PREFIX + "OIDC_ISSUER": issuer} @@ -2126,9 +2152,16 @@ def test_two_dsns_that_select_different_schemas_are_refused() -> None: PREFIX + "DATABASE_URL": "postgresql://u:p@h/db", PREFIX + "MIGRATION_DATABASE_URL": "postgresql://m:p@h/db"}) - # AND A MIGRATION DSN THAT IS SIMPLY ABSENT is the documented single-role - # deployment, not a mismatch. - assert load_settings({**base, PREFIX + "DATABASE_URL": served}) + # AND A MIGRATION DSN THAT IS SIMPLY ABSENT is refused NOW (plan 034, + # 13.3), where it used to be accepted as "the documented single-role + # deployment, not a mismatch": that reading is exactly what `load_settings` + # reading the setting as OPTIONAL bought, and the setting stopped being + # optional. A single-role install still names both DSNs, distinctly — + # `test_a_dsn_that_names_no_database_still_reaches_one`'s "two different + # secrets for one role" is what single-role now looks like. + with pytest.raises(ConfigurationError) as absent: + load_settings({**base, PREFIX + "DATABASE_URL": served}) + assert PREFIX + "MIGRATION_DATABASE_URL" in str(absent.value) # THE OTHER DIRECTION IS REFUSED TOO: a served DSN that names no schema # beside a migration DSN that names one is the same split, and the @@ -2140,6 +2173,86 @@ def test_two_dsns_that_select_different_schemas_are_refused() -> None: assert "the connection default" in str(either_way.value) +def test_a_non_postgresql_dsn_is_refused_naming_the_dialect_kept() -> None: + """13.2: a second dialect is refused, not supported. + + RULING Q1 keeps this database DOCUMENT-FREE, so a second dialect would + double every migration and every schema test forever for a database that + holds nothing. `_refuse_non_postgresql_dsn` asks it of both DSNs + `load_settings` holds, before either reaches the checks above that + compare them. + """ + from opendox.runtime.config import ConfigurationError, load_settings + + base = {PREFIX + "OIDC_ISSUER": "https://broker/realms/x", + PREFIX + "OIDC_AUDIENCE": "opendox"} + with pytest.raises(ConfigurationError) as served: + load_settings({**base, + PREFIX + "DATABASE_URL": "sqlite:///x.db", + PREFIX + "MIGRATION_DATABASE_URL": + "postgresql://m:p@h/db"}) + assert "postgres" in str(served.value).lower() + assert PREFIX + "DATABASE_URL" in str(served.value) + + with pytest.raises(ConfigurationError) as migration: + load_settings({**base, + PREFIX + "DATABASE_URL": "postgresql://u:p@h/db", + PREFIX + "MIGRATION_DATABASE_URL": "mysql://m:p@h/db"}) + assert "postgres" in str(migration.value).lower() + assert PREFIX + "MIGRATION_DATABASE_URL" in str(migration.value) + + # `postgres://` IS THE OTHER SPELLING LIBPQ ACCEPTS, not a second dialect. + assert load_settings({**base, + PREFIX + "DATABASE_URL": "postgres://u:p@h/db", + PREFIX + "MIGRATION_DATABASE_URL": + "postgres://m:p@h/db"}) + + # AND THE KEYWORD/VALUE FORM NAMES NO DIALECT AT ALL, so it is not refused + # here: libpq's own conninfo grammar reaches no other driver, and this + # module already reads an empty scheme as "says nothing" the way + # `schema_selected_by` does for a DSN that names no schema. + assert load_settings({**base, + PREFIX + "DATABASE_URL": "host=h dbname=db", + PREFIX + "MIGRATION_DATABASE_URL": + "host=h dbname=db user=m"}) + + +def test_the_same_dsn_in_both_settings_is_refused_naming_the_migration_one( +) -> None: + """13.3: one credential pasted into both settings is refused. + + `_refuse_the_same_dsn_in_both_settings` is asked only once the two DSNs + are known to AGREE on where they land + (`_refuse_two_dsns_that_select_different_schemas`, above it): agreement + bought by two DIFFERENT secrets for the one role is the accepted + single-role shape + (`test_a_dsn_that_names_no_database_still_reaches_one`'s last case); + agreement bought by writing the SAME value into both settings is this + refusal instead. + """ + from opendox.runtime.config import ConfigurationError, load_settings + + base = {PREFIX + "OIDC_ISSUER": "https://broker/realms/x", + PREFIX + "OIDC_AUDIENCE": "opendox"} + one = "postgresql://one:hunter2@h/opendox" + with pytest.raises(ConfigurationError) as collapsed: + load_settings({**base, + PREFIX + "DATABASE_URL": one, + PREFIX + "MIGRATION_DATABASE_URL": one}) + message = str(collapsed.value) + assert PREFIX + "MIGRATION_DATABASE_URL" in message + assert PREFIX + "DATABASE_URL" in message + # THE VALUE IS NOT REPEATED: a DSN carries a password. + assert "hunter2" not in message and "one:" not in message + + # DIFFERENT STRINGS THAT STILL AGREE are NOT this refusal, whether the + # difference is the secret alone (single-role) or the whole identity. + assert load_settings({**base, + PREFIX + "DATABASE_URL": "postgresql://a:p@h/db", + PREFIX + "MIGRATION_DATABASE_URL": + "postgresql://b:p@h/db"}) + + def test_two_dsns_naming_different_databases_are_refused_too() -> None: """The schema comparison means nothing across two databases. @@ -2348,6 +2461,7 @@ def test_a_credential_shaped_parameter_name_is_a_WORD_and_not_a_substring() -> N assert redacted_url("https://broker/certs?token=x" ) == "https://broker/certs?token=" base = {PREFIX + "DATABASE_URL": "postgresql://u:p@h/db", + PREFIX + "MIGRATION_DATABASE_URL": "postgresql://m:p@h/db", PREFIX + "OIDC_ISSUER": "https://broker/realms/x", PREFIX + "OIDC_AUDIENCE": "opendox"} settings = load_settings({**base, @@ -2548,6 +2662,8 @@ def test_init_creates_the_repository_root_private_whatever_the_umask_is( root = tmp_path / "projects" monkeypatch.setenv(PREFIX + "DATABASE_URL", "postgresql://nobody@127.0.0.1:1/none") + monkeypatch.setenv(PREFIX + "MIGRATION_DATABASE_URL", + "postgresql://migrate@127.0.0.1:1/none") monkeypatch.setenv(PREFIX + "OIDC_ISSUER", "https://broker/realms/x") monkeypatch.setenv(PREFIX + "OIDC_AUDIENCE", "opendox-runtime") monkeypatch.setenv(PREFIX + "PROJECT_REPOSITORY_ROOT", str(root)) @@ -2584,6 +2700,8 @@ def test_init_reports_an_existing_root_s_mode_and_does_not_change_it( os.chmod(root, 0o755) monkeypatch.setenv(PREFIX + "DATABASE_URL", "postgresql://nobody@127.0.0.1:1/none") + monkeypatch.setenv(PREFIX + "MIGRATION_DATABASE_URL", + "postgresql://migrate@127.0.0.1:1/none") monkeypatch.setenv(PREFIX + "OIDC_ISSUER", "https://broker/realms/x") monkeypatch.setenv(PREFIX + "OIDC_AUDIENCE", "opendox-runtime") monkeypatch.setenv(PREFIX + "PROJECT_REPOSITORY_ROOT", str(root)) diff --git a/tests_runtime/test_runtime_surface.py b/tests_runtime/test_runtime_surface.py index ff6a73a3..b36c6203 100644 --- a/tests_runtime/test_runtime_surface.py +++ b/tests_runtime/test_runtime_surface.py @@ -258,6 +258,7 @@ def test_only_asymmetric_algorithms_can_be_configured_and_they_keep_pyjwts_spell ) base = {PREFIX + "DATABASE_URL": "postgresql://x/y", + PREFIX + "MIGRATION_DATABASE_URL": "postgresql://m@x/y", PREFIX + "OIDC_ISSUER": "https://broker/realms/x", PREFIX + "OIDC_AUDIENCE": "opendox-runtime"} for name in ASYMMETRIC_ALGORITHMS: @@ -318,6 +319,7 @@ def test_every_integer_setting_is_bounded_above_as_well_as_below() -> None: ) base = {PREFIX + "DATABASE_URL": "postgresql://u:p@h/db", + PREFIX + "MIGRATION_DATABASE_URL": "postgresql://m:p@h/db", PREFIX + "OIDC_AUDIENCE": "opendox", PREFIX + "OIDC_ISSUER": "https://broker/realms/x"} From 91f797329ce81fa325f3c9b24512f209478316de Mon Sep 17 00:00:00 2001 From: Brett Heap <1513478+brettheap@users.noreply.github.com> Date: Mon, 28 Sep 2026 19:04:30 +0000 Subject: [PATCH 02/39] Fix round: _refuse_non_postgresql_dsn never raises a bare ValueError (Copilot review of this PR) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `urlsplit` itself raises for a DSN it cannot parse — MEASURED, ValueError("Invalid IPv6 URL") for an unbracketed IPv6 host, which tests_runtime/conftest.py's own postgres_dsn docstring names as "the ordinary way to mis-set this variable". `_refuse_non_postgresql_dsn` called `urlsplit(dsn).scheme` unguarded, so that ValueError escaped load_settings as a bare exception instead of the promised ConfigurationError — the CLI's boundary catches only ConfigurationError, so a malformed OPENDOX_DATABASE_URL or OPENDOX_MIGRATION_DATABASE_URL would have printed a traceback instead of a redacted refusal. Wrapped the same way _split_url already wraps it for the broker settings (Copilot review of openDox-code#25, round 24), with DSN-appropriate wording rather than reused verbatim ("set it to the broker endpoint" does not fit a database DSN). New test test_an_unparseable_dsn_is_refused_and_never_raises_a_bare_valueerror proves both DSNs are covered and that the value is never repeated in the message. Arc: neutral-product-standalone-operability Lane: openxfactory-4 (openXfactory-4-openDox_extraction) Co-Authored-By: Claude Opus 5.5 (1M context) --- src/opendox/runtime/config.py | 17 ++++++++++++- tests_runtime/test_runtime_cli.py | 40 +++++++++++++++++++++++++++++++ 2 files changed, 56 insertions(+), 1 deletion(-) diff --git a/src/opendox/runtime/config.py b/src/opendox/runtime/config.py index 046a46ab..ecc53d65 100644 --- a/src/opendox/runtime/config.py +++ b/src/opendox/runtime/config.py @@ -1313,8 +1313,23 @@ def _refuse_non_postgresql_dsn(name: str, dsn: str) -> None: scheme for the keyword/value form (there is no `://` to split on), and an empty scheme is read as "says nothing" here, exactly as `schema_selected_by` reads a DSN that names no schema as `None` rather than as a refusal. + + `urlsplit` ITSELF RAISES for a DSN it cannot parse — MEASURED, + `ValueError("Invalid IPv6 URL")` for an unbracketed IPv6 host, which + `tests_runtime/conftest.py`'s own `postgres_dsn` docstring names as "the + ordinary way to mis-set this variable". `_split_url` exists for exactly + this shape in the broker settings (Copilot review of openDox-code#25, + round 24); this is its DSN-flavoured twin; a bad `OPENDOX_DATABASE_URL` + is not "set it to the broker endpoint", so it is not reused verbatim. """ - scheme = urllib.parse.urlsplit(dsn).scheme + try: + scheme = urllib.parse.urlsplit(dsn).scheme + except ValueError as exc: + raise ConfigurationError( + f"{name} is not a DSN this runtime can parse " + f"({type(exc).__name__}); the value is not repeated here, " + "because a DSN this runtime cannot parse can still carry a " + "password") from None if scheme and scheme not in POSTGRESQL_SCHEMES: raise ConfigurationError( f"{name} names the {scheme!r} dialect. PostgreSQL " diff --git a/tests_runtime/test_runtime_cli.py b/tests_runtime/test_runtime_cli.py index 363caced..2c3c97eb 100644 --- a/tests_runtime/test_runtime_cli.py +++ b/tests_runtime/test_runtime_cli.py @@ -2217,6 +2217,46 @@ def test_a_non_postgresql_dsn_is_refused_naming_the_dialect_kept() -> None: "host=h dbname=db user=m"}) +def test_an_unparseable_dsn_is_refused_and_never_raises_a_bare_valueerror( +) -> None: + """`urlsplit` itself raises for a DSN it cannot parse, and this module's + whole contract is that a bad variable produces a named + `ConfigurationError`, never a bare exception the CLI's boundary does not + catch (Copilot review of this PR). + + MEASURED: `urllib.parse.urlsplit("postgresql://u:p@[::1/db")` raises + `ValueError("Invalid IPv6 URL")` — an unbracketed IPv6 host, which + `tests_runtime/conftest.py`'s own `postgres_dsn` docstring names as "the + ordinary way to mis-set this variable". `_split_url` exists for exactly + this shape in the broker settings (Copilot review of openDox-code#25, + round 24); `_refuse_non_postgresql_dsn` is its own boundary for the two + DSNs, asked of both. + """ + from opendox.runtime.config import ConfigurationError, load_settings + + base = {PREFIX + "OIDC_ISSUER": "https://broker/realms/x", + PREFIX + "OIDC_AUDIENCE": "opendox"} + broken = "postgresql://opendox:hunter2@[::1/opendox" + + with pytest.raises(ConfigurationError) as served: + load_settings({**base, + PREFIX + "DATABASE_URL": broken, + PREFIX + "MIGRATION_DATABASE_URL": + "postgresql://m:p@h/db"}) + message = str(served.value) + assert PREFIX + "DATABASE_URL" in message + assert "ValueError" in message + # THE VALUE IS NOT REPEATED: a DSN this runtime cannot parse can still + # carry a password. + assert "hunter2" not in message and "opendox:" not in message + + with pytest.raises(ConfigurationError) as migration: + load_settings({**base, + PREFIX + "DATABASE_URL": "postgresql://u:p@h/db", + PREFIX + "MIGRATION_DATABASE_URL": broken}) + assert PREFIX + "MIGRATION_DATABASE_URL" in str(migration.value) + + def test_the_same_dsn_in_both_settings_is_refused_naming_the_migration_one( ) -> None: """13.3: one credential pasted into both settings is refused. From b5296f91c5df003964bf5c8f0126f6005da1fbbd Mon Sep 17 00:00:00 2001 From: Brett Heap <1513478+brettheap@users.noreply.github.com> Date: Mon, 28 Sep 2026 22:45:45 +0000 Subject: [PATCH 03/39] Rework #60 to Brett's ruling: OPENDOX_MIGRATION_DATABASE_URL required only for migrate MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Brett ruled on the held conflict (openxFactory#656, on the claim thread for plan 034's T071, 2026-09-28), choosing "Required only for migrate (Recommended)" over making the setting required everywhere: - OPENDOX_MIGRATION_DATABASE_URL goes back to OPTIONAL in `load_settings` (the `Setting` row's `required` flag, `RuntimeSettings.migration_ database_url`'s type back to `str | None`, `_optional` in place of `_require`). `load_migration_settings` is unaffected either way — it already independently required one, for `migrate`/`reset` alone. - Both refusals from the previous commits stay, and are now no-ops on an ABSENT migration DSN rather than being unreachable: `_refuse_non_ postgresql_dsn` and `_refuse_the_same_dsn_in_both_settings` each return early when the migration value is falsy, exactly the way `_refuse_two_ dsns_that_select_different_schemas` already treated "nothing to compare" as nothing to fault. When BOTH are given, every check still runs, in the same order as before (dialect, then schema-mismatch, then collapse). It is never defaulted from OPENDOX_DATABASE_URL. - This matches #1144 13.3's own text and `deploy/compose/docker-compose. yaml`'s separation (the `opendox` service never gets a migration DSN; `docs/runtime.md` § 3 never lists it as required) — neither file needed a change; both already said the now-ruled behavior. The plan's "stops being optional" line is a holder-side correction, not part of this PR, and #1144's own wording is unchanged. Reverted the 27-call-site ripple the `required` flip had forced, now that it is not needed: `tests_runtime/conftest.py`'s `migration_dsn` fixture is gone; `test_api_endpoints.py`, `test_migrations_apply.py`, `test_runtime_ cli.py` and `test_runtime_surface.py` are back to threading only the served DSN through every call site that does not itself test the migration path. All four files after conftest.py are byte-for-byte `main` again. `test_two_dsns_that_select_different_schemas_are_refused`'s "absent migration" case is back to ACCEPTED (with a note on why it was briefly the opposite), which is what the setting being optional again means for that test. Added three tests showing the ruled behavior, at the CLI dispatch level rather than only `load_settings` directly, next to the existing `migrate` counterpart: - `test_serve_and_status_load_with_no_migration_dsn_configured`: `status` reports no configuration refusal and `settings[…MIGRATION_DATABASE_URL] ` as `null` with only the served DSN set; `serve` starts (`ok: true`) the same way. - `test_the_collapse_is_refused_through_the_served_workload_too`: 13.3's collapse refusal still fires through `status`, not only through `load_settings` called directly, the moment both DSNs are given and are the same value. - `test_migrate_refuses_rather_than_borrowing_the_served_identity` (pre-existing, untouched) already covers "migrate refuses without it". Measured locally against this change (own Postgres container, bridge IP): `python -m pytest -q` reports 2472 passed, 11 skipped, 1 failed — the one failure is the same `tests/test_model_provider_broker.py:: test_the_broker_child_inherits_no_credential_shaped_environment` LC_CTYPE sandbox artifact already characterized as pre-existing and unrelated in the first commit on this branch. Against main's 2479 selected / 11 skipped in this same environment, this change is +5/+5/+0 (five tests: the three already on this branch plus the two new ones above) — `validate.yml`'s `Pin the triple` floors (`MIN_SELECTED=2476`, `MIN_PASSED=2465`, `EXPECT_SKIPPED=11`) permit the rise unchanged, and the exact skip count is unchanged. Arc: neutral-product-standalone-operability Lane: openxfactory-4 (openXfactory-4-openDox_extraction) Co-Authored-By: Claude Sonnet 5 --- src/opendox/runtime/config.py | 61 +++++++++++---- tests_runtime/conftest.py | 31 +------- tests_runtime/test_api_endpoints.py | 18 ++--- tests_runtime/test_migrations_apply.py | 7 -- tests_runtime/test_runtime_cli.py | 101 +++++++++++++++---------- tests_runtime/test_runtime_surface.py | 2 - 6 files changed, 113 insertions(+), 107 deletions(-) diff --git a/src/opendox/runtime/config.py b/src/opendox/runtime/config.py index ecc53d65..deaf53e9 100644 --- a/src/opendox/runtime/config.py +++ b/src/opendox/runtime/config.py @@ -81,7 +81,7 @@ class Setting: "it must not be the identity migrations run as", ), Setting( - PREFIX + "MIGRATION_DATABASE_URL", None, True, True, + PREFIX + "MIGRATION_DATABASE_URL", None, False, True, "the PRIVILEGED DSN ordered-SQL migrations are applied with, used by " "`opendox runtime migrate` alone and never by the served application", ), @@ -181,13 +181,17 @@ class RuntimeSettings: Construct with :func:`load_settings`; the fields are in `SETTINGS` order. - `migration_database_url` is never `None` from either loader (plan 034, - 13.3): `load_settings` requires it exactly as it requires `database_url`, - and `load_migration_settings` already refused to load without one. + `migration_database_url` is `None` from `load_settings` whenever + `OPENDOX_MIGRATION_DATABASE_URL` is unset — RULED "required only for + migrate" (openxFactory#656, on the claim thread for plan 034's T071, + 2026-09-28): the served workload never needs it, `deploy/compose/ + docker-compose.yaml`'s `opendox` service and `docs/runtime.md` § 3 never + supply it, and `load_migration_settings` is the loader that actually + requires one (unaffected by this: it already refused to load without one). """ database_url: str - migration_database_url: str + migration_database_url: str | None oidc_issuer: str oidc_audience: str oidc_jwks_url: str | None @@ -1304,9 +1308,16 @@ def effective_schema(dsn: str) -> str | None: POSTGRESQL_SCHEMES = frozenset({"postgresql", "postgres"}) -def _refuse_non_postgresql_dsn(name: str, dsn: str) -> None: +def _refuse_non_postgresql_dsn(name: str, dsn: str | None) -> None: """`name`'s DSN is refused unless it selects a PostgreSQL scheme. + `None` OR EMPTY IS A NO-OP, not a refusal: `OPENDOX_MIGRATION_DATABASE_URL` + is optional for `load_settings` (RULED "required only for migrate", + openxFactory#656, on the claim thread for plan 034's T071, 2026-09-28), + so an absent migration DSN has no dialect to check — the same shape + `_refuse_two_dsns_that_select_different_schemas` below already reads as + "nothing to compare" rather than as a fault. + A DSN in the keyword/value form (`host=h dbname=d …`) names NO DIALECT AT ALL — that syntax is libpq's own conninfo grammar, and no other driver reads it — so only the URI form is checked: `urlsplit` reports an EMPTY @@ -1322,6 +1333,8 @@ def _refuse_non_postgresql_dsn(name: str, dsn: str) -> None: round 24); this is its DSN-flavoured twin; a bad `OPENDOX_DATABASE_URL` is not "set it to the broker endpoint", so it is not reused verbatim. """ + if not dsn: + return try: scheme = urllib.parse.urlsplit(dsn).scheme except ValueError as exc: @@ -1339,9 +1352,18 @@ def _refuse_non_postgresql_dsn(name: str, dsn: str) -> None: "document (RULING Q1)") -def _refuse_the_same_dsn_in_both_settings(served: str, migration: str) -> None: +def _refuse_the_same_dsn_in_both_settings( + served: str, migration: str | None) -> None: """One credential pasted into both settings is refused (plan 034, 13.3). + A NO-OP WHEN MIGRATION IS ABSENT, exactly like + `_refuse_two_dsns_that_select_different_schemas` below: with nothing to + compare, there is nothing to have collapsed. `OPENDOX_MIGRATION_DATABASE_ + URL` is optional (RULED "required only for migrate", openxFactory#656, on + the claim thread for plan 034's T071, 2026-09-28) — but WHEN BOTH ARE + GIVEN, this refusal still applies, on every path `load_settings` serves, + not only the falsifier's. + `OPENDOX_DATABASE_URL` is the least-privileged identity the API serves with; `OPENDOX_MIGRATION_DATABASE_URL` is the privileged one ordered-SQL migrations run as — the whole point of keeping two settings. A @@ -1354,6 +1376,8 @@ def _refuse_the_same_dsn_in_both_settings(served: str, migration: str) -> None: credentials, not one pasted twice, and this checks the value actually given, not the identity it happens to resolve to. """ + if not migration: + return if served == migration: raise ConfigurationError( f"{PREFIX}MIGRATION_DATABASE_URL is the same value as " @@ -1477,21 +1501,26 @@ def load_settings(env: Mapping[str, str] | None = None) -> RuntimeSettings: fetch from the broker's JWKS, and discovering that on the first request means it is already serving. - BOTH DSNs ARE REQUIRED (plan 034, 13.3): `OPENDOX_MIGRATION_DATABASE_URL` - used to default to `None`, so the collapsed, single-role shape this - refuses (below) was reachable only by accident, through a caller who - happened to set both to the same value — an install that left the - migration credential unset was never asked the question at all. Naming it - explicitly, even at the same database a served DSN already names, is what - keeps the two identities two DECISIONS rather than one remembered twice. + `OPENDOX_MIGRATION_DATABASE_URL` STAYS OPTIONAL HERE (RULED "required only + for migrate", openxFactory#656, on the claim thread for plan 034's T071, + 2026-09-28): the served workload never needs it — + `deploy/compose/docker-compose.yaml`'s `opendox` service and + `docs/runtime.md` § 3 never supply it, keeping the two identities in + different containers — and `load_migration_settings` below is the loader + that actually requires one. It is never DEFAULTED from + `OPENDOX_DATABASE_URL` either way. WHEN BOTH ARE GIVEN, though, the two + checks below still apply: a non-PostgreSQL migration DSN is refused + (13.2), and the two being the exact same value is refused (13.3) — + optional does not mean unchecked. """ env = os.environ if env is None else env algorithms = _algorithms(env) served = _require(env, _by_name(PREFIX + "DATABASE_URL")) - migration = _require(env, _by_name(PREFIX + "MIGRATION_DATABASE_URL")) + migration = _optional(env, _by_name(PREFIX + "MIGRATION_DATABASE_URL")) # THE DIALECT FIRST: a scheme this module cannot parse as PostgreSQL is not - # yet a DSN worth comparing at all. + # yet a DSN worth comparing at all. A no-op on an ABSENT migration DSN — + # see `_refuse_non_postgresql_dsn`. _refuse_non_postgresql_dsn(PREFIX + "DATABASE_URL", served) _refuse_non_postgresql_dsn(PREFIX + "MIGRATION_DATABASE_URL", migration) # THEN WHETHER THEY DISAGREE. See `_refuse_two_dsns_that_select_different_ diff --git a/tests_runtime/conftest.py b/tests_runtime/conftest.py index 93f53a4d..2c6e0b7b 100644 --- a/tests_runtime/conftest.py +++ b/tests_runtime/conftest.py @@ -219,29 +219,6 @@ def postgres_dsn() -> str: return dsn -@pytest.fixture(scope="session") -def migration_dsn(postgres_dsn: str) -> str: - """A DSN for `OPENDOX_MIGRATION_DATABASE_URL`, distinct from `postgres_dsn`. - - Plan 034, 13.3: `load_settings` now REQUIRES this setting and refuses a - value that is simply `postgres_dsn` repeated (13.3's own collapse - refusal), so every fixture built from the pair below needs a second, - genuinely different string — not a second database. None of this - module's fixtures apply migrations through this identity: `database` - applies them directly with `MigrationRunner`, and the served app under - test is never asked to `migrate`. So `load_settings` is the only reader - that cares about this value at all, and what it asks of a DSN is a - PostgreSQL scheme (13.2), a STRING distinct from `postgres_dsn` (13.3), - and a database and schema that agree with it - (`_refuse_two_dsns_that_select_different_schemas`). A URI FRAGMENT is - invisible to `database_named_by`, `user_named_by` and `schema_selected_by` - alike — none of the three inspects `urlsplit(...).fragment` — so - appending one changes the STRING without moving the identity those - functions compare. - """ - return postgres_dsn + "#opendox-test-migration-identity" - - @pytest.fixture def database(postgres_dsn: str) -> Iterator[object]: """A `Database` on a throwaway schema, with the migrations already applied.""" @@ -380,7 +357,7 @@ def verifier(jwks_path: str): @pytest.fixture() -def client(database, postgres_dsn: str, migration_dsn: str, verifier): +def client(database, postgres_dsn: str, verifier): """A `TestClient` over the REAL application, on this test's own schema. The application is given its OWN `Database` on the same schema rather than @@ -399,7 +376,6 @@ def client(database, postgres_dsn: str, migration_dsn: str, verifier): settings = load_settings({ PREFIX + "DATABASE_URL": postgres_dsn, - PREFIX + "MIGRATION_DATABASE_URL": migration_dsn, PREFIX + "OIDC_ISSUER": TEST_ISSUER, PREFIX + "OIDC_AUDIENCE": TEST_AUDIENCE, }) @@ -423,8 +399,8 @@ def project_repository_root(tmp_path): @pytest.fixture() -def client_with_repositories(database, postgres_dsn: str, migration_dsn: str, - verifier, project_repository_root): +def client_with_repositories(database, postgres_dsn: str, verifier, + project_repository_root): """`client`, with the repository root pointed at this test's own directory.""" fastapi_testclient = _import_fastapi_testclient() from opendox.runtime.app import create_app @@ -433,7 +409,6 @@ def client_with_repositories(database, postgres_dsn: str, migration_dsn: str, settings = load_settings({ PREFIX + "DATABASE_URL": postgres_dsn, - PREFIX + "MIGRATION_DATABASE_URL": migration_dsn, PREFIX + "OIDC_ISSUER": TEST_ISSUER, PREFIX + "OIDC_AUDIENCE": TEST_AUDIENCE, PREFIX + "PROJECT_REPOSITORY_ROOT": str(project_repository_root), diff --git a/tests_runtime/test_api_endpoints.py b/tests_runtime/test_api_endpoints.py index 84caccfe..21897203 100644 --- a/tests_runtime/test_api_endpoints.py +++ b/tests_runtime/test_api_endpoints.py @@ -425,7 +425,7 @@ def test_the_unauthenticated_surface_is_exactly_the_two_probes( def test_the_schema_viewers_appear_only_when_the_install_says_so( - database, postgres_dsn: str, migration_dsn: str, verifier) -> None: + database, postgres_dsn: str, verifier) -> None: from fastapi.testclient import TestClient from opendox.runtime.app import create_app @@ -435,7 +435,6 @@ def test_the_schema_viewers_appear_only_when_the_install_says_so( settings = load_settings({ PREFIX + "DATABASE_URL": postgres_dsn, - PREFIX + "MIGRATION_DATABASE_URL": migration_dsn, PREFIX + "OIDC_ISSUER": TEST_ISSUER, PREFIX + "OIDC_AUDIENCE": TEST_AUDIENCE, PREFIX + "PUBLISH_OPENAPI": "true", @@ -449,7 +448,7 @@ def test_the_schema_viewers_appear_only_when_the_install_says_so( def test_readiness_refuses_an_unmigrated_database_by_name( - postgres_dsn: str, migration_dsn: str, verifier) -> None: + postgres_dsn: str, verifier) -> None: """`select 1` succeeds against a schema with no tables in it at all. Readiness without a migration check therefore turns a fresh install READY @@ -472,7 +471,6 @@ def test_readiness_refuses_an_unmigrated_database_by_name( try: settings = load_settings({ PREFIX + "DATABASE_URL": postgres_dsn, - PREFIX + "MIGRATION_DATABASE_URL": migration_dsn, PREFIX + "OIDC_ISSUER": TEST_ISSUER, PREFIX + "OIDC_AUDIENCE": TEST_AUDIENCE, }) @@ -590,8 +588,7 @@ def test_draft_pagination_returns_the_principals_drafts_not_an_empty_page( def test_readiness_refuses_a_database_whose_migration_file_has_changed( - database, postgres_dsn: str, migration_dsn: str, verifier, - tmp_path) -> None: + database, postgres_dsn: str, verifier, tmp_path) -> None: """Nothing pending is not the same as matching this tree.""" import shutil @@ -612,7 +609,6 @@ def test_readiness_refuses_a_database_whose_migration_file_has_changed( settings = load_settings({ PREFIX + "DATABASE_URL": postgres_dsn, - PREFIX + "MIGRATION_DATABASE_URL": migration_dsn, PREFIX + "OIDC_ISSUER": TEST_ISSUER, PREFIX + "OIDC_AUDIENCE": TEST_AUDIENCE, PREFIX + "MIGRATIONS_DIR": str(tmp_path), @@ -856,8 +852,7 @@ def test_a_hidden_user_and_a_missing_one_answer_byte_for_byte_the_same( def test_readiness_refuses_a_migrations_directory_without_the_pinned_0001( - database, postgres_dsn: str, migration_dsn: str, verifier, - tmp_path) -> None: + database, postgres_dsn: str, verifier, tmp_path) -> None: """An EMPTY directory made `plan()` and `drift()` both empty. `discover()` returns `[]` for a directory that exists and holds no @@ -879,7 +874,6 @@ def test_readiness_refuses_a_migrations_directory_without_the_pinned_0001( empty.mkdir() settings = load_settings({ PREFIX + "DATABASE_URL": postgres_dsn, - PREFIX + "MIGRATION_DATABASE_URL": migration_dsn, PREFIX + "OIDC_ISSUER": TEST_ISSUER, PREFIX + "OIDC_AUDIENCE": TEST_AUDIENCE, PREFIX + "MIGRATIONS_DIR": str(empty), @@ -1558,8 +1552,7 @@ def transaction(): def test_an_anonymous_token_naming_a_key_of_the_wrong_type_is_401_not_500( - database, postgres_dsn: str, migration_dsn: str, rsa_key_pair, - tmp_path: Path) -> None: + database, postgres_dsn: str, rsa_key_pair, tmp_path: Path) -> None: """The end of A25-2, measured where it was reported: at the HTTP boundary. A realm publishing an RSA and an EC signing key — Keycloak, the moment a @@ -1600,7 +1593,6 @@ def test_an_anonymous_token_naming_a_key_of_the_wrong_type_is_401_not_500( settings = load_settings({ PREFIX + "DATABASE_URL": postgres_dsn, - PREFIX + "MIGRATION_DATABASE_URL": migration_dsn, PREFIX + "OIDC_ISSUER": TEST_ISSUER, PREFIX + "OIDC_AUDIENCE": TEST_AUDIENCE, }) diff --git a/tests_runtime/test_migrations_apply.py b/tests_runtime/test_migrations_apply.py index 1b657628..2afc0fe1 100644 --- a/tests_runtime/test_migrations_apply.py +++ b/tests_runtime/test_migrations_apply.py @@ -345,11 +345,6 @@ def test_status_reports_a_reachable_database_and_its_applied_migrations( # libpq startup parameter `Database(schema=…)` sets for the harness. scoped = (f"{postgres_dsn}?options=-c%20search_path%3D{database.schema}") monkeypatch.setenv(PREFIX + "DATABASE_URL", scoped) - # A DISTINCT STRING (13.3's collapse refusal) THAT SELECTS THE SAME SCHEMA - # (`_refuse_two_dsns_that_select_different_schemas`): a URI FRAGMENT moves - # neither, since neither reader inspects one. - monkeypatch.setenv(PREFIX + "MIGRATION_DATABASE_URL", - scoped + "#opendox-test-migration-identity") monkeypatch.setenv(PREFIX + "OIDC_ISSUER", "https://broker.test/realms/x") monkeypatch.setenv(PREFIX + "OIDC_AUDIENCE", "opendox-runtime") monkeypatch.setenv(PREFIX + "MIGRATIONS_DIR", str(ROOT / "migrations")) @@ -725,8 +720,6 @@ def test_status_calls_an_unmigrated_database_unhealthy_and_blames_the_tree( conn.execute(f"create schema {schema}") try: monkeypatch.setenv(PREFIX + "DATABASE_URL", scoped) - monkeypatch.setenv(PREFIX + "MIGRATION_DATABASE_URL", - scoped + "#opendox-test-migration-identity") monkeypatch.setenv(PREFIX + "OIDC_ISSUER", "https://broker/realms/x") monkeypatch.setenv(PREFIX + "OIDC_AUDIENCE", "opendox-runtime") monkeypatch.setenv(PREFIX + "MIGRATIONS_DIR", str(ROOT / "migrations")) diff --git a/tests_runtime/test_runtime_cli.py b/tests_runtime/test_runtime_cli.py index 2c3c97eb..3b220eee 100644 --- a/tests_runtime/test_runtime_cli.py +++ b/tests_runtime/test_runtime_cli.py @@ -206,13 +206,66 @@ def test_migrate_refuses_rather_than_borrowing_the_served_identity( assert PREFIX + "MIGRATION_DATABASE_URL" in evidence["message"] +def test_serve_and_status_load_with_no_migration_dsn_configured( + monkeypatch: pytest.MonkeyPatch) -> None: + """Brett's ruling on #1144 13.3 ("Required only for migrate + (Recommended)"): `OPENDOX_MIGRATION_DATABASE_URL` stays OPTIONAL for the + served workload — `serve` and `status` both go through `load_settings`, + and this is the CLI-level proof that neither refuses at configuration + when only the served DSN is set. This is the shape + `deploy/compose/docker-compose.yaml`'s `opendox` service and + `docs/runtime.md` § 3 already document: the migration credential lives + only in the separate `migrate` service/profile. + """ + monkeypatch.delenv(PREFIX + "MIGRATION_DATABASE_URL", raising=False) + monkeypatch.setenv(PREFIX + "DATABASE_URL", + "postgresql://nobody@127.0.0.1:1/none") + monkeypatch.setenv(PREFIX + "OIDC_ISSUER", "https://broker/realms/x") + monkeypatch.setenv(PREFIX + "OIDC_AUDIENCE", "opendox-runtime") + + code, evidence = _run(cli.build_parser().parse_args( + ["runtime", "status", "--probe-timeout", "0.2"])) + assert evidence.get("refusal") != "configuration", evidence + assert "settings" in evidence, evidence + assert evidence["settings"][PREFIX + "MIGRATION_DATABASE_URL"] is None + + def _ok(self) -> None: + self.started = True + + _stub_uvicorn(monkeypatch, _ok) + monkeypatch.delenv(PREFIX + "MIGRATION_DATABASE_URL", raising=False) + code, evidence = _run_serve() + assert code == 0, evidence + assert evidence["ok"] is True, evidence + + +def test_the_collapse_is_refused_through_the_served_workload_too( + monkeypatch: pytest.MonkeyPatch) -> None: + """13.3's collapse refusal is `load_settings`'s own, not the falsifier's + special case: it fires for `status` (and every other served verb) too, + the moment an operator gives BOTH DSNs and they happen to be the exact + same value — even though migration is optional here (Brett's ruling, + above). + """ + same = "postgresql://opendox:hunter2@127.0.0.1:1/none" + monkeypatch.setenv(PREFIX + "DATABASE_URL", same) + monkeypatch.setenv(PREFIX + "MIGRATION_DATABASE_URL", same) + monkeypatch.setenv(PREFIX + "OIDC_ISSUER", "https://broker/realms/x") + monkeypatch.setenv(PREFIX + "OIDC_AUDIENCE", "opendox-runtime") + + code, evidence = _run(cli.build_parser().parse_args( + ["runtime", "status", "--probe-timeout", "0.2"])) + assert code == 1, evidence + assert evidence["refusal"] == "configuration", evidence + assert PREFIX + "MIGRATION_DATABASE_URL" in evidence["message"] + assert "hunter2" not in evidence["message"], evidence["message"] + + def test_init_creates_the_project_repository_root_and_touches_no_database( monkeypatch: pytest.MonkeyPatch, tmp_path) -> None: root = tmp_path / "projects" monkeypatch.setenv(PREFIX + "DATABASE_URL", "postgresql://nobody@127.0.0.1:1/none") - monkeypatch.setenv(PREFIX + "MIGRATION_DATABASE_URL", - "postgresql://migrate@127.0.0.1:1/none") monkeypatch.setenv(PREFIX + "OIDC_ISSUER", "https://broker/realms/x") monkeypatch.setenv(PREFIX + "OIDC_AUDIENCE", "opendox-runtime") monkeypatch.setenv(PREFIX + "PROJECT_REPOSITORY_ROOT", str(root)) @@ -364,8 +417,6 @@ def test_status_reports_every_declared_setting_and_none_as_null( from opendox.runtime.config import SETTING_NAMES monkeypatch.setenv(PREFIX + "DATABASE_URL", "postgresql://u:pw@127.0.0.1:1/x") - monkeypatch.setenv(PREFIX + "MIGRATION_DATABASE_URL", - "postgresql://m:pw@127.0.0.1:1/x") monkeypatch.setenv(PREFIX + "OIDC_ISSUER", "https://broker/realms/x") monkeypatch.setenv(PREFIX + "OIDC_AUDIENCE", "opendox-runtime") monkeypatch.setenv(PREFIX + "RUNTIME_PG_ROLE", "opendox_runtime") @@ -436,8 +487,6 @@ def test_init_refuses_a_repository_root_that_is_not_a_directory( root.write_text("not a directory", encoding="utf-8") monkeypatch.setenv(PREFIX + "DATABASE_URL", "postgresql://nobody@127.0.0.1:1/none") - monkeypatch.setenv(PREFIX + "MIGRATION_DATABASE_URL", - "postgresql://migrate@127.0.0.1:1/none") monkeypatch.setenv(PREFIX + "OIDC_ISSUER", "https://broker/realms/x") monkeypatch.setenv(PREFIX + "OIDC_AUDIENCE", "opendox-runtime") monkeypatch.setenv(PREFIX + "PROJECT_REPOSITORY_ROOT", str(root)) @@ -502,8 +551,6 @@ def run(self) -> None: monkeypatch.setitem(sys.modules, "opendox.runtime.app", app_stub) monkeypatch.setenv(PREFIX + "DATABASE_URL", "postgresql://nobody@127.0.0.1:1/none") - monkeypatch.setenv(PREFIX + "MIGRATION_DATABASE_URL", - "postgresql://migrate@127.0.0.1:1/none") monkeypatch.setenv(PREFIX + "OIDC_ISSUER", "https://broker/realms/x") monkeypatch.setenv(PREFIX + "OIDC_AUDIENCE", "opendox-runtime") code, evidence = _run(cli.build_parser().parse_args(["runtime", "serve"])) @@ -549,8 +596,6 @@ def run(self) -> None: monkeypatch.setitem(sys.modules, "opendox.runtime.app", app_stub) monkeypatch.setenv(PREFIX + "DATABASE_URL", "postgresql://nobody@127.0.0.1:1/none") - monkeypatch.setenv(PREFIX + "MIGRATION_DATABASE_URL", - "postgresql://migrate@127.0.0.1:1/none") monkeypatch.setenv(PREFIX + "OIDC_ISSUER", "https://broker/realms/x") monkeypatch.setenv(PREFIX + "OIDC_AUDIENCE", "opendox-runtime") code, evidence = _run(cli.build_parser().parse_args(["runtime", "serve"])) @@ -634,8 +679,6 @@ def __exit__(self, *exc: object) -> None: db_stub.Database = _Exploding # type: ignore[attr-defined] monkeypatch.setitem(sys.modules, "opendox.runtime.db", db_stub) monkeypatch.setenv(PREFIX + "DATABASE_URL", dsn) - monkeypatch.setenv(PREFIX + "MIGRATION_DATABASE_URL", - "postgresql://migrate@db.internal:5432/opendox") monkeypatch.setenv(PREFIX + "OIDC_ISSUER", "https://broker/realms/x") monkeypatch.setenv(PREFIX + "OIDC_AUDIENCE", "opendox-runtime") @@ -689,8 +732,6 @@ def __init__(self, config: object) -> None: monkeypatch.setitem(sys.modules, "opendox.runtime.app", app_stub) monkeypatch.setenv(PREFIX + "DATABASE_URL", "postgresql://nobody:hunter2@127.0.0.1:1/none") - monkeypatch.setenv(PREFIX + "MIGRATION_DATABASE_URL", - "postgresql://migrate@127.0.0.1:1/none") monkeypatch.setenv(PREFIX + "OIDC_ISSUER", "https://broker/realms/x") monkeypatch.setenv(PREFIX + "OIDC_AUDIENCE", "opendox-runtime") @@ -808,8 +849,6 @@ def connection(self): monkeypatch.setitem(sys.modules, "opendox.runtime.db", db_stub) monkeypatch.setenv(PREFIX + "DATABASE_URL", "postgresql://runtime:hunter2@127.0.0.1:5432/opendox") - monkeypatch.setenv(PREFIX + "MIGRATION_DATABASE_URL", - "postgresql://migrate@127.0.0.1:5432/opendox") monkeypatch.setenv(PREFIX + "OIDC_ISSUER", "https://broker/realms/x") monkeypatch.setenv(PREFIX + "OIDC_AUDIENCE", "opendox-runtime") monkeypatch.setenv(PREFIX + "MIGRATIONS_DIR", @@ -878,8 +917,6 @@ def transaction(self): monkeypatch.setitem(sys.modules, "opendox.runtime.db", db_stub) monkeypatch.setenv(PREFIX + "DATABASE_URL", "postgresql://runtime@127.0.0.1:5432/opendox") - monkeypatch.setenv(PREFIX + "MIGRATION_DATABASE_URL", - "postgresql://migrate@127.0.0.1:5432/opendox") monkeypatch.setenv(PREFIX + "OIDC_ISSUER", "https://broker/realms/x") monkeypatch.setenv(PREFIX + "OIDC_AUDIENCE", "opendox-runtime") monkeypatch.setenv(PREFIX + "PROJECT_REPOSITORY_ROOT", str(tmp_path)) @@ -1166,7 +1203,6 @@ def test_no_broker_url_this_runtime_prints_can_carry_a_credential() -> None: redacted_url) base = {PREFIX + "DATABASE_URL": "postgresql://u:p@h/db", - PREFIX + "MIGRATION_DATABASE_URL": "postgresql://m:p@h/db", PREFIX + "OIDC_AUDIENCE": "opendox-runtime"} for name in ("OIDC_ISSUER", "OIDC_JWKS_URL"): env = dict(base, **{PREFIX + "OIDC_ISSUER": "https://broker/realms/x"}) @@ -1214,7 +1250,6 @@ def test_a_credential_in_a_broker_urls_query_is_refused_like_one_in_its_userinfo redacted_url) base = {PREFIX + "DATABASE_URL": "postgresql://u:p@h/db", - PREFIX + "MIGRATION_DATABASE_URL": "postgresql://m:p@h/db", PREFIX + "OIDC_AUDIENCE": "opendox-runtime", PREFIX + "OIDC_ISSUER": "https://broker/realms/x"} for name, value, why in ( @@ -1263,7 +1298,6 @@ def test_the_broker_url_must_be_https_because_it_is_the_trust_anchor() -> None: from opendox.runtime.config import ConfigurationError, load_settings base = {PREFIX + "DATABASE_URL": "postgresql://u:p@h/db", - PREFIX + "MIGRATION_DATABASE_URL": "postgresql://m:p@h/db", PREFIX + "OIDC_AUDIENCE": "opendox-runtime"} for issuer in ("https://broker/realms/x", "http://localhost:8080/realms/x", "http://127.0.0.1:8080/realms/x", "http://[::1]:8080/x"): @@ -1445,7 +1479,6 @@ def test_a_broker_url_that_names_no_host_is_refused_at_the_door() -> None: from opendox.runtime.config import ConfigurationError, load_settings base = {PREFIX + "DATABASE_URL": "postgresql://u:p@h/db", - PREFIX + "MIGRATION_DATABASE_URL": "postgresql://m:p@h/db", PREFIX + "OIDC_AUDIENCE": "opendox-runtime"} for issuer in ("https:///realms/x", "https://", "https:///", "broker/realms/x", "https://user:hunter2@/realms/x"): @@ -1531,7 +1564,6 @@ def test_a_broker_url_whose_port_is_not_a_number_is_refused_at_the_door( from opendox.runtime.config import ConfigurationError, load_settings base = {PREFIX + "DATABASE_URL": "postgresql://u:p@h/db", - PREFIX + "MIGRATION_DATABASE_URL": "postgresql://m:p@h/db", PREFIX + "OIDC_AUDIENCE": "opendox-runtime"} for issuer in ("https://broker:not-a-port/realm", "https://broker:99999/realm", @@ -1577,7 +1609,6 @@ def test_the_issuer_carries_no_query_or_fragment_because_paths_are_appended( from opendox.runtime.config import ConfigurationError, load_settings base = {PREFIX + "DATABASE_URL": "postgresql://u:p@h/db", - PREFIX + "MIGRATION_DATABASE_URL": "postgresql://m:p@h/db", PREFIX + "OIDC_AUDIENCE": "opendox-runtime"} for issuer, component in (("https://broker/realms/x?tenant=a", "query"), ("https://broker/realms/x#frag", "fragment"), @@ -1623,7 +1654,6 @@ def test_the_loopback_exception_is_for_http_and_not_for_every_other_scheme( from opendox.runtime.config import ConfigurationError, load_settings base = {PREFIX + "DATABASE_URL": "postgresql://u:p@h/db", - PREFIX + "MIGRATION_DATABASE_URL": "postgresql://m:p@h/db", PREFIX + "OIDC_AUDIENCE": "opendox-runtime"} for issuer in ("ftp://localhost/realms/x", "file://127.0.0.1/realms/x", "ws://localhost:8080/realms/x", "ftp://[::1]/realms/x"): @@ -1935,7 +1965,6 @@ def test_a_malformed_broker_url_never_prints_its_own_password() -> None: issuer = "https://svc:hunter2@broker\u2100evil.example/realms/x" env = {PREFIX + "DATABASE_URL": "postgresql://u:p@h/db", - PREFIX + "MIGRATION_DATABASE_URL": "postgresql://m:p@h/db", PREFIX + "OIDC_AUDIENCE": "opendox", PREFIX + "OIDC_ISSUER": issuer} @@ -2152,16 +2181,11 @@ def test_two_dsns_that_select_different_schemas_are_refused() -> None: PREFIX + "DATABASE_URL": "postgresql://u:p@h/db", PREFIX + "MIGRATION_DATABASE_URL": "postgresql://m:p@h/db"}) - # AND A MIGRATION DSN THAT IS SIMPLY ABSENT is refused NOW (plan 034, - # 13.3), where it used to be accepted as "the documented single-role - # deployment, not a mismatch": that reading is exactly what `load_settings` - # reading the setting as OPTIONAL bought, and the setting stopped being - # optional. A single-role install still names both DSNs, distinctly — - # `test_a_dsn_that_names_no_database_still_reaches_one`'s "two different - # secrets for one role" is what single-role now looks like. - with pytest.raises(ConfigurationError) as absent: - load_settings({**base, PREFIX + "DATABASE_URL": served}) - assert PREFIX + "MIGRATION_DATABASE_URL" in str(absent.value) + # AND A MIGRATION DSN THAT IS SIMPLY ABSENT is the documented single-role + # deployment, not a mismatch (RULED "required only for migrate", + # openxFactory#656, on the claim thread for plan 034's T071, 2026-09-28 — + # this assertion was briefly the opposite of itself, reverted here). + assert load_settings({**base, PREFIX + "DATABASE_URL": served}) # THE OTHER DIRECTION IS REFUSED TOO: a served DSN that names no schema # beside a migration DSN that names one is the same split, and the @@ -2501,7 +2525,6 @@ def test_a_credential_shaped_parameter_name_is_a_WORD_and_not_a_substring() -> N assert redacted_url("https://broker/certs?token=x" ) == "https://broker/certs?token=" base = {PREFIX + "DATABASE_URL": "postgresql://u:p@h/db", - PREFIX + "MIGRATION_DATABASE_URL": "postgresql://m:p@h/db", PREFIX + "OIDC_ISSUER": "https://broker/realms/x", PREFIX + "OIDC_AUDIENCE": "opendox"} settings = load_settings({**base, @@ -2702,8 +2725,6 @@ def test_init_creates_the_repository_root_private_whatever_the_umask_is( root = tmp_path / "projects" monkeypatch.setenv(PREFIX + "DATABASE_URL", "postgresql://nobody@127.0.0.1:1/none") - monkeypatch.setenv(PREFIX + "MIGRATION_DATABASE_URL", - "postgresql://migrate@127.0.0.1:1/none") monkeypatch.setenv(PREFIX + "OIDC_ISSUER", "https://broker/realms/x") monkeypatch.setenv(PREFIX + "OIDC_AUDIENCE", "opendox-runtime") monkeypatch.setenv(PREFIX + "PROJECT_REPOSITORY_ROOT", str(root)) @@ -2740,8 +2761,6 @@ def test_init_reports_an_existing_root_s_mode_and_does_not_change_it( os.chmod(root, 0o755) monkeypatch.setenv(PREFIX + "DATABASE_URL", "postgresql://nobody@127.0.0.1:1/none") - monkeypatch.setenv(PREFIX + "MIGRATION_DATABASE_URL", - "postgresql://migrate@127.0.0.1:1/none") monkeypatch.setenv(PREFIX + "OIDC_ISSUER", "https://broker/realms/x") monkeypatch.setenv(PREFIX + "OIDC_AUDIENCE", "opendox-runtime") monkeypatch.setenv(PREFIX + "PROJECT_REPOSITORY_ROOT", str(root)) diff --git a/tests_runtime/test_runtime_surface.py b/tests_runtime/test_runtime_surface.py index b36c6203..ff6a73a3 100644 --- a/tests_runtime/test_runtime_surface.py +++ b/tests_runtime/test_runtime_surface.py @@ -258,7 +258,6 @@ def test_only_asymmetric_algorithms_can_be_configured_and_they_keep_pyjwts_spell ) base = {PREFIX + "DATABASE_URL": "postgresql://x/y", - PREFIX + "MIGRATION_DATABASE_URL": "postgresql://m@x/y", PREFIX + "OIDC_ISSUER": "https://broker/realms/x", PREFIX + "OIDC_AUDIENCE": "opendox-runtime"} for name in ASYMMETRIC_ALGORITHMS: @@ -319,7 +318,6 @@ def test_every_integer_setting_is_bounded_above_as_well_as_below() -> None: ) base = {PREFIX + "DATABASE_URL": "postgresql://u:p@h/db", - PREFIX + "MIGRATION_DATABASE_URL": "postgresql://m:p@h/db", PREFIX + "OIDC_AUDIENCE": "opendox", PREFIX + "OIDC_ISSUER": "https://broker/realms/x"} From f097fd889c1b07bad291794d41bad5a1969f6c5c Mon Sep 17 00:00:00 2001 From: Brett Heap <1513478+brettheap@users.noreply.github.com> Date: Mon, 28 Sep 2026 22:57:28 +0000 Subject: [PATCH 04/39] Fix round: load_migration_settings gets the same dialect gate load_settings has MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Copilot review of this PR (thread on _refuse_non_postgresql_dsn's own definition): 13.2's dialect gate was wired into `load_settings` only. `load_migration_settings` — the loader `runtime migrate`/`reset` actually use — read OPENDOX_MIGRATION_DATABASE_URL, checked only that it was non-empty, and handed it straight to `Database`, so a non-PostgreSQL migration DSN (`sqlite:///x.db`, say) reached the driver instead of being refused by name at configuration. That is the same un-named failure 13.2 exists to prevent for the served loader, just reachable through the one path F13.1's falsifier does not call. One call to the existing `_refuse_non_postgresql_dsn`, right after the existing empty-DSN refusal and before `database_url`/`migration_database_ url` are both set to the same value. New test `test_migrate_refuses_a_non_postgresql_migration_dsn_at_configuration` is the dialect-refused twin of the existing `test_migrate_and_reset_need_ no_served_identity_and_no_broker`, which already shows an unreachable but valid-dialect migration DSN getting PAST configuration — this one shows a wrong-dialect one refused AT configuration, naming the setting and never repeating the DSN. Measured locally (own Postgres container, bridge IP): 2473 passed (+1), 11 skipped, 1 failed (the same pre-existing, unrelated LC_CTYPE sandbox artifact) — the new test is the only change to the count. Arc: neutral-product-standalone-operability Lane: openxfactory-4 (openXfactory-4-openDox_extraction) Co-Authored-By: Claude Sonnet 5 --- src/opendox/runtime/config.py | 8 ++++++++ tests_runtime/test_runtime_cli.py | 23 +++++++++++++++++++++++ 2 files changed, 31 insertions(+) diff --git a/src/opendox/runtime/config.py b/src/opendox/runtime/config.py index deaf53e9..5c755006 100644 --- a/src/opendox/runtime/config.py +++ b/src/opendox/runtime/config.py @@ -1590,6 +1590,14 @@ def load_migration_settings(env: Mapping[str, str] | None = None) -> RuntimeSett f"{PREFIX}MIGRATION_DATABASE_URL is required to apply migrations; " f"{PREFIX}DATABASE_URL is the served runtime's least-privileged " "identity and is deliberately not used for schema changes") + # THE SAME DIALECT GATE `load_settings` ASKS, asked here too (Copilot + # review of this PR): this loader is the one path 13.2's own falsifier + # does not reach, and without this call a non-PostgreSQL migration DSN + # sailed past configuration entirely and reached `Database` instead, + # which is exactly the un-named, un-refused failure 13.2 exists to + # prevent for `load_settings`. `database_url` is set to this same `dsn` + # immediately below, so one call here covers both fields. + _refuse_non_postgresql_dsn(PREFIX + "MIGRATION_DATABASE_URL", dsn) return RuntimeSettings( database_url=dsn, migration_database_url=dsn, diff --git a/tests_runtime/test_runtime_cli.py b/tests_runtime/test_runtime_cli.py index 3b220eee..feadfa1a 100644 --- a/tests_runtime/test_runtime_cli.py +++ b/tests_runtime/test_runtime_cli.py @@ -386,6 +386,29 @@ def test_migrate_and_reset_need_no_served_identity_and_no_broker( assert evidence["ok"] is False +def test_migrate_refuses_a_non_postgresql_migration_dsn_at_configuration( + monkeypatch: pytest.MonkeyPatch) -> None: + """13.2 covers `load_migration_settings` too, not only `load_settings`. + + The test above shows an unreachable but VALID-dialect migration DSN + getting past configuration; this is its dialect-refused twin (Copilot + review of this PR): `load_migration_settings` read the DSN and handed it + straight to `Database` with no dialect check of its own, so a + non-PostgreSQL migration DSN reached the driver instead of being refused + by name here — the same un-named failure 13.2 exists to prevent for the + served loader. + """ + for name in ("DATABASE_URL", "OIDC_ISSUER", "OIDC_AUDIENCE"): + monkeypatch.delenv(PREFIX + name, raising=False) + monkeypatch.setenv(PREFIX + "MIGRATION_DATABASE_URL", "sqlite:///x.db") + code, evidence = _run(cli.build_parser().parse_args( + ["runtime", "migrate", "--connect-timeout", "0.2"])) + assert code == 1, evidence + assert evidence["refusal"] == "configuration", evidence + assert "postgres" in evidence["message"].lower(), evidence + assert PREFIX + "MIGRATION_DATABASE_URL" in evidence["message"] + + def test_the_entrypoint_turns_an_escaped_exception_into_evidence( monkeypatch: pytest.MonkeyPatch) -> None: """Every outcome is one redacted JSON object; none is a traceback.""" From b50e3b1dcb8c01a2619c987f3242ab9ba11e577d Mon Sep 17 00:00:00 2001 From: Brett Heap <1513478+brettheap@users.noreply.github.com> Date: Wed, 30 Sep 2026 00:21:49 +0000 Subject: [PATCH 05/39] =?UTF-8?q?T070:=2013.4,=2013.5=20and=2013.6=20?= =?UTF-8?q?=E2=80=94=20OPENDOX=5FINSTALL=5FMODE=20and=20generate-and-open?= =?UTF-8?q?=20--local=20(plan=20034)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit OPENDOX_INSTALL_MODE (`local` | `hosted`, default `hosted`) is read in runtime/config.py beside OPENDOX_OIDC_ISSUER and decides the install shape (#1144 13.4). `generate-and-open --local` makes the same selection (R1Q15 (b), as T007 batch H's 13.4 addendum reads); with neither the install is hosted (13.5). - A flag and a setting that disagree (`--local` beside OPENDOX_INSTALL_MODE=hosted) are refused, naming both. This is plan 034's fail-closed reading (Principle VII); no answer rules it and batch H does not write it into #1144. - LOCAL needs no broker: issuer, audience and key-set URL are empty. - LOCAL binds loopback only, with no opt-in. A non-loopback `--host` or OPENDOX_BIND_HOST is refused, naming the rule. The set is serve.py's own LOOPBACK_HOSTS, and a test holds the two equal. - HOSTED, set or by default, with no issuer refuses, naming OPENDOX_OIDC_ISSUER. generate-and-open asks the issuer first, so a run with nothing configured names it and `--local`. The hosted mode is otherwise unchanged (13.6). Holder readings on openxFactory#656 (Brett may overrule): - `runtime serve` refuses under local, because the API's identity is the broker's. - `runtime status` under local reports broker_keys "not configured (local mode)" and does not count it as a fault. - A broker setting beside local is refused by name. - An unrecognised mode value is refused, case-sensitively. The document server's generate-and-open resolves the shape before it scans, mints or binds anything. The hosted path loads the whole runtime configuration (R1Q16 (i); 13.4a). Also: - deploy/compose/.env.example gains OPENDOX_INSTALL_MODE=hosted, which test_every_runtime_setting_is_documented_in_env_example requires of every SETTINGS entry. - tests/test_doxbench_entrypoint.py's fixture now selects `--local` and scrubs the runtime settings, since the unset default is hosted and refuses with no issuer. Arc: neutral-product-standalone-operability Lane: openxfactory-4 (openXfactory-4-openDox_extraction) Co-Authored-By: Claude Opus 5.5 (1M context) --- deploy/compose/.env.example | 9 + src/opendox/cli.py | 64 ++++- src/opendox/runtime/cli.py | 39 ++- src/opendox/runtime/config.py | 240 ++++++++++++++++++- tests/test_doxbench_entrypoint.py | 11 + tests/test_install_mode_entrypoint.py | 208 ++++++++++++++++ tests_runtime/test_install_mode.py | 329 ++++++++++++++++++++++++++ 7 files changed, 889 insertions(+), 11 deletions(-) create mode 100644 tests/test_install_mode_entrypoint.py create mode 100644 tests_runtime/test_install_mode.py diff --git a/deploy/compose/.env.example b/deploy/compose/.env.example index 7d579a30..f8783755 100644 --- a/deploy/compose/.env.example +++ b/deploy/compose/.env.example @@ -48,6 +48,15 @@ OPENDOX_DATABASE_URL=postgresql://opendox_runtime:change-me-local-only-too@postg # `opendox-runtime runtime migrate` alone; the served application never receives it. OPENDOX_MIGRATION_DATABASE_URL=postgresql://opendox:change-me-local-only@postgres:5432/opendox +# The install shape: `hosted` or `local`, and this package is HOSTED — a +# broker, its pinned issuer and this file's own database. Unset means hosted +# too, which is the point: a hosted install that forgets its issuer REFUSES +# rather than falling into the local single-user mode (plan 034 T070; #1144 +# 13.4, 13.5). `local` is the one-user install `opendox generate-and-open +# --local` starts on a laptop, with no broker and loopback only; it refuses the +# broker settings below, so it is never selected here. +OPENDOX_INSTALL_MODE=hosted + # REQUIRED. The Keycloak broker's issuer, pinned (RULING Q2): a token from any # other issuer is refused rather than trusted. OPENDOX_OIDC_ISSUER=https://keycloak.example/realms/opendox diff --git a/src/opendox/cli.py b/src/opendox/cli.py index 994111dc..c7923abd 100644 --- a/src/opendox/cli.py +++ b/src/opendox/cli.py @@ -83,6 +83,11 @@ # `opendox.corpus_adapter` besides the stdlib. from opendox import corpus_adapter # noqa: E402 from opendox.runtime import local_git_adapter # noqa: E402 +# THE INSTALL SHAPE (plan 034 T070; #1144 13.4-13.6): `generate-and-open` +# resolves `--local` against `OPENDOX_INSTALL_MODE` here, before it generates +# or serves anything. Stdlib-only, like `local_git_adapter` above, which +# already imports it, so this adds no reach and no import weight. +from opendox.runtime import config as runtime_config # noqa: E402 from opendox.boundary import ( # noqa: E402 BoundaryViolation, HumanGate, OutputBoundary, ) @@ -409,11 +414,57 @@ def _validate(written: Path, args: argparse.Namespace, *, return 0 +def _resolve_install_shape(args: argparse.Namespace, + env=None) -> str: + """The install shape this run serves as, or `ConfigurationError` naming why. + + `--local` and `OPENDOX_INSTALL_MODE` are resolved by + `runtime_config.install_mode`, the one reading of the selector, which + refuses the two disagreeing (plan 034 T070's fail-closed reading) and + defaults to HOSTED (#1144 13.4, 13.5). Then each shape asks what it needs: + + * LOCAL binds loopback only, with no opt-in: a non-loopback `--host` is + refused naming the rule (13.4), and so is anything a local install + cannot be (`refuse_what_a_local_install_cannot_be`: a broker setting + beside it, or a non-loopback `OPENDOX_BIND_HOST`). It needs no broker. + Its datastore is 13.1's, and arrives with T072. + * HOSTED, set or by default, refuses with no issuer, NAMING THE ISSUER + (13.5), and then loads the runtime's whole configuration, because the + serving process is the one whose settings are the install's (13.4a; + R1Q16 (i)). Otherwise unchanged (13.6). + + Asked before anything is scanned, minted or bound, so a refused run leaves + nothing behind and exits at once rather than starting a server that a + bound would have to kill (F13.1's `test "$rc" -ne 124`). + """ + env = os.environ if env is None else env + mode = runtime_config.install_mode( + env, local_flag=bool(getattr(args, "local", False))) + if mode == runtime_config.INSTALL_MODE_LOCAL: + runtime_config.refuse_a_non_loopback_local_bind("--host", args.host) + runtime_config.refuse_what_a_local_install_cannot_be(env) + else: + runtime_config.require_the_hosted_issuer(env) + runtime_config.load_settings(env) + return mode + + def cmd_generate_and_open(args: argparse.Namespace, *, opener=webbrowser.open) -> int: """Regenerate the snapshot from the working tree into a run dir, start the local server, print the URL (ALWAYS), and open the browser. `--no-open` suppresses the browser; `--no-serve` returns after printing the URL without - blocking (used by tests). `opener` is injectable for testing.""" + blocking (used by tests). `opener` is injectable for testing. + + THE INSTALL SHAPE IS RESOLVED FIRST (plan 034 T070): `--local`, or + `OPENDOX_INSTALL_MODE=local`, selects the local single-user install, and + with neither the install is hosted — see `_resolve_install_shape`. A + refusal there is printed on stderr and the command exits 1, before any + other work.""" + try: + args.install_mode = _resolve_install_shape(args) + except runtime_config.ConfigurationError as exc: + print(f"generate-and-open refused: {exc}", file=sys.stderr) + return 1 # Ahead of minting the run dir, so a refused root leaves not even an empty # temp directory behind. `_generate_and_write` is still the guard that MATTERS # (it is the one no caller can skip); these are the same checks, earlier. @@ -968,6 +1019,17 @@ def build_parser(*, subcommand_extensions: tuple = ()) -> argparse.ArgumentParse gao.add_argument("--actor", default=None, help="human identity for loopback gate actions " "(default: the checkout's git user.name)") + # THE INSTALL SHAPE'S FLAG (plan 034 T070; R1Q15 (b), as T007 batch H's + # 13.4 addendum reads): the documented command is + # `opendox generate-and-open --local …`. The same selection as + # `OPENDOX_INSTALL_MODE=local`; with neither the install is hosted, and the + # flag beside `OPENDOX_INSTALL_MODE=hosted` is refused. + gao.add_argument(runtime_config.LOCAL_FLAG, action="store_true", + dest="local", + help="the LOCAL single-user install: no identity broker, " + "loopback only (the same selection as " + "OPENDOX_INSTALL_MODE=local; with neither, the " + "install is hosted and needs its broker's issuer)") gao.add_argument("--host", default=serve_mod.DEFAULT_HOST, help="bind host (default: 127.0.0.1, loopback only)") gao.add_argument("--port", type=int, default=0, help="bind port (default: ephemeral)") diff --git a/src/opendox/runtime/cli.py b/src/opendox/runtime/cli.py index 42921498..af0ac390 100644 --- a/src/opendox/runtime/cli.py +++ b/src/opendox/runtime/cli.py @@ -90,6 +90,8 @@ from opendox.runtime import identity, migrations from opendox.runtime.config import ( + INSTALL_MODE_LOCAL, + LOCAL_FLAG, SECRET_NAMES, SETTINGS, ConfigurationError, @@ -341,6 +343,11 @@ def _redacted_settings(settings: RuntimeSettings) -> dict[str, Any]: values = { "OPENDOX_DATABASE_URL": settings.database_url, "OPENDOX_MIGRATION_DATABASE_URL": settings.migration_database_url, + # THE INSTALL SHAPE this process loaded (plan 034 T070): a name and + # never a credential, and the first thing an operator reading `status` + # needs to know, because it decides whether the broker lines below + # mean anything at all. + "OPENDOX_INSTALL_MODE": settings.install_mode, # THE BROKER URLS ARE REDACTED HERE TOO. `load_settings` refuses # userinfo in the issuer and in an explicit JWKS URL — but this report # prints a DERIVED value, and a settings object can also be built by @@ -565,10 +572,31 @@ def cmd_migrate(args: argparse.Namespace) -> int: def cmd_serve(args: argparse.Namespace) -> int: - """Run the API. The pool is opened by the application's lifespan.""" + """Run the API. The pool is opened by the application's lifespan. + + NOT IN A LOCAL INSTALL (plan 034 T070; a holder reading on + openxFactory#656 that Brett may overrule). Every `/api/v1` route verifies + a token the BROKER signed (`oidc.build_verifier`), and the local mode has + no broker (#1144 13.4), so there is no identity this API could serve + with: started anyway, it would either refuse every request or, worse, + stand a local principal up that no task text defines. A local install is + served by `opendox generate-and-open --local`, and in release 1 its + document surface reads nothing from the store (R1Q16 (ii)). Refused + BEFORE anything is imported or bound, as evidence like every refusal. + """ settings = _settings_or_refusal(args) if isinstance(settings, int): return settings + if settings.install_mode == INSTALL_MODE_LOCAL: + return _emit({"verb": "serve", "refusal": "local-mode-has-no-broker", + "message": "the runtime API authenticates every request " + "with a token its identity broker signed, " + "and a LOCAL install has no broker, so this " + "API has no identity to serve with. A local " + "install is served by `opendox " + f"generate-and-open {LOCAL_FLAG}`; the " + "runtime API is a HOSTED install's surface " + "(13.4)"}, ok=False) try: import uvicorn @@ -744,6 +772,15 @@ def cmd_status(args: argparse.Namespace) -> int: f"unreachable: {type(exc).__name__}: {_safe_message(exc)}") ok = False + # A LOCAL INSTALL HAS NO BROKER TO PROBE (plan 034 T070; #1144 13.4), + # and that is its configuration rather than a fault: reported by name, and + # NOT counted against `ok`, so a healthy local install's `status` exits 0 + # — F13.1 runs it under `set -e`, and a verdict of "unhealthy" for a + # broker the install was never meant to have would be false. + if settings.install_mode == INSTALL_MODE_LOCAL: + report["broker_keys"] = "not configured (local mode)" + report["broker_discovery"] = None + return _emit(report, ok=ok) try: from opendox.runtime.oidc import build_verifier diff --git a/src/opendox/runtime/config.py b/src/opendox/runtime/config.py index 5c755006..6edd5c88 100644 --- a/src/opendox/runtime/config.py +++ b/src/opendox/runtime/config.py @@ -85,10 +85,23 @@ class Setting: "the PRIVILEGED DSN ordered-SQL migrations are applied with, used by " "`opendox runtime migrate` alone and never by the served application", ), + # THE INSTALL SHAPE, READ BESIDE THE ISSUER IT DECIDES ABOUT (plan 034 + # T070; #1144 13.4, 13.5). Not `required`: its default is the SAFE value, + # and "unset" is the case 13.4 names as the one that must be safe. + Setting( + PREFIX + "INSTALL_MODE", "hosted", False, False, + "the install shape: `hosted` (the default — the broker, the pinned " + "issuer and an operator's database, exactly as before) or `local` " + "(one user, no broker, loopback only). `generate-and-open --local` " + "makes the same selection; the two may not disagree, and with " + "neither the install is hosted, so a hosted install with no issuer " + "refuses rather than falling into local mode (13.4, 13.5)", + ), Setting( PREFIX + "OIDC_ISSUER", None, True, False, "the Keycloak broker's issuer, pinned: a token from any other issuer " - "is refused rather than trusted (RULING Q2)", + "is refused rather than trusted (RULING Q2). Required by a HOSTED " + "install; a LOCAL install has no broker and refuses one given here", ), Setting( PREFIX + "OIDC_AUDIENCE", None, True, False, @@ -188,10 +201,19 @@ class RuntimeSettings: docker-compose.yaml`'s `opendox` service and `docs/runtime.md` § 3 never supply it, and `load_migration_settings` is the loader that actually requires one (unaffected by this: it already refused to load without one). + + `install_mode` is `INSTALL_MODE_HOSTED` or `INSTALL_MODE_LOCAL` (plan 034 + T070; #1144 13.4). A LOCAL install has no broker, so its `oidc_issuer` and + `oidc_audience` are EMPTY and its `oidc_jwks_url` is `None` — never a + placeholder that looks like an endpoint — and `jwks_url()` and + `discovery_url()` answer the empty string for it rather than a path glued + onto nothing. A hosted install always carries a real issuer, because + `load_settings` refuses one without it. """ database_url: str migration_database_url: str | None + install_mode: str oidc_issuer: str oidc_audience: str oidc_jwks_url: str | None @@ -213,6 +235,7 @@ def __repr__(self) -> str: "RuntimeSettings(database_url=, " "migration_database_url=" f"{'' if self.migration_database_url else 'None'}, " + f"install_mode={self.install_mode!r}, " # REDACTED TOO, and not because `load_settings` allows userinfo # here — it refuses it. A `RuntimeSettings` built by hand, in a # test or by a future caller, does not go through that door, and @@ -241,13 +264,25 @@ def jwks_url(self) -> str: it saves one variable in the common case; setting it explicitly is what a broker behind a rewriting proxy needs, which is why the variable exists at all rather than the URL always being computed. + + EMPTY FOR A LOCAL INSTALL, which has no issuer to derive one from + (plan 034 T070): the derivation would otherwise answer the bare path + `/protocol/openid-connect/certs`, which `status` would print as if it + were a configured endpoint. """ if self.oidc_jwks_url: return self.oidc_jwks_url + if not self.oidc_issuer: + return "" return self.oidc_issuer.rstrip("/") + "/protocol/openid-connect/certs" def discovery_url(self) -> str: - """The issuer's discovery document, for `opendox runtime status`.""" + """The issuer's discovery document, for `opendox runtime status`. + + Empty for a local install, for the reason `jwks_url` gives. + """ + if not self.oidc_issuer: + return "" return self.oidc_issuer.rstrip("/") + "/.well-known/openid-configuration" @@ -1488,7 +1523,171 @@ def _refuse_two_dsns_that_select_different_schemas( "carries a password)") -def load_settings(env: Mapping[str, str] | None = None) -> RuntimeSettings: +#: THE TWO INSTALL SHAPES (plan 034 T070; #1144 13.4). One selector decides +#: the whole shape at once — the identity mode here, and the datastore source +#: 13.1 adds — because requirements 12 and 13 both describe "the standalone +#: install" and one deliberate choice should decide both. +INSTALL_MODE_HOSTED = "hosted" +INSTALL_MODE_LOCAL = "local" +INSTALL_MODES: tuple[str, ...] = (INSTALL_MODE_LOCAL, INSTALL_MODE_HOSTED) + +#: The flag that makes the same selection as `OPENDOX_INSTALL_MODE=local`, +#: spelled ONCE: `opendox.cli` declares `generate-and-open`'s option with this +#: constant, and every refusal below names it with the same one (R1Q15 (b), as +#: T007 batch H's 13.4 addendum reads). +LOCAL_FLAG = "--local" + +#: LOOPBACK, AS THE DOCUMENT SERVER ALREADY JUDGES IT. `serve.py` makes its +#: `session` capability conditional on a bind in exactly this set +#: (`serve.LOOPBACK_HOSTS`), and 13.4 asks the local mode to "make the same +#: judgement at the mode's own boundary" — so it is the same set, not +#: `_is_loopback` above: that one reads `127.0.0.0/8` as loopback, and a local +#: install bound to `127.0.0.2` would then pass here while the server it starts +#: treats that very bind as off-loopback. This module cannot import `serve` +#: (the import weight in `opendox/runtime/__init__.py`), so the set is spelled +#: here and `tests_runtime/test_install_mode.py` holds it equal to serve's. +LOCAL_BIND_HOSTS: frozenset[str] = frozenset({"127.0.0.1", "::1", "localhost"}) + +#: THE SETTINGS ONLY A HOSTED INSTALL READS. Given beside the local mode, each +#: is REFUSED, by name (a holder reading on openxFactory#656, plan 034 T070, +#: the same fail-closed reading as the disagreeing flag and setting): an issuer +#: next to `local` says a broker was meant, and honouring `local` over it would +#: silently drop the authentication the operator configured. T072 adds the two +#: DSNs, which the local install supplies itself (13.1). +HOSTED_ONLY_SETTINGS: tuple[str, ...] = ( + PREFIX + "OIDC_ISSUER", + PREFIX + "OIDC_AUDIENCE", + PREFIX + "OIDC_JWKS_URL", +) + + +def install_mode(env: Mapping[str, str] | None = None, *, + local_flag: bool = False) -> str: + """`INSTALL_MODE_LOCAL` or `INSTALL_MODE_HOSTED`, or a refusal naming why. + + THE DEFAULT IS HOSTED, and it is the default because it is the safe one + (#1144 13.4: "It is UNSET, not `local`, that must be safe"): an install + that sets nothing is hosted, and a hosted install with no issuer refuses + (13.5), so single-user operation is never reached by forgetting to + configure something. A BLANK value is unset, the reading `_optional` gives + every other setting. + + `local_flag` is `generate-and-open --local` (R1Q15 (b)). It selects local + exactly as `OPENDOX_INSTALL_MODE=local` does, and the two may not + DISAGREE: `--local` beside `OPENDOX_INSTALL_MODE=hosted` is refused naming + both, so no explicit selection is silently overridden by the other. No + answer on #656 rules that pair; the refusal is plan 034's fail-closed + reading (Principle VII, T070), recorded for Brett in + `evidence/analyze-round-2.md` and NOT written into #1144. + + AN UNRECOGNISED VALUE IS REFUSED, matched case-sensitively (a holder + reading on #656, T070): `Local` or `single-user` is not a spelling of + either shape, and guessing which one was meant is the one thing a + selector whose default is a safety property must not do. + """ + env = os.environ if env is None else env + setting = PREFIX + "INSTALL_MODE" + raw = env.get(setting, "").strip() + if raw and raw not in INSTALL_MODES: + raise ConfigurationError( + f"{setting} is {raw!r}, which is neither `local` nor `hosted` (the " + "two values are matched exactly, case included). Unset means " + "hosted; a single-user install selects `local` explicitly, with " + f"{setting}=local or `generate-and-open {LOCAL_FLAG}`") + if local_flag and raw == INSTALL_MODE_HOSTED: + raise ConfigurationError( + f"{LOCAL_FLAG} selects the LOCAL install and {setting}=hosted " + "selects the HOSTED one. Both are explicit selections and they " + "disagree, so neither is allowed to override the other: drop the " + f"flag for a hosted install, or unset {setting} (or set it to " + "`local`) for a local one") + if local_flag: + return INSTALL_MODE_LOCAL + return raw or INSTALL_MODE_HOSTED + + +def refuse_a_non_loopback_local_bind(name: str, host: str) -> None: + """A LOCAL install binds loopback only, with NO opt-in (#1144 13.4). + + `name` is what set the address — `--host` on `generate-and-open`, or + `OPENDOX_BIND_HOST` for the runtime's own listener — so the refusal names + the thing the operator actually typed. The local mode has no broker, so a + local install other machines can reach is an unauthenticated multi-user + service wearing the word "local"; an install that must be reachable from + another machine is a HOSTED install, with a broker. + """ + if host in LOCAL_BIND_HOSTS: + return + raise ConfigurationError( + f"{name} {host!r} is not a loopback address, and a LOCAL install binds " + f"LOOPBACK ONLY ({', '.join(sorted(LOCAL_BIND_HOSTS))}). The local " + "mode has no identity broker, so a local install another machine can " + "reach would be an unauthenticated multi-user service. There is no " + "opt-in: an install that must be reachable from another machine is a " + "HOSTED install, with a broker (13.4)") + + +def refuse_what_a_local_install_cannot_be(env: Mapping[str, str]) -> None: + """The two refusals a LOCAL install makes of its own environment. + + Every hosted-only setting given beside it (`HOSTED_ONLY_SETTINGS`), and a + non-loopback `OPENDOX_BIND_HOST`, the runtime's own listener. One function, + because `load_settings` and `generate-and-open --local` both ask it and the + two must not come to disagree about what a local install is. + """ + _refuse_hosted_only_settings(env) + refuse_a_non_loopback_local_bind( + PREFIX + "BIND_HOST", + _optional(env, _by_name(PREFIX + "BIND_HOST")) or "127.0.0.1") + + +def _refuse_hosted_only_settings(env: Mapping[str, str]) -> None: + """Every setting in `HOSTED_ONLY_SETTINGS` given beside `local`, named.""" + given = [name for name in HOSTED_ONLY_SETTINGS + if env.get(name, "").strip()] + if given: + raise ConfigurationError( + f"{' and '.join(given)} {'are' if len(given) > 1 else 'is'} set, " + "and this is a LOCAL install, which has no broker and reads " + f"{'none of them' if len(given) > 1 else 'none'}. A broker " + "setting beside the local mode says a HOSTED install was meant, " + "and honouring `local` over it would silently drop that " + f"authentication: unset {'them' if len(given) > 1 else 'it'} for " + f"a local install, or drop {LOCAL_FLAG} / " + f"{PREFIX}INSTALL_MODE=local for a hosted one (the values are not " + "repeated here)") + + +def require_the_hosted_issuer(env: Mapping[str, str] | None = None) -> None: + """A HOSTED install with no issuer refuses, NAMING THE ISSUER (#1144 13.5). + + `load_settings` asks for the served DSN before it asks for the issuer, so + a hosted `generate-and-open` run with NOTHING configured would otherwise + be refused naming `OPENDOX_DATABASE_URL` — true, and not the refusal 13.5 + and plan 034's requirement-13 scenario ask for, which is the one that + tells an operator this install is HOSTED and how to select the other one. + So the document server's entry point asks this first, and `load_settings` + keeps its own order for every verb that already relies on it (13.6: the + hosted mode is otherwise unchanged). + """ + env = os.environ if env is None else env + issuer = PREFIX + "OIDC_ISSUER" + if env.get(issuer, "").strip(): + return + selected = (f"{PREFIX}INSTALL_MODE=hosted" + if env.get(PREFIX + "INSTALL_MODE", "").strip() + else f"{PREFIX}INSTALL_MODE is unset, and unset means hosted") + raise ConfigurationError( + f"{issuer} is required and is not set, and this install is HOSTED " + f"({selected}). A hosted install authenticates through the broker " + "whose issuer this names, and it does NOT fall back to single-user " + "operation without one (13.5). A single-user install selects the " + f"local mode explicitly: `generate-and-open {LOCAL_FLAG}` or " + f"{PREFIX}INSTALL_MODE=local") + + +def load_settings(env: Mapping[str, str] | None = None, *, + local_flag: bool = False) -> RuntimeSettings: """Resolve :class:`RuntimeSettings` from `env` (default `os.environ`). Refuses with :class:`ConfigurationError` naming the variable — never with a @@ -1512,9 +1711,22 @@ def load_settings(env: Mapping[str, str] | None = None) -> RuntimeSettings: checks below still apply: a non-PostgreSQL migration DSN is refused (13.2), and the two being the exact same value is refused (13.3) — optional does not mean unchecked. + + THE INSTALL MODE FIRST (plan 034 T070; #1144 13.4-13.6). `local_flag` is + `generate-and-open --local`, resolved against `OPENDOX_INSTALL_MODE` by + `install_mode`, which refuses the two disagreeing. A HOSTED install — the + default — is exactly what this function has always loaded, in the same + order, with the issuer and audience required (13.6). A LOCAL install needs + no broker: its issuer, audience and key-set URL are empty, and any of the + three GIVEN beside it is refused (`HOSTED_ONLY_SETTINGS`); and its own + listener, `OPENDOX_BIND_HOST`, must be loopback, with no opt-in. """ env = os.environ if env is None else env + mode = install_mode(env, local_flag=local_flag) + local = mode == INSTALL_MODE_LOCAL + if local: + refuse_what_a_local_install_cannot_be(env) algorithms = _algorithms(env) served = _require(env, _by_name(PREFIX + "DATABASE_URL")) migration = _optional(env, _by_name(PREFIX + "MIGRATION_DATABASE_URL")) @@ -1538,18 +1750,23 @@ def load_settings(env: Mapping[str, str] | None = None) -> RuntimeSettings: # make. _refuse_the_same_dsn_in_both_settings(served, migration) + bind_host = _optional(env, _by_name(PREFIX + "BIND_HOST")) or "127.0.0.1" + return RuntimeSettings( database_url=served, migration_database_url=migration, - oidc_issuer=_broker_url(env, _by_name(PREFIX + "OIDC_ISSUER"), - required=True, is_a_base_url=True) or "", - oidc_audience=_require(env, _by_name(PREFIX + "OIDC_AUDIENCE")), - oidc_jwks_url=_broker_url(env, _by_name(PREFIX + "OIDC_JWKS_URL"), - required=False), + install_mode=mode, + oidc_issuer="" if local else _broker_url( + env, _by_name(PREFIX + "OIDC_ISSUER"), + required=True, is_a_base_url=True) or "", + oidc_audience="" if local else _require( + env, _by_name(PREFIX + "OIDC_AUDIENCE")), + oidc_jwks_url=None if local else _broker_url( + env, _by_name(PREFIX + "OIDC_JWKS_URL"), required=False), oidc_algorithms=algorithms, oidc_jwks_ttl_seconds=_positive_int(env, _by_name(PREFIX + "OIDC_JWKS_TTL_SECONDS")), oidc_leeway_seconds=_positive_int(env, _by_name(PREFIX + "OIDC_LEEWAY_SECONDS")), - bind_host=_optional(env, _by_name(PREFIX + "BIND_HOST")) or "127.0.0.1", + bind_host=bind_host, bind_port=_positive_int(env, _by_name(PREFIX + "BIND_PORT")), runtime_pg_role=_role_name(env), served_schema=_served_schema(env), @@ -1601,6 +1818,11 @@ def load_migration_settings(env: Mapping[str, str] | None = None) -> RuntimeSett return RuntimeSettings( database_url=dsn, migration_database_url=dsn, + # READ, SO AN UNRECOGNISED VALUE IS REFUSED HERE TOO (plan 034 T070): + # a migration run is part of the same install and one reading of the + # selector serves every verb. It changes nothing else a migration run + # does; the broker fields below are sentinels in either shape. + install_mode=install_mode(env), oidc_issuer=MIGRATION_SENTINEL_ISSUER, oidc_audience=MIGRATION_SENTINEL_AUDIENCE, oidc_jwks_url=None, diff --git a/tests/test_doxbench_entrypoint.py b/tests/test_doxbench_entrypoint.py index 0eac4803..2488fdf0 100644 --- a/tests/test_doxbench_entrypoint.py +++ b/tests/test_doxbench_entrypoint.py @@ -72,6 +72,7 @@ from opendox import doxbench_install as install_mod from opendox import doxbench_model from opendox import serve as serve_mod +from opendox.runtime import config as runtime_config def _handler_class(httpd): @@ -163,8 +164,18 @@ def _capture(*args, **kwargs): checkout = tmp_path / "checkout" checkout.mkdir() session_root = tmp_path / "model-sessions" + # THE LOCAL INSTALL, SELECTED EXPLICITLY (plan 034 T070; #1144 13.4, as + # T007 batch H's addendum reads). With neither `--local` nor + # `OPENDOX_INSTALL_MODE=local` the install is HOSTED, and a hosted install + # with no issuer refuses (13.5) before `build_server` is ever reached. This + # fixture drives the single-user entrypoint a student runs, so it says so, + # and it scrubs every runtime setting first: a broker setting inherited + # from the shell would be refused beside the local mode, by design. + for name in runtime_config.SETTING_NAMES: + monkeypatch.delenv(name, raising=False) args = cli_mod.build_parser().parse_args([ "generate-and-open", + runtime_config.LOCAL_FLAG, "--repo-root", str(checkout), "--repository", "fixture-repo", "--source-revision", PINNED_REVISION, diff --git a/tests/test_install_mode_entrypoint.py b/tests/test_install_mode_entrypoint.py new file mode 100644 index 00000000..2a121ef4 --- /dev/null +++ b/tests/test_install_mode_entrypoint.py @@ -0,0 +1,208 @@ +"""`generate-and-open`'s install shape: F13.1's refusals, run the way F13.1 +runs them (plan 034 T070; #1144 13.4, 13.5, 13.6). + +F13.1's refusal probes take the SAME `generate-and-open` path its local probe +takes, under `timeout 30`, and each must exit nonzero, must not be the bound's +124, and must name its rule on stderr. These cases run that path in a child +process for the same reason: a regression that silently STARTED a server +would block in-process forever, where a child is killed by the bound and the +case fails on `TimeoutExpired` instead of hanging the suite. The child is +`python -m opendox.cli`, which `cli.py`'s `__main__` block hands to the +package's `main()`, so it is the same `main()` the `opendox` console script +runs. + +Each probe gives a well-formed `--repo-root` (a fresh git repository) and every +other setting its case needs, so the install shape is the only fault; and the +install shape is resolved BEFORE the repo root is scanned, so the refusal is +the install's, whatever the tree holds. No child reaches a database, a broker +or a socket. + +The disagreeing flag and setting are plan 034's fail-closed reading (T070), +not a line of #1144; the rest is #1144's. +""" + +from __future__ import annotations + +import os +import subprocess +import sys +from pathlib import Path + +import pytest + +from opendox import cli as cli_mod +from opendox import serve as serve_mod +from opendox.runtime import config as runtime_config + +SRC = Path(__file__).resolve().parents[1] / "src" +PREFIX = runtime_config.PREFIX +MODE = PREFIX + "INSTALL_MODE" + +#: F13.1's hosted probes' environment: every setting a hosted install needs +#: EXCEPT the issuer, so the issuer is the only fault. +HOSTED_WITHOUT_ISSUER = { + PREFIX + "DATABASE_URL": "postgresql://serve@127.0.0.1:1/opendox", + PREFIX + "MIGRATION_DATABASE_URL": "postgresql://migrate@127.0.0.1:1/opendox", + PREFIX + "OIDC_AUDIENCE": "fixture", +} + + +@pytest.fixture() +def corpus(tmp_path: Path) -> Path: + """A fresh repository, as F13.1's preamble makes one.""" + root = tmp_path / "plain-documents" + root.mkdir() + (root / "note.md").write_text("# A note\n\nPlain text.\n", encoding="utf-8") + env = {**os.environ, + "GIT_AUTHOR_NAME": "fixture", "GIT_AUTHOR_EMAIL": "fixture@example.invalid", + "GIT_COMMITTER_NAME": "fixture", + "GIT_COMMITTER_EMAIL": "fixture@example.invalid"} + for argv in (["git", "init", "-q"], ["git", "add", "-A"], + ["git", "commit", "-qm", "fixture"]): + subprocess.run(argv, cwd=root, env=env, check=True) + return root + + +def _probe(corpus: Path, tmp_path: Path, *extra: str, + env: dict[str, str] | None = None) -> tuple[int, str]: + """One bounded `generate-and-open` child: `(returncode, stderr)`.""" + child_env = {name: value for name, value in os.environ.items() + if name not in runtime_config.SETTING_NAMES} + child_env.update(env or {}) + child_env["PYTHONPATH"] = os.pathsep.join( + [str(SRC), child_env.get("PYTHONPATH", "")]).rstrip(os.pathsep) + try: + done = subprocess.run( + [sys.executable, "-m", "opendox.cli", "generate-and-open", + "--repo-root", str(corpus), "--repository", "fixture", + "--run-dir", str(tmp_path / "run"), "--no-open", *extra], + env=child_env, capture_output=True, text=True, timeout=30) + except subprocess.TimeoutExpired as exc: # F13.1's rc 124 + raise AssertionError( + "generate-and-open did not refuse: it was still running when the " + "30-second bound killed it, which is a server that STARTED") from exc + return done.returncode, done.stderr + + +def test_local_mode_refuses_a_non_loopback_bind_naming_the_rule( + corpus: Path, tmp_path: Path) -> None: + """F13.1: `OPENDOX_INSTALL_MODE=local … --host 0.0.0.0` is refused, BOUNDED, + and stderr names loopback (13.4).""" + rc, err = _probe(corpus, tmp_path, "--host", "0.0.0.0", "--port", "0", + env={MODE: "local"}) + assert rc != 0, err + assert "loopback" in err.lower(), err + assert "--host" in err, err + + +def test_the_flag_refuses_a_non_loopback_bind_exactly_as_the_setting_does( + corpus: Path, tmp_path: Path) -> None: + rc, err = _probe(corpus, tmp_path, runtime_config.LOCAL_FLAG, + "--host", "0.0.0.0") + assert rc != 0, err + assert "loopback" in err.lower(), err + + +@pytest.mark.parametrize("mode", ["hosted", None]) +def test_a_hosted_or_unset_install_with_no_issuer_refuses_naming_it( + corpus: Path, tmp_path: Path, mode) -> None: + """F13.1's last two probes: hosted, and then the selector UNSET, each with + every hosted setting except the issuer; each refuses, BOUNDED, naming + `OPENDOX_OIDC_ISSUER` (13.5). The second is what proves the unset DEFAULT + refuses exactly as `hosted` does.""" + env = dict(HOSTED_WITHOUT_ISSUER) + if mode is not None: + env[MODE] = mode + rc, err = _probe(corpus, tmp_path, "--port", "0", env=env) + assert rc != 0, err + assert PREFIX + "OIDC_ISSUER" in err, err + + +def test_with_nothing_configured_the_refusal_names_the_issuer_and_the_flag( + corpus: Path, tmp_path: Path) -> None: + """Plan 034's requirement-13 scenario 2: with no setting at all the + install is hosted, and the refusal is about the ISSUER and names how to + select local — not `OPENDOX_DATABASE_URL`, which `load_settings` would + have asked for first.""" + rc, err = _probe(corpus, tmp_path, "--port", "0") + assert rc != 0, err + assert PREFIX + "OIDC_ISSUER" in err, err + assert runtime_config.LOCAL_FLAG in err, err + assert PREFIX + "DATABASE_URL" not in err, err + + +def test_a_flag_and_a_setting_that_disagree_are_refused_naming_both( + corpus: Path, tmp_path: Path) -> None: + """T070's own case (plan 034's fail-closed reading): `--local` beside + `OPENDOX_INSTALL_MODE=hosted`, with a COMPLETE hosted configuration, so + either selection alone would have been accepted.""" + env = {**HOSTED_WITHOUT_ISSUER, + PREFIX + "OIDC_ISSUER": "https://issuer.example.invalid/realms/x", + MODE: "hosted"} + rc, err = _probe(corpus, tmp_path, runtime_config.LOCAL_FLAG, env=env) + assert rc != 0, err + assert runtime_config.LOCAL_FLAG in err, err + assert f"{MODE}=hosted" in err, err + + +def test_a_broker_setting_beside_the_local_flag_is_refused_by_name( + corpus: Path, tmp_path: Path) -> None: + rc, err = _probe(corpus, tmp_path, runtime_config.LOCAL_FLAG, + env={PREFIX + "OIDC_ISSUER": + "https://issuer.example.invalid/realms/x"}) + assert rc != 0, err + assert PREFIX + "OIDC_ISSUER" in err, err + + +# -- in process: what each shape resolves to ---------------------------------- + + +def _args(*extra: str): + return cli_mod.build_parser().parse_args( + ["generate-and-open", "--repo-root", "/nonexistent", + "--repository", "fixture", *extra]) + + +def test_the_local_shape_needs_no_broker_and_no_setting_at_all() -> None: + """13.4: `local` needs no broker. Resolved with an EMPTY environment.""" + assert cli_mod._resolve_install_shape( + _args(runtime_config.LOCAL_FLAG), env={}) == \ + runtime_config.INSTALL_MODE_LOCAL + assert cli_mod._resolve_install_shape( + _args(), env={MODE: "local"}) == runtime_config.INSTALL_MODE_LOCAL + + +@pytest.mark.parametrize("host", sorted(serve_mod.LOOPBACK_HOSTS)) +def test_the_local_shape_accepts_each_loopback_host(host: str) -> None: + assert cli_mod._resolve_install_shape( + _args(runtime_config.LOCAL_FLAG, "--host", host), env={}) == \ + runtime_config.INSTALL_MODE_LOCAL + + +def test_a_complete_hosted_configuration_resolves_hosted_unchanged() -> None: + """13.6: a hosted install with its broker configured serves as before.""" + env = {**HOSTED_WITHOUT_ISSUER, + PREFIX + "OIDC_ISSUER": "https://issuer.example.invalid/realms/x"} + assert cli_mod._resolve_install_shape(_args(), env=env) == \ + runtime_config.INSTALL_MODE_HOSTED + # a hosted document server may still bind beyond loopback, as it always + # could: the loopback rule is the LOCAL mode's + assert cli_mod._resolve_install_shape( + _args("--host", "0.0.0.0"), env=env) == \ + runtime_config.INSTALL_MODE_HOSTED + + +def test_the_local_bind_rule_is_the_document_servers_own_loopback_set() -> None: + """13.4: "the same judgement at the mode's own boundary". `config` cannot + import `serve`, so it spells the set; this holds the two equal.""" + assert runtime_config.LOCAL_BIND_HOSTS == serve_mod.LOOPBACK_HOSTS + + +def test_the_flag_is_declared_on_generate_and_open_and_follows_the_verb() -> None: + """10.1: every option follows its verb; `--local` is `generate-and-open`'s.""" + assert _args(runtime_config.LOCAL_FLAG).local is True + assert _args().local is False + with pytest.raises(SystemExit): + cli_mod.build_parser().parse_args( + [runtime_config.LOCAL_FLAG, "generate-and-open", "--repo-root", + "/x", "--repository", "fixture"]) diff --git a/tests_runtime/test_install_mode.py b/tests_runtime/test_install_mode.py new file mode 100644 index 00000000..de9ca524 --- /dev/null +++ b/tests_runtime/test_install_mode.py @@ -0,0 +1,329 @@ +"""`OPENDOX_INSTALL_MODE`: the local single-user install, and the hosted one +it cannot be reached from by omission (plan 034 T070; #1144 13.4, 13.5, 13.6). + +HERMETIC: standard library plus `opendox.runtime.config` and the runtime CLI, +both stdlib-only at import. No database is reached: every DSN below is a +well-formed PostgreSQL URI aimed at port 1 of the loopback, so a refusal here +is always a CONFIGURATION refusal, which is the whole of what 13.4-13.6 ask of +`load_settings`. + +WHAT IS RULED AND WHAT IS READ, so a reviewer can tell them apart: + + * RULED: the selector, its two values and its hosted default (#1144 13.4); + a hosted install with no issuer refuses naming it (13.5); the hosted mode + unchanged (13.6); local binds loopback only with no opt-in (13.4); + `generate-and-open --local` is the same selection (R1Q15 (b), T007 batch + H's 13.4 addendum). + * PLAN 034's FAIL-CLOSED READING, not in #1144: a `--local` flag and an + `OPENDOX_INSTALL_MODE` setting that disagree are refused, naming both + (T070; `evidence/analyze-round-2.md` U2-1, V2-6). + * HOLDER READINGS on openxFactory#656 that Brett may overrule (T070): an + unrecognised value is refused, case-sensitively; a broker setting given + beside `local` is refused by name; `runtime serve` refuses under `local`; + `runtime status` under `local` reports the broker as not configured and + does not count it as a fault. +""" + +from __future__ import annotations + +import argparse +import io +import json +from contextlib import redirect_stdout + +import pytest + +from opendox.runtime import cli +from opendox.runtime.config import ( + HOSTED_ONLY_SETTINGS, + INSTALL_MODE_HOSTED, + INSTALL_MODE_LOCAL, + INSTALL_MODES, + LOCAL_BIND_HOSTS, + LOCAL_FLAG, + PREFIX, + SETTING_NAMES, + ConfigurationError, + install_mode, + load_migration_settings, + load_settings, + require_the_hosted_issuer, +) + +MODE = PREFIX + "INSTALL_MODE" + +#: Two DSNs that pass T071's three refusals — one dialect, one database, two +#: different values — so the install shape is the only thing under test. +DSNS = { + PREFIX + "DATABASE_URL": "postgresql://serve@127.0.0.1:1/opendox", + PREFIX + "MIGRATION_DATABASE_URL": "postgresql://migrate@127.0.0.1:1/opendox", +} +#: Every setting a HOSTED install needs, well-formed. +HOSTED = {**DSNS, + PREFIX + "OIDC_ISSUER": "https://issuer.example.invalid/realms/fixture", + PREFIX + "OIDC_AUDIENCE": "fixture"} + + +def _refusal(env: dict, **kwargs) -> str: + try: + load_settings(env, **kwargs) + except ConfigurationError as exc: + return str(exc) + raise AssertionError(f"accepted: {env} {kwargs}") + + +@pytest.fixture() +def scrubbed(monkeypatch: pytest.MonkeyPatch) -> pytest.MonkeyPatch: + """No runtime setting inherited from the shell reaches a CLI verb.""" + for name in SETTING_NAMES: + monkeypatch.delenv(name, raising=False) + return monkeypatch + + +def _run(argv: list[str]) -> tuple[int, dict]: + args: argparse.Namespace = cli.build_parser().parse_args(argv) + buffer = io.StringIO() + with redirect_stdout(buffer): + code = args.func(args) + return code, json.loads(buffer.getvalue()) + + +# -- the selector ------------------------------------------------------------- + + +def test_the_selector_has_two_values_and_its_default_is_hosted() -> None: + """13.4: `local` and `hosted`, defaulting to `hosted`; UNSET is the safe one.""" + assert set(INSTALL_MODES) == {"local", "hosted"} + assert install_mode({}) == INSTALL_MODE_HOSTED + assert install_mode({MODE: ""}) == INSTALL_MODE_HOSTED + assert install_mode({MODE: " "}) == INSTALL_MODE_HOSTED + assert install_mode({MODE: "hosted"}) == INSTALL_MODE_HOSTED + assert install_mode({MODE: "local"}) == INSTALL_MODE_LOCAL + + +def test_the_flag_selects_local_exactly_as_the_setting_does() -> None: + """R1Q15 (b), as T007 batch H's 13.4 addendum reads.""" + assert LOCAL_FLAG == "--local" + assert install_mode({}, local_flag=True) == INSTALL_MODE_LOCAL + assert install_mode({MODE: "local"}, local_flag=True) == INSTALL_MODE_LOCAL + assert install_mode({MODE: "local"}) == install_mode({}, local_flag=True) + + +def test_a_flag_and_a_setting_that_disagree_are_refused_naming_both() -> None: + """T070's falsifier's second half: the disagreeing pair (plan 034's reading). + + Neither explicit selection may silently override the other, in either + direction the loader can see: the flag cannot win over a hosted setting, + and the setting cannot win over the flag. + """ + with pytest.raises(ConfigurationError) as caught: + install_mode({MODE: "hosted"}, local_flag=True) + message = str(caught.value) + assert LOCAL_FLAG in message, message + assert f"{MODE}=hosted" in message, message + # ...and the same refusal on the loader every served verb goes through, + # so the pair cannot be resolved one way here and another way there. + assert f"{MODE}=hosted" in _refusal({**HOSTED, MODE: "hosted"}, + local_flag=True) + + +@pytest.mark.parametrize("value", ["Local", "LOCAL", "Hosted", "single-user", + "locals", "loc al"]) +def test_an_unrecognised_value_is_refused_naming_the_two(value: str) -> None: + """A holder reading (#656, T070): matched case-sensitively, never guessed.""" + with pytest.raises(ConfigurationError) as caught: + install_mode({MODE: value}) + message = str(caught.value) + assert MODE in message and "`local`" in message and "`hosted`" in message + # every loader reads the one selector, so every loader refuses it + assert MODE in _refusal({**HOSTED, MODE: value}) + with pytest.raises(ConfigurationError): + load_migration_settings({**DSNS, MODE: value}) + + +# -- 13.5 and 13.6: the hosted install --------------------------------------- + + +@pytest.mark.parametrize("mode", [None, "hosted", ""]) +def test_a_hosted_install_with_no_issuer_refuses_naming_it(mode) -> None: + """13.5, set or by default, with every other hosted setting well-formed.""" + env = {k: v for k, v in HOSTED.items() if k != PREFIX + "OIDC_ISSUER"} + if mode is not None: + env[MODE] = mode + assert PREFIX + "OIDC_ISSUER" in _refusal(env) + with pytest.raises(ConfigurationError) as caught: + require_the_hosted_issuer(env) + assert PREFIX + "OIDC_ISSUER" in str(caught.value) + assert LOCAL_FLAG in str(caught.value), ( + "the refusal must say how a single-user install selects local") + + +def test_the_hosted_issuer_is_asked_first_by_the_entry_point_check() -> None: + """With NOTHING set, the entry point's refusal names the issuer (13.5), + not the served DSN `load_settings` happens to ask for first.""" + with pytest.raises(ConfigurationError) as caught: + require_the_hosted_issuer({}) + assert PREFIX + "OIDC_ISSUER" in str(caught.value) + assert "unset" in str(caught.value) + require_the_hosted_issuer(HOSTED) # and a present issuer passes + + +def test_the_hosted_mode_is_unchanged() -> None: + """13.6: same broker, same pinned issuer, loaded exactly as before.""" + for env in (HOSTED, {**HOSTED, MODE: "hosted"}): + settings = load_settings(env) + assert settings.install_mode == INSTALL_MODE_HOSTED + assert settings.oidc_issuer == HOSTED[PREFIX + "OIDC_ISSUER"] + assert settings.oidc_audience == "fixture" + assert settings.jwks_url() == (HOSTED[PREFIX + "OIDC_ISSUER"] + + "/protocol/openid-connect/certs") + # and a hosted install may still bind wherever its operator says + assert load_settings({**HOSTED, PREFIX + "BIND_HOST": "0.0.0.0"} + ).bind_host == "0.0.0.0" + # and it still refuses a missing audience, as it always did + env = {k: v for k, v in HOSTED.items() if k != PREFIX + "OIDC_AUDIENCE"} + assert PREFIX + "OIDC_AUDIENCE" in _refusal(env) + + +# -- 13.4: the local install ------------------------------------------------- + + +@pytest.mark.parametrize("selection", ["setting", "flag"]) +def test_a_local_install_needs_no_broker(selection: str) -> None: + env = dict(DSNS) + kwargs = {} + if selection == "setting": + env[MODE] = "local" + else: + kwargs["local_flag"] = True + settings = load_settings(env, **kwargs) + assert settings.install_mode == INSTALL_MODE_LOCAL + assert settings.oidc_issuer == "" + assert settings.oidc_audience == "" + assert settings.oidc_jwks_url is None + # no endpoint is derived from an issuer that does not exist + assert settings.jwks_url() == "" + assert settings.discovery_url() == "" + assert "install_mode='local'" in repr(settings) + + +@pytest.mark.parametrize("name", HOSTED_ONLY_SETTINGS) +def test_a_broker_setting_beside_the_local_mode_is_refused_by_name( + name: str) -> None: + """A holder reading (#656, T070): a broker setting says hosted was meant.""" + assert set(HOSTED_ONLY_SETTINGS) == {PREFIX + "OIDC_ISSUER", + PREFIX + "OIDC_AUDIENCE", + PREFIX + "OIDC_JWKS_URL"} + secret = "https://svc:hunter2@broker.example.invalid/realms/x" + message = _refusal({**DSNS, MODE: "local", name: secret}) + assert name in message, message + assert "hunter2" not in message, "the value must not be repeated" + # and the flag spelling of the same selection refuses it the same way + assert name in _refusal({**DSNS, name: secret}, local_flag=True) + + +def test_every_broker_setting_given_is_named_at_once() -> None: + env = {**HOSTED, MODE: "local"} + message = _refusal(env) + for name in (PREFIX + "OIDC_ISSUER", PREFIX + "OIDC_AUDIENCE"): + assert name in message, message + + +@pytest.mark.parametrize("host", sorted(LOCAL_BIND_HOSTS)) +def test_a_local_install_binds_each_loopback_spelling(host: str) -> None: + settings = load_settings({**DSNS, MODE: "local", + PREFIX + "BIND_HOST": host}) + assert settings.bind_host == host + + +@pytest.mark.parametrize("host", ["0.0.0.0", "::", "192.0.2.10", + "127.0.0.2", "example.invalid"]) +def test_a_local_install_refuses_a_non_loopback_bind_naming_the_rule( + host: str) -> None: + """13.4: loopback ONLY, and no opt-in. `127.0.0.2` is refused too: the + document server does not treat it as loopback (`serve.LOOPBACK_HOSTS`), + and the mode makes the SAME judgement at its own boundary.""" + message = _refusal({**DSNS, MODE: "local", PREFIX + "BIND_HOST": host}) + assert PREFIX + "BIND_HOST" in message, message + assert "loopback" in message.lower(), message + assert "no opt-in" in message.lower(), message + + +# -- the runtime CLI under the two modes -------------------------------------- + + +def test_runtime_serve_refuses_under_the_local_mode(scrubbed) -> None: + """A holder reading (#656, T070): the API's identity is the broker's. + + uvicorn and the application are STUBBED, so that a regression which + served anyway returns at once — and fails the assertions below — instead + of binding a real listener and blocking the suite forever (measured: the + un-stubbed form of this case hung under exactly that mutant). + """ + import sys + import types + + from opendox.runtime import app as app_module + + served = [] + + class _Config: + def __init__(self, app: object, **kwargs: object) -> None: + pass + + class _Server: + def __init__(self, config: object) -> None: + self.started = False + + def run(self) -> None: + served.append(True) + self.started = True + + stub = types.ModuleType("uvicorn") + stub.Config, stub.Server = _Config, _Server + scrubbed.setitem(sys.modules, "uvicorn", stub) + scrubbed.setattr(app_module, "create_app", lambda **kwargs: object()) + for name, value in DSNS.items(): + scrubbed.setenv(name, value) + scrubbed.setenv(MODE, "local") + code, evidence = _run(["runtime", "serve"]) + assert served == [], "the API was started for a LOCAL install" + assert code == 1 + assert evidence["ok"] is False + assert evidence["refusal"] == "local-mode-has-no-broker", evidence + assert f"generate-and-open {LOCAL_FLAG}" in evidence["message"] + + +def test_runtime_status_under_the_local_mode_probes_no_broker( + scrubbed, monkeypatch: pytest.MonkeyPatch) -> None: + """`broker_keys` is reported as not configured, and the verifier is never + built: a local install has no broker to reach, and a status verb that + called that a fault would exit nonzero for a healthy install.""" + from opendox.runtime import oidc + + def _no_broker(_settings): + raise AssertionError("status built a broker verifier for a LOCAL " + "install, which has no broker") + + monkeypatch.setattr(oidc, "build_verifier", _no_broker) + for name, value in DSNS.items(): + scrubbed.setenv(name, value) + scrubbed.setenv(MODE, "local") + code, evidence = _run(["runtime", "status", "--probe-timeout", "0.2"]) + assert evidence.get("refusal") is None, evidence + assert evidence["broker_keys"] == "not configured (local mode)" + assert evidence["broker_discovery"] is None + assert evidence["settings"][MODE] == INSTALL_MODE_LOCAL + assert evidence["settings"][PREFIX + "OIDC_ISSUER"] == "" + assert evidence["settings"][PREFIX + "OIDC_JWKS_URL"] == "" + # the database half (port 1, unreachable) is the ONLY reason `ok` is false + assert evidence["database"].startswith("unreachable"), evidence + assert code == 1 + + +def test_runtime_status_reports_the_hosted_mode_it_loaded(scrubbed) -> None: + for name, value in HOSTED.items(): + scrubbed.setenv(name, value) + _code, evidence = _run(["runtime", "status", "--probe-timeout", "0.2"]) + assert evidence["settings"][MODE] == INSTALL_MODE_HOSTED + assert evidence["broker_keys"].startswith("unreachable"), evidence From 2c96dfbbbfefe29cd75eb6000ec0ba8c087d5f19 Mon Sep 17 00:00:00 2001 From: Brett Heap <1513478+brettheap@users.noreply.github.com> Date: Wed, 30 Sep 2026 00:47:47 +0000 Subject: [PATCH 06/39] =?UTF-8?q?T072:=2013.1=20=E2=80=94=20the=20bundled?= =?UTF-8?q?=20PostgreSQL=20server,=20the=20local=20install's=20own=20child?= =?UTF-8?q?=20(plan=20034)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A LOCAL install (T070's `generate-and-open --local`, or OPENDOX_INSTALL_MODE=local) now brings its own database (#1144 13.1, as T007 batch H's addendum reads; RULED R1Q16 (i)-(iv), 5850003126). - (i) `generate-and-open --local` starts a PostgreSQL server as its own direct child (subprocess.Popen, never pg_ctl) and reports it. The document server a user reaches is the process that owns it. - (ii) The server is started AND migrated: initdb once, an idempotent bootstrap (the database, the served role, and the compose stack's grants narrowed to this install's owner), then migrations.MigrationRunner as the owner, with the served role and database declared. - (iii) It ships as the `opendox[local]` extra: `opendox[runtime]` plus `pgserver>=0.1.4`, whose bundled binaries link only libc and libz. The `test` extra joins it, so F9.1's `.[test]` install still runs every case. - (iv) It stops with the entry point. SIGTERM is read as the Ctrl-C the serve loop already stops on, followed by a fast shutdown. PR_SET_PDEATHSIG is the backstop when the entry point is SIGKILLed. - Its data and socket directories live under OPENDOX_STATE_DIR, a new setting that defaults per user and must be absolute. The server listens on a 0700 Unix socket with listen_addresses empty: no TCP listener at all. - Both DSNs are supplied: two users over the one socket, which pass T071's three checks. An operator DSN beside `local` is refused by name, joining T070's broker settings (a holder reading on openxFactory#656). - `runtime status` reports database_bundle (data_dir, socket_dir, pid). `runtime migrate` under local migrates the bundle. THE MIGRATIONS GAP (assigned to T072 by the holder). pyproject maps migrations/*.sql into the wheel's data directory (share/opendox/migrations), without moving the root migrations/ that the image copies. An unset OPENDOX_MIGRATIONS_DIR is `migrations` wherever the working directory has one (today's default, unchanged), and otherwise the copy the installed distribution records. A test builds the wheel, installs it outside the checkout, runs from a directory with no migrations/, and migrates the bundled server. Also: - deploy/compose/.env.example gains OPENDOX_STATE_DIR=, because every SETTINGS entry is named there. - tests/test_doxbench_entrypoint.py stands the bundle in, since those cases test the model port. - T070's own tests stop passing DSNs beside `local`. Arc: neutral-product-standalone-operability Lane: openxfactory-4 (openXfactory-4-openDox_extraction) Co-Authored-By: Claude Opus 5.5 (1M context) --- deploy/compose/.env.example | 7 + pyproject.toml | 48 +++ src/opendox/cli.py | 82 ++++- src/opendox/runtime/bundle.py | 444 +++++++++++++++++++++++ src/opendox/runtime/cli.py | 18 + src/opendox/runtime/config.py | 267 +++++++++++++- tests/test_doxbench_entrypoint.py | 26 ++ tests/test_install_mode_entrypoint.py | 22 +- tests_runtime/local_entrypoint_driver.py | 77 ++++ tests_runtime/test_bundled_postgres.py | 415 +++++++++++++++++++++ tests_runtime/test_install_mode.py | 37 +- tests_runtime/test_runtime_surface.py | 5 + 12 files changed, 1397 insertions(+), 51 deletions(-) create mode 100644 src/opendox/runtime/bundle.py create mode 100644 tests_runtime/local_entrypoint_driver.py create mode 100644 tests_runtime/test_bundled_postgres.py diff --git a/deploy/compose/.env.example b/deploy/compose/.env.example index f8783755..5916932d 100644 --- a/deploy/compose/.env.example +++ b/deploy/compose/.env.example @@ -57,6 +57,13 @@ OPENDOX_MIGRATION_DATABASE_URL=postgresql://opendox:change-me-local-only@postgre # broker settings below, so it is never selected here. OPENDOX_INSTALL_MODE=hosted +# Where a LOCAL install keeps its bundled PostgreSQL server's data directory and +# Unix socket (plan 034 T072; #1144 13.1). This package is hosted and never +# reads it; it is here because every setting the runtime reads is named here. +# Empty means the per-user default, `$XDG_STATE_HOME/opendox` or +# `~/.local/state/opendox`. +OPENDOX_STATE_DIR= + # REQUIRED. The Keycloak broker's issuer, pinned (RULING Q2): a token from any # other issuer is refused rather than trusted. OPENDOX_OIDC_ISSUER=https://keycloak.example/realms/opendox diff --git a/pyproject.toml b/pyproject.toml index 10a82163..7be64df0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -93,9 +93,44 @@ dependencies = [ # (`tests_runtime/conftest.py`), and a skipped case does not make a whole # suite. `opendox[runtime]` names the extra below instead of copying its five # lines, so the runtime's dependency list is still declared once. +# +# AND THE LOCAL EXTRA, the same way and for the same reason (plan 034 T072; +# R1Q16 (iii), `5850003126`): the local install's bundled PostgreSQL server is +# what `tests_runtime/test_bundled_postgres.py` starts, so F9.1's `.[test]` +# install still runs every case. `opendox[local]` carries `opendox[runtime]`; +# both are named so the T036 reason above stays readable on its own. +# +# AND `setuptools`, the backend `[build-system]` names: the same test BUILDS +# this package's wheel and installs it outside the checkout, to prove a wheel +# carries its migrations (plan 034 T072). Declared here so that build runs +# offline, with `--no-build-isolation`, in the environment the suite runs in. +# `>=70.1` and not `[build-system]`'s `>=68`: 70.1 is the first release that +# builds a wheel with no separate `wheel` package installed. test = [ "pytest>=8,<9", "opendox[runtime]", + "opendox[local]", + "setuptools>=70.1", +] + +# THE LOCAL EXTRA — the standalone install (plan 034 T072; #1144 13.1, as T007 +# batch H's addendum reads; RULED R1Q16 (iii), openxFactory#656 comment +# `5850003126`): `pip install "opendox[local]"`, then +# `opendox generate-and-open --local …`. It carries the runtime's packages and +# the bundled server's own, and nothing else. +# +# `pgserver` IS THE SERVER'S CARRIER, and only its binaries are used (see +# `opendox/runtime/bundle.py` for why its own manager is not): it ships +# PostgreSQL 16 as `initdb` and `postgres` inside the wheel, built to link only +# libc and libz, so a manylinux2014 host needs nothing else installed — no +# system PostgreSQL, no ICU. Apache-2.0; the server it carries is under the +# PostgreSQL License. `>=0.1.4` is the release this package has been exercised +# against (measured 2026-09-30, python 3.12.3: PostgreSQL 16.2), the same rule +# every floor in this file is set by. Its own three requirements (psutil, +# platformdirs, fasteners) arrive with it and are imported by nothing here. +local = [ + "opendox[runtime]", + "pgserver>=0.1.4", ] # THE RUNTIME EXTRA — `split-opendox-two-layer-product` § 3.5, RULED Q2 @@ -206,6 +241,19 @@ py-modules = ["route_extension", "subcommand_extension"] [tool.setuptools.packages.find] where = ["src"] +# THE MIGRATIONS, SHIPPED IN THE WHEEL (plan 034 T072). `migrations/` stays at +# the repository root, where the image copies it (`deploy/compose/Dockerfile`: +# `COPY migrations ./migrations`, run from `/app`) and every checkout run finds +# it; this maps the same files into the wheel's data directory, so an install +# run OUTSIDE a checkout still has the migrations its bundled server applies. +# `opendox.runtime.config.packaged_migrations_dir` finds them through the +# installed distribution's own record, and the canonical digest gate is what +# proves the copy found is the pinned one. A data directory and not package +# data because package data must live inside the package, and the one tree is +# not moved. +[tool.setuptools.data-files] +"share/opendox/migrations" = ["migrations/*.sql"] + # THE SERVED BUNDLE, PACKAGED — § 3.4 slice S5, and it is a gap this slice had # to close to make RULED Q5 true rather than true-in-a-source-checkout. # diff --git a/src/opendox/cli.py b/src/opendox/cli.py index c7923abd..11c3b203 100644 --- a/src/opendox/cli.py +++ b/src/opendox/cli.py @@ -18,6 +18,7 @@ import argparse import os +import signal import sys import tempfile import webbrowser @@ -88,6 +89,12 @@ # or serves anything. Stdlib-only, like `local_git_adapter` above, which # already imports it, so this adds no reach and no import weight. from opendox.runtime import config as runtime_config # noqa: E402 +# THE LOCAL INSTALL'S BUNDLED POSTGRESQL SERVER (plan 034 T072; #1144 13.1, +# R1Q16 (i)-(iv)): started as THIS process's child by `generate-and-open +# --local`, and stopped with it. Stdlib-only at import, like `runtime_config`; +# the driver is imported when the server is started, never here. +from opendox.runtime import bundle as bundle_mod # noqa: E402 +from opendox.runtime import migrations as migrations_mod # noqa: E402 from opendox.boundary import ( # noqa: E402 BoundaryViolation, HumanGate, OutputBoundary, ) @@ -415,8 +422,8 @@ def _validate(written: Path, args: argparse.Namespace, *, def _resolve_install_shape(args: argparse.Namespace, - env=None) -> str: - """The install shape this run serves as, or `ConfigurationError` naming why. + env=None) -> runtime_config.RuntimeSettings: + """The settings this run serves with, or `ConfigurationError` naming why. `--local` and `OPENDOX_INSTALL_MODE` are resolved by `runtime_config.install_mode`, the one reading of the selector, which @@ -426,27 +433,35 @@ def _resolve_install_shape(args: argparse.Namespace, * LOCAL binds loopback only, with no opt-in: a non-loopback `--host` is refused naming the rule (13.4), and so is anything a local install cannot be (`refuse_what_a_local_install_cannot_be`: a broker setting - beside it, or a non-loopback `OPENDOX_BIND_HOST`). It needs no broker. - Its datastore is 13.1's, and arrives with T072. + beside it, a DSN beside it, or a non-loopback `OPENDOX_BIND_HOST`). It + needs no broker, and it supplies BOTH DSNs itself, from the server it + bundles under `OPENDOX_STATE_DIR` (13.1; plan 034 T072). * HOSTED, set or by default, refuses with no issuer, NAMING THE ISSUER - (13.5), and then loads the runtime's whole configuration, because the - serving process is the one whose settings are the install's (13.4a; - R1Q16 (i)). Otherwise unchanged (13.6). + (13.5), and then loads the runtime's whole configuration. + Otherwise unchanged (13.6). + + Either way the result is the runtime's own `load_settings`, because the + serving process is the one whose settings are the install's (13.4a; + R1Q16 (i)). Asked before anything is scanned, minted or bound, so a refused run leaves nothing behind and exits at once rather than starting a server that a bound would have to kill (F13.1's `test "$rc" -ne 124`). """ env = os.environ if env is None else env - mode = runtime_config.install_mode( - env, local_flag=bool(getattr(args, "local", False))) + local_flag = bool(getattr(args, "local", False)) + mode = runtime_config.install_mode(env, local_flag=local_flag) if mode == runtime_config.INSTALL_MODE_LOCAL: runtime_config.refuse_a_non_loopback_local_bind("--host", args.host) runtime_config.refuse_what_a_local_install_cannot_be(env) else: runtime_config.require_the_hosted_issuer(env) - runtime_config.load_settings(env) - return mode + return runtime_config.load_settings(env, local_flag=local_flag) + + +def _terminate_as_interrupt(signum, frame): # pragma: no cover - a signal + """SIGTERM, read as the Ctrl-C the serve loop already stops cleanly on.""" + raise KeyboardInterrupt def cmd_generate_and_open(args: argparse.Namespace, *, opener=webbrowser.open) -> int: @@ -459,12 +474,53 @@ def cmd_generate_and_open(args: argparse.Namespace, *, opener=webbrowser.open) - `OPENDOX_INSTALL_MODE=local`, selects the local single-user install, and with neither the install is hosted — see `_resolve_install_shape`. A refusal there is printed on stderr and the command exits 1, before any - other work.""" + other work. + + A LOCAL RUN OWNS ITS DATABASE (plan 034 T072; #1144 13.1, R1Q16 (i)-(iv)). + Once the corpus root and the anchors are known good, the bundled + PostgreSQL server is started as THIS process's child, bootstrapped and + migrated — starting and migrating it is all release 1 asks of it — and it + is stopped when this command returns, however it returns: a served run + ended by Ctrl-C or by SIGTERM (read here as the same interrupt), a + `--no-serve` run, or a failure. A refused start exits 1 on stderr, like + every other refusal of this verb.""" try: - args.install_mode = _resolve_install_shape(args) + settings = _resolve_install_shape(args) except runtime_config.ConfigurationError as exc: print(f"generate-and-open refused: {exc}", file=sys.stderr) return 1 + args.install_mode = settings.install_mode + args.runtime_settings = settings + if settings.install_mode != runtime_config.INSTALL_MODE_LOCAL: + return _generate_and_open(args, opener=opener) + # THE CHEAP REFUSALS FIRST, so a mistyped root never costs a database start. + _refuse_non_corpus_repo_root(args) + _refuse_malformed_generated_at(args) + server = bundle_mod.BundledServer(settings) + args.database_bundle = server + try: + previous = signal.signal(signal.SIGTERM, _terminate_as_interrupt) + except ValueError: # not the main thread: no handler to own + previous = None + try: + try: + server.start() + except (bundle_mod.BundleRefused, runtime_config.ConfigurationError, + migrations_mod.MigrationError) as exc: + print(f"generate-and-open refused: {exc}", file=sys.stderr) + return 1 + report = server.report() + print(f" database {report['socket_dir']} (bundled, pid {report['pid']}, " + f"migrations applied now: {server.applied or 'none pending'})") + return _generate_and_open(args, opener=opener) + finally: + server.stop() + if previous is not None: + signal.signal(signal.SIGTERM, previous) + + +def _generate_and_open(args: argparse.Namespace, *, opener) -> int: + """`generate-and-open`'s generate-then-serve half, once the install is known.""" # Ahead of minting the run dir, so a refused root leaves not even an empty # temp directory behind. `_generate_and_write` is still the guard that MATTERS # (it is the one no caller can skip); these are the same checks, earlier. diff --git a/src/opendox/runtime/bundle.py b/src/opendox/runtime/bundle.py new file mode 100644 index 00000000..701aa1bf --- /dev/null +++ b/src/opendox/runtime/bundle.py @@ -0,0 +1,444 @@ +"""A LOCAL install's bundled PostgreSQL server (plan 034 T072; #1144 13.1). + +WHAT #1144 FIXES AND WHAT IT LEAVES. 13.1 fixes the server's IDENTITY: its data +directory and its Unix socket live under the install's own `OPENDOX_STATE_DIR`, +it listens on that socket and on NO TCP port, the product supplies BOTH DSNs +itself, and `runtime status` reports a `database_bundle` naming `data_dir`, +`socket_dir` and the server's `pid`. It leaves the PACKAGING to the realization, +and R1Q16's answer (openxFactory#656 `5850003126`, as T007 batch H's 13.1 +addendum records it) names it: + + (i) the document server starts the bundled server as ITS OWN CHILD and + reports it, so the process a user reaches is the one that owns it; + (ii) starting and migrating the store is all release 1 asks of it, since + the document surface reads nothing from it yet; + (iii) it ships as the `opendox[local]` extra, which carries the runtime's + packages and the server's own; + (iv) it stops with the entry point. + +THE SERVER'S OWN PACKAGE IS `pgserver` (pyproject.toml's `local` extra), and +only its BINARIES are used: `initdb` and `postgres` from the wheel's +`pginstall/bin`, found by `importlib.util.find_spec` without importing +`pgserver` at all. Its Python manager is deliberately not used — it daemonizes +the server through `pg_ctl`, which re-parents it away from this process +(against (i)), shares one server between processes by reference count and +stops it from `atexit` (which a SIGTERM never runs, against (iv)), and may put +the socket under the user's runtime directory instead of the state directory +(against 13.1). The binaries themselves link only libc and libz, so they run on +any manylinux2014 host, which a wheel that links the system's ICU does not. + +THE LIFECYCLE, IN FULL: + + * `initdb` once per data directory: the MIGRATION identity + (`config.BUNDLE_OWNER_ROLE`) is the bootstrap superuser, local connections + are `trust` and host connections are `reject`, UTF-8 in the `C` locale. + Trust is safe BECAUSE of the socket: its directory is 0700, owned by the + user running the install, and the server opens no TCP port at all, so + reaching the socket is the credential and nothing else can. + * `postgres` started as a DIRECT CHILD of this process (`subprocess.Popen`, + never `pg_ctl`), with `listen_addresses` empty and the socket directory + given. On Linux it also carries `PR_SET_PDEATHSIG`, so an entry point + killed without any chance to clean up (SIGKILL) still takes its server + with it; an ordinary stop is `stop()`, a fast shutdown. + * a bootstrap, idempotent: the database, the SERVED role with no password + and the grants `deploy/compose/init-runtime-role.sh` makes for the compose + stack's role of the same name, narrowed to what this install's owner + creates. + * the ordered-SQL migrations, through `migrations.MigrationRunner`, as the + owner, with the served role and the database declared, so the run narrows + the ledger and verifies the served role's access exactly as a hosted + `runtime migrate` does. + +IMPORT WEIGHT: standard library and `opendox.runtime.{config,migrations}` at +module level. `psycopg` and `opendox.runtime.db` are imported at CALL time, +inside the functions that connect, like every other module the +`tests_runtime/test_runtime_surface.py` contract names. +""" + +from __future__ import annotations + +import ctypes +import importlib.util +import os +import signal +import subprocess +import sys +import time +from pathlib import Path +from typing import Any + +from opendox.runtime import migrations +from opendox.runtime.config import ( + BUNDLE_DATABASE, + BUNDLE_OWNER_ROLE, + BUNDLE_PORT, + BUNDLE_SERVED_ROLE, + LOCAL_FLAG, + PREFIX, + DatabaseBundle, + RuntimeSettings, + database_bundle, +) + +#: The distribution the `local` extra installs for the server's binaries. +SERVER_DISTRIBUTION = "pgserver" + +#: How long a start may take before it is a failure: `initdb` on a slow disk, +#: plus the server's own recovery on a data directory an earlier run did not +#: stop cleanly. +START_TIMEOUT_SECONDS = 60.0 + +#: How long a fast shutdown may take before the server is told to stop NOW. +STOP_TIMEOUT_SECONDS = 30.0 + +#: `prctl(2)`'s option number for the parent-death signal (linux/prctl.h). +_PR_SET_PDEATHSIG = 1 + + +class BundleRefused(Exception): + """The bundled server could not be started, named. Carries no credential. + + One exception, because the caller does nothing different for any of them: + the local install does not start. + """ + + +def server_binaries() -> Path: + """The directory holding the bundled `initdb` and `postgres`, or a refusal. + + Found WITHOUT importing `pgserver`: its package initializer imports its + manager, which this module does not use and whose import creates a lock + object under the user's runtime directory as a side effect. + """ + spec = importlib.util.find_spec(SERVER_DISTRIBUTION) + locations = list(spec.submodule_search_locations or ()) if spec else [] + if not locations: + raise BundleRefused( + "the local install's PostgreSQL server is not installed: it " + "arrives with the `local` extra, `pip install \"opendox[local]\"` " + "(R1Q16 (iii)). A local install brings its own database and never " + "borrows one") + binaries = Path(locations[0]) / "pginstall" / "bin" + missing = [name for name in ("initdb", "postgres") + if not os.access(binaries / name, os.X_OK)] + if missing: + raise BundleRefused( + f"the `{SERVER_DISTRIBUTION}` package is installed but carries no " + f"executable {' or '.join(missing)} under {binaries}; reinstall " + "the `local` extra") + return binaries + + +def running_pid(bundle: DatabaseBundle) -> int | None: + """The pid of a live server on `bundle`'s data directory, or `None`. + + Read from the server's own `postmaster.pid` (its first line), and only + believed while that process exists: a file left by a server that did not + stop cleanly names a pid that is gone, or that the kernel has since given + to something else, and PostgreSQL itself treats such a file as stale. + """ + try: + first = (bundle.data_dir / "postmaster.pid").read_text( + encoding="utf-8").splitlines()[0] + pid = int(first.strip()) + except (OSError, IndexError, ValueError): + return None + try: + os.kill(pid, 0) + except ProcessLookupError: + return None + except PermissionError: + return pid # alive, and not ours to signal + # A REUSED PID IS NOT A SERVER: where the kernel says what runs there, it + # must be a postgres, or the file is stale whatever its first line says. + try: + command = Path(f"/proc/{pid}/cmdline").read_bytes() + except OSError: + return pid # no /proc to ask; believe the live pid + return pid if b"postgres" in command else None + + +def report(bundle: DatabaseBundle) -> dict[str, Any]: + """The `database_bundle` block `runtime status` prints (#1144 13.1).""" + return {"data_dir": str(bundle.data_dir), + "socket_dir": str(bundle.socket_dir), + "pid": running_pid(bundle)} + + +def _child_environment() -> dict[str, str]: + """This process's environment, less libpq's and PostgreSQL's own variables. + + `PGDATA`, `PGPORT`, `PGHOST`, `PGOPTIONS` and the rest would otherwise + steer `initdb` and the server somewhere other than the paths given on + their command lines; the command line is the configuration, stated once. + """ + return {name: value for name, value in os.environ.items() + if not name.startswith("PG")} + + +def _die_with_parent(): + """A `preexec_fn` that signals the server when its parent goes away (iv). + + `PR_SET_PDEATHSIG` with SIGINT, PostgreSQL's FAST shutdown, so an entry + point killed outright (SIGKILL, an OOM kill) still stops its server + instead of leaving it running with the socket held. Linux only, and `None` + elsewhere, where the ordinary `stop()` is the whole of (iv). + + `prctl` is RESOLVED HERE, in the parent, so the forked child only calls + it; and the child re-checks its parent afterwards, because a parent that + died between the fork and the `prctl` would never deliver the signal. + """ + if not sys.platform.startswith("linux"): + return None + try: + prctl = ctypes.CDLL(None, use_errno=True).prctl + except (OSError, AttributeError): # pragma: no cover - a libc without it + return None + parent = os.getpid() + + def _preexec() -> None: # pragma: no cover - runs in the child + prctl(_PR_SET_PDEATHSIG, int(signal.SIGINT)) + if os.getppid() != parent: + os._exit(1) + + return _preexec + + +class BundledServer: + """One local install's PostgreSQL server: started as this process's child. + + Built from the settings a LOCAL `load_settings` returned, so the DSNs it + serves are the ones every other verb of the same install derives. `start` + initializes, launches, bootstraps and migrates; `stop` shuts it down; both + are safe to call more than once, and it is a context manager. + """ + + def __init__(self, settings: RuntimeSettings) -> None: + self.settings = settings + self.bundle = database_bundle(settings.state_dir) + self.process: subprocess.Popen[bytes] | None = None + self.applied: list[str] = [] + + @property + def log_path(self) -> Path: + return self.bundle.data_dir.parent / "postgres.log" + + def report(self) -> dict[str, Any]: + live = self.process is not None and self.process.poll() is None + return {"data_dir": str(self.bundle.data_dir), + "socket_dir": str(self.bundle.socket_dir), + "pid": self.process.pid if live else None} + + # -- start ------------------------------------------------------------- + + def start(self) -> BundledServer: + if hasattr(os, "geteuid") and os.geteuid() == 0: + raise BundleRefused( + "the bundled PostgreSQL server refuses to run as root, and so " + "does this local install: run it as the ordinary user whose " + "documents it serves") + binaries = server_binaries() + already = running_pid(self.bundle) + if already is not None: + raise BundleRefused( + f"a server is already running on {self.bundle.data_dir} " + f"(pid {already}). One local install's database belongs to one " + "entry point at a time: stop the other `generate-and-open " + f"{LOCAL_FLAG}`, or give this one its own {PREFIX}STATE_DIR") + self._prepare_directories() + if not (self.bundle.data_dir / "PG_VERSION").is_file(): + self._initdb(binaries) + self._launch(binaries) + try: + self._wait_until_ready() + self._bootstrap() + self._migrate() + except (BundleRefused, migrations.MigrationError): + self.stop() + raise + except Exception as exc: # noqa: BLE001 - the driver's, named not quoted + # THE DRIVER'S TEXT IS NOT REPEATED, for the reason `runtime/cli.py` + # gives: it quotes the connection string it was handed. The class + # names what went wrong and the server's own log says the rest. + self.stop() + raise BundleRefused( + f"the bundled PostgreSQL server started but could not be " + f"prepared ({type(exc).__name__}); its log is " + f"{self.log_path}") from None + except BaseException: + self.stop() + raise + return self + + def _prepare_directories(self) -> None: + """The state, data and socket directories, each 0700 where this makes it. + + A directory that already exists is NOT re-moded, on the rule the + runtime's `init` keeps for an operator's own path — except the socket + directory, whose mode IS the access control of a `trust` server: it is + this install's own, under its own state directory, and it is narrowed + to 0700 whatever it was. + """ + for directory in (self.bundle.state_dir, self.bundle.data_dir.parent): + directory.mkdir(mode=0o700, parents=True, exist_ok=True) + self.bundle.socket_dir.mkdir(mode=0o700, parents=True, exist_ok=True) + os.chmod(self.bundle.socket_dir, 0o700) + + def _initdb(self, binaries: Path) -> None: + done = subprocess.run( + [str(binaries / "initdb"), "-D", str(self.bundle.data_dir), + "-U", BUNDLE_OWNER_ROLE, "--auth-local=trust", "--auth-host=reject", + "--encoding=UTF8", "--locale=C", "--no-instructions"], + env=_child_environment(), capture_output=True, text=True, + timeout=START_TIMEOUT_SECONDS) + if done.returncode != 0: + detail = (done.stderr or done.stdout).strip().splitlines()[-5:] + raise BundleRefused( + f"initdb could not initialize {self.bundle.data_dir} " + f"(exit {done.returncode}): {' | '.join(detail)}") + + def _launch(self, binaries: Path) -> None: + log = open(self.log_path, "ab") # noqa: SIM115 - the child keeps it + try: + self.process = subprocess.Popen( + [str(binaries / "postgres"), "-D", str(self.bundle.data_dir), + "-k", str(self.bundle.socket_dir), "-p", str(BUNDLE_PORT), + # NO TCP LISTENER (13.1): an empty `listen_addresses` opens + # no TCP socket at all, so no pre-existing service on any + # port can stand in for this server, and nothing off this + # machine can reach it. + "-c", "listen_addresses=", + "-c", "unix_socket_permissions=0700"], + stdin=subprocess.DEVNULL, stdout=log, stderr=subprocess.STDOUT, + env=_child_environment(), + # ITS OWN SESSION, so a terminal's Ctrl-C reaches this process + # and not the server behind its back: the stop is `stop()`'s, + # in order, after the document server has closed. + start_new_session=True, + preexec_fn=_die_with_parent()) + finally: + log.close() + + def _wait_until_ready(self) -> None: + import psycopg + + deadline = time.monotonic() + START_TIMEOUT_SECONDS + last = "no attempt" + while time.monotonic() < deadline: + if self.process is not None and self.process.poll() is not None: + raise BundleRefused( + f"the bundled PostgreSQL server exited during start " + f"(exit {self.process.returncode}); its log is " + f"{self.log_path}: {self._log_tail()}") + try: + with psycopg.connect(self._dsn(BUNDLE_OWNER_ROLE, "postgres"), + connect_timeout=2, autocommit=True) as conn: + conn.execute("select 1") + return + except psycopg.OperationalError as exc: + # THE CLASS NAME ONLY: a driver's message quotes the DSN it + # could not reach, and this package never repeats one + # (`runtime/cli.py`'s redaction contract). + last = type(exc).__name__ + time.sleep(0.1) + raise BundleRefused( + f"the bundled PostgreSQL server did not accept a connection within " + f"{START_TIMEOUT_SECONDS:.0f}s ({last}); its log is " + f"{self.log_path}") + + def _dsn(self, role: str, database: str) -> str: + """`DatabaseBundle.dsn`, aimed at a database other than the served one.""" + return self.bundle.dsn(role).replace( + f"@/{BUNDLE_DATABASE}?", f"@/{database}?", 1) + + def _bootstrap(self) -> None: + """The database, the served role and its grants — idempotent. + + The grants are the compose stack's (`init-runtime-role.sh`), narrowed + the same way: CONNECT on the database, USAGE on `public`, and DML on + every table the OWNER creates from here on, by default privileges — + which on this install's own fresh database is the six coordination + tables and the ledger, and nothing else. The migration then narrows + the served role's rights on the ledger to SELECT. + """ + import psycopg + from psycopg import sql + + with psycopg.connect(self._dsn(BUNDLE_OWNER_ROLE, "postgres"), + autocommit=True) as conn: + if conn.execute("select 1 from pg_database where datname = %s", + (BUNDLE_DATABASE,)).fetchone() is None: + conn.execute(sql.SQL("create database {} owner {}").format( + sql.Identifier(BUNDLE_DATABASE), + sql.Identifier(BUNDLE_OWNER_ROLE))) + if conn.execute("select 1 from pg_roles where rolname = %s", + (BUNDLE_SERVED_ROLE,)).fetchone() is None: + conn.execute(sql.SQL("create role {} login").format( + sql.Identifier(BUNDLE_SERVED_ROLE))) + with psycopg.connect(self.bundle.migration_dsn, autocommit=True) as conn: + served, owner = (sql.Identifier(BUNDLE_SERVED_ROLE), + sql.Identifier(BUNDLE_OWNER_ROLE)) + conn.execute(sql.SQL("grant connect on database {} to {}").format( + sql.Identifier(BUNDLE_DATABASE), served)) + conn.execute(sql.SQL("grant usage on schema public to {}").format( + served)) + conn.execute(sql.SQL( + "alter default privileges for role {} in schema public grant " + "select, insert, update, delete on tables to {}").format( + owner, served)) + + def _migrate(self) -> None: + """Starting AND MIGRATING is all release 1 asks of the store (R1Q16 (ii)).""" + from opendox.runtime.db import Database + + database = Database(self.bundle.migration_dsn, + application_name="opendox-local-migrate") + with database: + self.applied = migrations.MigrationRunner( + database, migrations_dir=self.settings.migrations_dir, + runtime_role=self.settings.runtime_pg_role, + served_schema=self.settings.served_schema, + served_database=self.settings.served_database).apply() + + def _log_tail(self) -> str: + try: + lines = self.log_path.read_text(encoding="utf-8", + errors="replace").splitlines() + except OSError: + return "(no log)" + return " | ".join(lines[-5:]) + + # -- stop -------------------------------------------------------------- + + def stop(self) -> None: + """A FAST shutdown (SIGINT), then an immediate one, then a kill. + + FAST, and not PostgreSQL's default "smart" shutdown, because the + document server that is its only client has already closed: waiting + for sessions to end would wait for nothing, or for a leaked + connection. Safe to call twice, and on a server that never started. + """ + process, self.process = self.process, None + if process is None or process.poll() is not None: + return + for sig, wait in ((signal.SIGINT, STOP_TIMEOUT_SECONDS), + (signal.SIGQUIT, 5.0), (signal.SIGKILL, 5.0)): + try: + process.send_signal(sig) + except ProcessLookupError: + return + try: + process.wait(timeout=wait) + return + except subprocess.TimeoutExpired: + continue + + def __enter__(self) -> BundledServer: + return self.start() + + def __exit__(self, *exc: object) -> None: + self.stop() + + +__all__ = ["BUNDLE_PORT", "BundleRefused", "BundledServer", "report", + "running_pid", "server_binaries"] diff --git a/src/opendox/runtime/cli.py b/src/opendox/runtime/cli.py index af0ac390..e1c7a041 100644 --- a/src/opendox/runtime/cli.py +++ b/src/opendox/runtime/cli.py @@ -348,6 +348,10 @@ def _redacted_settings(settings: RuntimeSettings) -> dict[str, Any]: # needs to know, because it decides whether the broker lines below # mean anything at all. "OPENDOX_INSTALL_MODE": settings.install_mode, + # WHERE A LOCAL INSTALL'S BUNDLED SERVER LIVES (plan 034 T072): a path + # and never a credential. Reported for a hosted install too, which + # never reads it, so the report covers the whole declared list. + "OPENDOX_STATE_DIR": str(settings.state_dir), # THE BROKER URLS ARE REDACTED HERE TOO. `load_settings` refuses # userinfo in the issuer and in an explicit JWKS URL — but this report # prints a DERIVED value, and a settings object can also be built by @@ -680,6 +684,20 @@ def cmd_status(args: argparse.Namespace) -> int: return _emit({"verb": "status", "refusal": "configuration", "message": _safe_message(exc)}, ok=False) report["settings"] = _redacted_settings(settings) + # THE BUNDLED SERVER THIS INSTALL OWNS (plan 034 T072; #1144 13.1): where + # its data directory and socket are, and the pid of the server running on + # them, read from the server's own `postmaster.pid`. `null` for a hosted + # install, which brings no server. Reported, never started: `status` + # changes nothing, and the process that owns the server is the document + # server that started it (R1Q16 (i)). + if settings.install_mode == INSTALL_MODE_LOCAL: + from opendox.runtime import bundle as bundle_mod + from opendox.runtime.config import database_bundle + + report["database_bundle"] = bundle_mod.report( + database_bundle(settings.state_dir)) + else: + report["database_bundle"] = None try: report["canonical_sha256"] = migrations.verify_canonical_digest( diff --git a/src/opendox/runtime/config.py b/src/opendox/runtime/config.py index 6edd5c88..fa8bf8db 100644 --- a/src/opendox/runtime/config.py +++ b/src/opendox/runtime/config.py @@ -26,14 +26,16 @@ from __future__ import annotations +import importlib.metadata import ipaddress import os import re import shlex +import sys import urllib.parse from collections.abc import Mapping from dataclasses import dataclass -from pathlib import Path +from pathlib import Path, PurePath #: The environment prefix. One string, so a rename is one edit. PREFIX = "OPENDOX_" @@ -97,6 +99,18 @@ class Setting: "neither the install is hosted, so a hosted install with no issuer " "refuses rather than falling into local mode (13.4, 13.5)", ), + # WHERE A LOCAL INSTALL KEEPS ITS OWN STATE (plan 034 T072; #1144 13.1): + # the bundled PostgreSQL server's data directory and its Unix socket. No + # default string, because the default is COMPUTED, per user — see + # `state_dir`. A hosted install never reads it. + Setting( + PREFIX + "STATE_DIR", None, False, False, + "the directory a LOCAL install owns: the bundled PostgreSQL server's " + "data directory and its Unix socket live under it, and the server " + "listens on that socket and on no TCP port (13.1). Unset, it is " + "`$XDG_STATE_HOME/opendox`, else `~/.local/state/opendox`. A hosted " + "install never reads it", + ), Setting( PREFIX + "OIDC_ISSUER", None, True, False, "the Keycloak broker's issuer, pinned: a token from any other issuer " @@ -172,7 +186,11 @@ class Setting: ), Setting( PREFIX + "MIGRATIONS_DIR", "migrations", False, False, - "the ordered-SQL directory, repository-root-relative", + "the ordered-SQL directory, repository-root-relative. Unset, it is " + "`migrations` where the working directory holds one (a checkout, or " + "the image's /app), and otherwise the copy the installed package " + "ships (plan 034 T072), so an install run from anywhere else still " + "has the migrations it applies", ), Setting( PREFIX + "PROJECT_REPOSITORY_ROOT", "var/projects", False, False, @@ -214,6 +232,7 @@ class RuntimeSettings: database_url: str migration_database_url: str | None install_mode: str + state_dir: Path oidc_issuer: str oidc_audience: str oidc_jwks_url: str | None @@ -236,6 +255,7 @@ def __repr__(self) -> str: "migration_database_url=" f"{'' if self.migration_database_url else 'None'}, " f"install_mode={self.install_mode!r}, " + f"state_dir={str(self.state_dir)!r}, " # REDACTED TOO, and not because `load_settings` allows userinfo # here — it refuses it. A `RuntimeSettings` built by hand, in a # test or by a future caller, does not go through that door, and @@ -1552,14 +1572,190 @@ def _refuse_two_dsns_that_select_different_schemas( #: is REFUSED, by name (a holder reading on openxFactory#656, plan 034 T070, #: the same fail-closed reading as the disagreeing flag and setting): an issuer #: next to `local` says a broker was meant, and honouring `local` over it would -#: silently drop the authentication the operator configured. T072 adds the two -#: DSNs, which the local install supplies itself (13.1). +#: silently drop the authentication the operator configured. +#: +#: AND THE TWO DSNs (plan 034 T072; the same holder reading): a local install +#: SUPPLIES BOTH ITSELF, from the server it bundles (#1144 13.1, "so no +#: pre-existing service can stand in for it"), so an operator's DSN beside +#: `local` is either about to be silently overridden or is another server +#: trying to stand in for the bundled one. Neither is accepted. HOSTED_ONLY_SETTINGS: tuple[str, ...] = ( PREFIX + "OIDC_ISSUER", PREFIX + "OIDC_AUDIENCE", PREFIX + "OIDC_JWKS_URL", + PREFIX + "DATABASE_URL", + PREFIX + "MIGRATION_DATABASE_URL", ) +#: THE BUNDLED SERVER'S IDENTITY (plan 034 T072; #1144 13.1, as T007 batch H's +#: addendum reads). The data directory and the socket directory live under the +#: install's own `OPENDOX_STATE_DIR`, at these paths — SHORT ONES, because a +#: Unix socket's whole path is bounded by the kernel (`sun_path`) and the +#: socket file is `/.s.PGSQL.`. +BUNDLE_DATA_DIR = PurePath("postgres", "data") +BUNDLE_SOCKET_DIR = PurePath("postgres", "run") +#: The port NUMBER, which names the socket file and opens NO TCP port: the +#: server is started with `listen_addresses` empty (13.1: "on NO TCP port"). +BUNDLE_PORT = 5432 +#: The two identities 13.3 keeps apart, and the one database. The MIGRATION +#: identity owns the database and every table it creates; the SERVED identity +#: is the least-privileged role the API reads and writes as, granted what +#: `deploy/compose/init-runtime-role.sh` grants the compose stack's role of the +#: same name. Two DSNs, two users, never one pasted twice (13.3). +BUNDLE_OWNER_ROLE = "opendox" +BUNDLE_SERVED_ROLE = "opendox_runtime" +BUNDLE_DATABASE = "opendox" + +#: The longest socket path the kernel takes, in bytes: `sizeof(sun_path)` less +#: its terminating NUL — 108 on Linux, 104 on macOS and the BSDs. PostgreSQL +#: refuses a longer one at startup; this refuses it at configuration, naming +#: the setting that made it long. +UNIX_SOCKET_PATH_MAX = 107 if sys.platform.startswith("linux") else 103 + + +@dataclass(frozen=True) +class DatabaseBundle: + """Where a LOCAL install's bundled PostgreSQL server lives, and its DSNs. + + Pure path and string arithmetic over `OPENDOX_STATE_DIR`, so `load_settings` + can name both DSNs without starting anything, and `runtime status` in a + second process derives the SAME ones and finds the same server. + `opendox.runtime.bundle` is what starts and stops it. + """ + + state_dir: Path + + @property + def data_dir(self) -> Path: + return self.state_dir / BUNDLE_DATA_DIR + + @property + def socket_dir(self) -> Path: + return self.state_dir / BUNDLE_SOCKET_DIR + + @property + def socket_path(self) -> Path: + return self.socket_dir / f".s.PGSQL.{BUNDLE_PORT}" + + def dsn(self, role: str) -> str: + """A DSN for `role` over the bundle's Unix socket, and never TCP. + + `host` is the socket DIRECTORY (libpq's rule for a value that starts + with `/`), percent-encoded so a state directory holding a space or a + `&` is still one value; `port` is spelled so a `PGPORT` in the + environment cannot send libpq to a different socket file. No password: + the socket directory is 0700 and the server's own `pg_hba.conf` + trusts local connections only, so reaching the socket IS the + credential, and a host connection is rejected outright. + """ + host = urllib.parse.quote(str(self.socket_dir), safe="/") + return (f"postgresql://{role}@/{BUNDLE_DATABASE}" + f"?host={host}&port={BUNDLE_PORT}") + + @property + def served_dsn(self) -> str: + return self.dsn(BUNDLE_SERVED_ROLE) + + @property + def migration_dsn(self) -> str: + return self.dsn(BUNDLE_OWNER_ROLE) + + +def state_dir(env: Mapping[str, str] | None = None) -> Path: + """The install's own state directory: `OPENDOX_STATE_DIR`, or the per-user one. + + ABSOLUTE, or refused: the document server that starts the bundled server + and a `runtime status` run from another directory must derive the same + socket, and a relative value would give each its own. Unset, it is + `$XDG_STATE_HOME/opendox` where that is absolute (the XDG rule ignores a + relative one), and otherwise `~/.local/state/opendox`. + """ + env = os.environ if env is None else env + setting = PREFIX + "STATE_DIR" + raw = env.get(setting, "").strip() + if raw: + path = Path(raw).expanduser() + if not path.is_absolute(): + raise ConfigurationError( + f"{setting} is {raw!r}, which is not an absolute path. The " + "document server that starts the bundled PostgreSQL server " + "and a `runtime status` run from another directory must find " + "the same socket, so the state directory is named absolutely") + return path + xdg = env.get("XDG_STATE_HOME", "").strip() + base = Path(xdg) if xdg and Path(xdg).is_absolute() else ( + Path.home() / ".local" / "state") + return base / "opendox" + + +def database_bundle(state: Path) -> DatabaseBundle: + """The bundle under `state`, refusing a socket path the kernel cannot bind.""" + bundle = DatabaseBundle(state_dir=state) + length = len(os.fsencode(str(bundle.socket_path))) + if length > UNIX_SOCKET_PATH_MAX: + raise ConfigurationError( + f"{PREFIX}STATE_DIR is too long for the bundled server's Unix " + f"socket: {bundle.socket_path} is {length} bytes and this kernel " + f"takes at most {UNIX_SOCKET_PATH_MAX}. The socket must live under " + "the install's own state directory (13.1), so choose a shorter " + f"{PREFIX}STATE_DIR") + return bundle + + +#: WHERE AN INSTALLED WHEEL KEEPS ITS MIGRATIONS (plan 034 T072). The +#: repository's `migrations/` stays where it is — the image copies it to +#: `/app/migrations` and runs from `/app` — and `pyproject.toml` maps the same +#: files into the wheel's data directory under this path, so an install run +#: outside any checkout still has the migrations it applies. The canonical +#: digest gate (`migrations.verify_canonical_digest`) is what proves any copy +#: found this way is the pinned one. +PACKAGED_MIGRATIONS = PurePath("share", "opendox", "migrations") + + +def packaged_migrations_dir() -> Path | None: + """The migrations this INSTALLATION carries, or `None` where it carries none. + + Two places, in order. The installed distribution's own data files — a + wheel install, whose `RECORD` lists them wherever the install scheme put + them. Then the source tree this module was imported from — an editable + install, which installs no data files, or `src/` on the path — whose + `migrations/` sits beside `src/`. + """ + try: + files = importlib.metadata.distribution("opendox").files or () + except importlib.metadata.PackageNotFoundError: + files = () + for entry in files: + parts = PurePath(entry).parts + if (entry.name.endswith(".sql") + and tuple(parts[-4:-1]) == PACKAGED_MIGRATIONS.parts): + return Path(entry.locate()).resolve().parent + source = Path(__file__).resolve().parents[3] + if (source / "pyproject.toml").is_file() and (source / "migrations").is_dir(): + return source / "migrations" + return None + + +def migrations_dir(env: Mapping[str, str] | None = None) -> Path: + """`OPENDOX_MIGRATIONS_DIR`, or where this install's migrations are. + + Set, it is used as given, as it always was. Unset, it is `migrations` + wherever the working directory holds one — a checkout, or the image's + `/app` — which is today's default, unchanged; and OTHERWISE the copy the + installed package carries (`packaged_migrations_dir`), so `pip install + "opendox[local]"` run from a user's home directory migrates its bundled + server instead of refusing for a missing directory. Where neither exists + it is still `migrations`, and the canonical gate refuses it by name. + """ + env = os.environ if env is None else env + raw = env.get(PREFIX + "MIGRATIONS_DIR", "").strip() + if raw: + return Path(raw) + here = Path("migrations") + if here.is_dir(): + return here + return packaged_migrations_dir() or here + def install_mode(env: Mapping[str, str] | None = None, *, local_flag: bool = False) -> str: @@ -1686,6 +1882,19 @@ def require_the_hosted_issuer(env: Mapping[str, str] | None = None) -> None: f"{PREFIX}INSTALL_MODE=local") +def _hosted_state_dir(env: Mapping[str, str]) -> Path: + """A HOSTED install's `state_dir`: reported, never read, never refused. + + A hosted install has no bundled server, so a value it will never use is + not a reason for it to refuse to start (13.6: otherwise unchanged). It is + still reported, so `status` describes the whole declared setting list. + """ + try: + return state_dir(env) + except ConfigurationError: + return Path(env.get(PREFIX + "STATE_DIR", "").strip()) + + def load_settings(env: Mapping[str, str] | None = None, *, local_flag: bool = False) -> RuntimeSettings: """Resolve :class:`RuntimeSettings` from `env` (default `os.environ`). @@ -1728,8 +1937,20 @@ def load_settings(env: Mapping[str, str] | None = None, *, if local: refuse_what_a_local_install_cannot_be(env) algorithms = _algorithms(env) - served = _require(env, _by_name(PREFIX + "DATABASE_URL")) - migration = _optional(env, _by_name(PREFIX + "MIGRATION_DATABASE_URL")) + # A LOCAL INSTALL SUPPLIES BOTH DSNs ITSELF (plan 034 T072; #1144 13.1), + # from the server it bundles under its own state directory, and an + # operator's DSN beside it was refused above. The two it supplies are two + # users over one socket, so T071's three checks below pass them for the + # reason they exist: one dialect, one database, and never one credential + # in both settings. + state = state_dir(env) if local else _hosted_state_dir(env) + if local: + bundle = database_bundle(state) + served: str = bundle.served_dsn + migration: str | None = bundle.migration_dsn + else: + served = _require(env, _by_name(PREFIX + "DATABASE_URL")) + migration = _optional(env, _by_name(PREFIX + "MIGRATION_DATABASE_URL")) # THE DIALECT FIRST: a scheme this module cannot parse as PostgreSQL is not # yet a DSN worth comparing at all. A no-op on an ABSENT migration DSN — # see `_refuse_non_postgresql_dsn`. @@ -1756,6 +1977,7 @@ def load_settings(env: Mapping[str, str] | None = None, *, database_url=served, migration_database_url=migration, install_mode=mode, + state_dir=state, oidc_issuer="" if local else _broker_url( env, _by_name(PREFIX + "OIDC_ISSUER"), required=True, is_a_base_url=True) or "", @@ -1768,11 +1990,15 @@ def load_settings(env: Mapping[str, str] | None = None, *, oidc_leeway_seconds=_positive_int(env, _by_name(PREFIX + "OIDC_LEEWAY_SECONDS")), bind_host=bind_host, bind_port=_positive_int(env, _by_name(PREFIX + "BIND_PORT")), - runtime_pg_role=_role_name(env), + # THE BUNDLE'S OWN NAMES where the operator declares none: the served + # role the migration narrows and verifies, and the database it may + # touch. An operator's declaration still wins, and a wrong one is + # refused by the migration run's own guards, as on a hosted install. + runtime_pg_role=_role_name(env) or (BUNDLE_SERVED_ROLE if local else None), served_schema=_served_schema(env), - served_database=_served_database(env), + served_database=_served_database(env) or (BUNDLE_DATABASE if local else None), publish_openapi=_boolean(env, _by_name(PREFIX + "PUBLISH_OPENAPI")), - migrations_dir=Path(_optional(env, _by_name(PREFIX + "MIGRATIONS_DIR")) or "migrations"), + migrations_dir=migrations_dir(env), project_repository_root=Path( _optional(env, _by_name(PREFIX + "PROJECT_REPOSITORY_ROOT")) or "var/projects" ), @@ -1801,7 +2027,18 @@ def load_migration_settings(env: Mapping[str, str] | None = None) -> RuntimeSett not, because those are the served runtime and must have the real thing. """ env = os.environ if env is None else env - dsn = env.get(PREFIX + "MIGRATION_DATABASE_URL", "").strip() + mode = install_mode(env) + local = mode == INSTALL_MODE_LOCAL + if local: + # THE BUNDLE'S OWNER, over its socket (plan 034 T072): a local install + # supplies its migration DSN as it supplies the served one, and an + # operator's beside it is refused, exactly as `load_settings` refuses. + refuse_what_a_local_install_cannot_be(env) + state = state_dir(env) + dsn = database_bundle(state).migration_dsn + else: + state = _hosted_state_dir(env) + dsn = env.get(PREFIX + "MIGRATION_DATABASE_URL", "").strip() if not dsn: raise ConfigurationError( f"{PREFIX}MIGRATION_DATABASE_URL is required to apply migrations; " @@ -1822,7 +2059,8 @@ def load_migration_settings(env: Mapping[str, str] | None = None) -> RuntimeSett # a migration run is part of the same install and one reading of the # selector serves every verb. It changes nothing else a migration run # does; the broker fields below are sentinels in either shape. - install_mode=install_mode(env), + install_mode=mode, + state_dir=state, oidc_issuer=MIGRATION_SENTINEL_ISSUER, oidc_audience=MIGRATION_SENTINEL_AUDIENCE, oidc_jwks_url=None, @@ -1831,12 +2069,11 @@ def load_migration_settings(env: Mapping[str, str] | None = None) -> RuntimeSett oidc_leeway_seconds=1, bind_host="127.0.0.1", bind_port=1, - runtime_pg_role=_role_name(env), + runtime_pg_role=_role_name(env) or (BUNDLE_SERVED_ROLE if local else None), served_schema=_served_schema(env), - served_database=_served_database(env), + served_database=_served_database(env) or (BUNDLE_DATABASE if local else None), publish_openapi=False, - migrations_dir=Path( - _optional(env, _by_name(PREFIX + "MIGRATIONS_DIR")) or "migrations"), + migrations_dir=migrations_dir(env), project_repository_root=Path( _optional(env, _by_name(PREFIX + "PROJECT_REPOSITORY_ROOT")) or "var/projects"), diff --git a/tests/test_doxbench_entrypoint.py b/tests/test_doxbench_entrypoint.py index 2488fdf0..47c3eae2 100644 --- a/tests/test_doxbench_entrypoint.py +++ b/tests/test_doxbench_entrypoint.py @@ -173,6 +173,31 @@ def _capture(*args, **kwargs): # from the shell would be refused beside the local mode, by design. for name in runtime_config.SETTING_NAMES: monkeypatch.delenv(name, raising=False) + # AND THE LOCAL INSTALL'S DATABASE IS STOOD IN, with a tripwire of its own + # (plan 034 T072). A local `generate-and-open` starts its bundled + # PostgreSQL server before it serves, and these cases are about the model + # port the entrypoint declares, which reads nothing from the store (R1Q16 + # (ii)). `tests_runtime/test_bundled_postgres.py` starts the real one, on + # this same entry point, and owns every assertion about it. + bundles = [] + + class _StandInBundle: + applied: list = [] + + def __init__(self, settings): + assert settings.install_mode == runtime_config.INSTALL_MODE_LOCAL + bundles.append(self) + + def start(self): + return self + + def stop(self): + pass + + def report(self): + return {"data_dir": None, "socket_dir": "(stood in)", "pid": None} + + monkeypatch.setattr(cli_mod.bundle_mod, "BundledServer", _StandInBundle) args = cli_mod.build_parser().parse_args([ "generate-and-open", runtime_config.LOCAL_FLAG, @@ -186,6 +211,7 @@ def _capture(*args, **kwargs): ]) rc = cli_mod.cmd_generate_and_open(args, opener=lambda url: None) assert rc == 0, "the entrypoint did not complete" + assert len(bundles) == 1, "a LOCAL entrypoint run must own one database" assert built, "the entrypoint never reached build_server" yield _handler_class(built[-1]), spawned, session_root diff --git a/tests/test_install_mode_entrypoint.py b/tests/test_install_mode_entrypoint.py index 2a121ef4..86e3e58d 100644 --- a/tests/test_install_mode_entrypoint.py +++ b/tests/test_install_mode_entrypoint.py @@ -164,31 +164,33 @@ def _args(*extra: str): def test_the_local_shape_needs_no_broker_and_no_setting_at_all() -> None: - """13.4: `local` needs no broker. Resolved with an EMPTY environment.""" - assert cli_mod._resolve_install_shape( - _args(runtime_config.LOCAL_FLAG), env={}) == \ - runtime_config.INSTALL_MODE_LOCAL - assert cli_mod._resolve_install_shape( - _args(), env={MODE: "local"}) == runtime_config.INSTALL_MODE_LOCAL + """13.4: `local` needs no broker. Resolved with an EMPTY environment, and + (plan 034 T072) with both DSNs supplied by the install itself.""" + for args, env in ((_args(runtime_config.LOCAL_FLAG), {}), + (_args(), {MODE: "local"})): + settings = cli_mod._resolve_install_shape(args, env=env) + assert settings.install_mode == runtime_config.INSTALL_MODE_LOCAL + assert settings.oidc_issuer == "" + assert settings.database_url != settings.migration_database_url @pytest.mark.parametrize("host", sorted(serve_mod.LOOPBACK_HOSTS)) def test_the_local_shape_accepts_each_loopback_host(host: str) -> None: assert cli_mod._resolve_install_shape( - _args(runtime_config.LOCAL_FLAG, "--host", host), env={}) == \ - runtime_config.INSTALL_MODE_LOCAL + _args(runtime_config.LOCAL_FLAG, "--host", host), env={} + ).install_mode == runtime_config.INSTALL_MODE_LOCAL def test_a_complete_hosted_configuration_resolves_hosted_unchanged() -> None: """13.6: a hosted install with its broker configured serves as before.""" env = {**HOSTED_WITHOUT_ISSUER, PREFIX + "OIDC_ISSUER": "https://issuer.example.invalid/realms/x"} - assert cli_mod._resolve_install_shape(_args(), env=env) == \ + assert cli_mod._resolve_install_shape(_args(), env=env).install_mode == \ runtime_config.INSTALL_MODE_HOSTED # a hosted document server may still bind beyond loopback, as it always # could: the loopback rule is the LOCAL mode's assert cli_mod._resolve_install_shape( - _args("--host", "0.0.0.0"), env=env) == \ + _args("--host", "0.0.0.0"), env=env).install_mode == \ runtime_config.INSTALL_MODE_HOSTED diff --git a/tests_runtime/local_entrypoint_driver.py b/tests_runtime/local_entrypoint_driver.py new file mode 100644 index 00000000..43e6d63c --- /dev/null +++ b/tests_runtime/local_entrypoint_driver.py @@ -0,0 +1,77 @@ +"""Run `opendox generate-and-open` for real, with ONLY its generation stood in. + +`tests_runtime/test_bundled_postgres.py` launches this file as a child process +(`python tests_runtime/local_entrypoint_driver.py generate-and-open --local …`) +so that F13.1's local probe can be run the way F13.1 runs it: in the +BACKGROUND, reached over HTTP, asked about by a SECOND process +(`runtime status`), and then stopped with a signal (plan 034 T072). + +WHAT IS STOOD IN, AND WHY IT IS ONLY THIS. At this stack's base, the generate +verbs still reach openXdox for three names (`corpus_root_refusal`, +`generate_snapshot`, `snapshot.write_snapshot`) and `serve` for two +(`_checkout_real`, `registry_mod`'s binding constants): phase 2's T055 and T056 +(openDox-code#59 and its successor) give openDox its own, and a lone checkout +cannot ask for them before then. They are replaced here exactly as +`tests/test_doxbench_entrypoint.py` replaces them, and NOTHING ELSE is: +`cli.main`, the install shape, the bundled server, `build_server` and the serve +loop all run as a user's `opendox generate-and-open --local` runs them. Once +T055 and T056 have landed on this branch's base, this driver's stand-ins go and +the probe runs the real entry point on `tests/fixtures/plain-documents`. +""" + +from __future__ import annotations + +import json +import sys +import types +from pathlib import Path + + +class _StandInSource: + """`build_server`'s snapshot source, as T011's `standalone` fixture has it.""" + + refresh_binding = None + baked_repository = None + + class registry: + active = None + + def bootstrap(self): + pass + + +def _generate_snapshot(repo_root, repository, *, source_revision=None, + **_ignored): + return {"repository": repository, + "generation": {"source_revision": source_revision}, + "documents": [{"id": "stand-in.md"}]} + + +def _write_snapshot(snapshot, output, boundary): + Path(output).write_text(json.dumps(snapshot), encoding="utf-8") + return output + + +def main(argv: list[str]) -> int: + from opendox import cli as cli_mod + from opendox import serve as serve_mod + + cli_mod.corpus_root_refusal = lambda root, shape=None: None + cli_mod.generate_snapshot = _generate_snapshot + cli_mod.snapshot_mod = types.SimpleNamespace(write_snapshot=_write_snapshot) + serve_mod._checkout_real = lambda root: False + serve_mod.registry_mod = types.SimpleNamespace( + BINDING_REGENERATE="regenerate", BINDING_REFETCH="refetch") + real_build_server = serve_mod.build_server + + def _build_server(*args, **kwargs): + kwargs.setdefault("snapshot_source", _StandInSource()) + return real_build_server(*args, **kwargs) + + serve_mod.build_server = _build_server + return cli_mod.main(argv) + + +if __name__ == "__main__": + sys.stdout.reconfigure(line_buffering=True) + sys.exit(main(sys.argv[1:])) diff --git a/tests_runtime/test_bundled_postgres.py b/tests_runtime/test_bundled_postgres.py new file mode 100644 index 00000000..895bcfb9 --- /dev/null +++ b/tests_runtime/test_bundled_postgres.py @@ -0,0 +1,415 @@ +"""The LOCAL install's bundled PostgreSQL server (plan 034 T072; #1144 13.1, as +T007 batch H's addendum reads; RULED R1Q16 (i)-(iv), `5850003126`). + +T072's falsifier is F13.1's TCP-listener block, which reads the kernel's +socket table at run time, and its `runtime status` block. Both are run here +against a server the REAL entry point started: `generate-and-open --local`, +launched in the background as F13.1 launches it, reached over HTTP, asked +about by a second process, and stopped with a signal. Where F13.1 reads the +server's pid from `caps.json`, these cases read the same pid from +`runtime status`'s `database_bundle`. `/capabilities`' `install` block is +T073's, and nothing here pretends it exists. + +R1Q16, each part asserted: + (i) the server is a CHILD of the entry point's process (its `PPid`); + (ii) it is started AND migrated, the ledger holding every migration; + (iii) the `local` extra carries it, and the `test` extra joins the extra; + (iv) it stops with the entry point: on SIGTERM, which the serve loop reads + as Ctrl-C, and — the backstop — on SIGKILL, through the parent-death + signal. + +And the migrations gap the holder assigned to T072: a WHEEL install, run from +a directory that is not a checkout, migrates its bundled server from the copy +the wheel carries. + +NOT SKIPPED IN CI. These cases need the `local` extra's server, which the +`test` extra installs; under `CI` its absence is a FAILURE, as a missing +`postgres:16` service is for the DB-backed suites (`conftest._skip_or_fail`'s +rule, restated below), because `validate.yml` pins the skip count exactly. +""" + +from __future__ import annotations + +import json +import os +import re +import shutil +import signal +import subprocess +import sys +import sysconfig +import tempfile +import time +import tomllib +import urllib.request +from pathlib import Path + +import pytest + +from opendox.runtime import bundle as bundle_mod +from opendox.runtime import config +from opendox.runtime.config import PREFIX + +ROOT = Path(__file__).resolve().parents[1] +SRC = ROOT / "src" +DRIVER = Path(__file__).resolve().parent / "local_entrypoint_driver.py" +MODE = PREFIX + "INSTALL_MODE" +STATE = PREFIX + "STATE_DIR" + + +def _in_ci() -> bool: + return os.environ.get("CI", "").strip().lower() in {"1", "true", "yes", "on"} + + +@pytest.fixture(scope="module", autouse=True) +def _the_server_is_installed() -> None: + """The `local` extra's binaries, or this module's refusal to pass silently.""" + try: + bundle_mod.server_binaries() + except bundle_mod.BundleRefused as exc: + if _in_ci(): + pytest.fail(f"CI is set, so the bundled-server suite must RUN: {exc}", + pytrace=False) + pytest.skip(str(exc)) + if hasattr(os, "geteuid") and os.geteuid() == 0: # pragma: no cover + pytest.fail("the bundled server refuses root; run the suite as a user") + + +@pytest.fixture() +def state_dir(): + """A state directory nothing else has touched, with a SHORT path. + + Short because the socket's whole path is bounded by the kernel, and + pytest's own `tmp_path` grows with the test's name. Removed afterwards, + once any server on it has been checked stopped. + """ + base = "/tmp" if os.path.isdir("/tmp") else None + path = Path(tempfile.mkdtemp(prefix="odx-", dir=base)) + yield path + pid = bundle_mod.running_pid(config.DatabaseBundle(path)) + if pid is not None: # pragma: no cover + os.kill(pid, signal.SIGKILL) + shutil.rmtree(path, ignore_errors=True) + + +def _clean_env(**extra: str) -> dict[str, str]: + env = {name: value for name, value in os.environ.items() + if name not in config.SETTING_NAMES and not name.startswith("PG")} + env["PYTHONPATH"] = os.pathsep.join( + [str(SRC), env.get("PYTHONPATH", "")]).rstrip(os.pathsep) + env.update(extra) + return env + + +def _parent_of(pid: int) -> int: + for line in Path(f"/proc/{pid}/status").read_text().splitlines(): + if line.startswith("PPid:"): + return int(line.split()[1]) + raise AssertionError(f"no PPid for {pid}") # pragma: no cover + + +def _tcp_listeners(pid: int) -> list[tuple[str, str]]: + """F13.1's TCP-listener block, verbatim in substance: every socket the + process holds, looked up in the KERNEL's TCP tables, never self-report.""" + inodes = set() + for fd in os.listdir(f"/proc/{pid}/fd"): + try: + m = re.match(r"socket:\[(\d+)\]", os.readlink(f"/proc/{pid}/fd/{fd}")) + except OSError: + continue + if m: + inodes.add(m.group(1)) + return [(tbl, row.split()[1]) for tbl in ("/proc/net/tcp", "/proc/net/tcp6") + for row in open(tbl).read().splitlines()[1:] + if row.split()[3] == "0A" and row.split()[9] in inodes] + + +def _wait_gone(pid: int, seconds: float = 30.0) -> bool: + deadline = time.monotonic() + seconds + while time.monotonic() < deadline: + try: + os.kill(pid, 0) + except ProcessLookupError: + return True + # a zombie is gone as a server: reaped by init once its parent is + try: + if "State:\tZ" in Path(f"/proc/{pid}/status").read_text(): + return True + except OSError: + return True + time.sleep(0.1) + return False + + +def _launch(corpus: Path, state: Path, run_dir: Path) -> tuple[subprocess.Popen, str]: + """`generate-and-open --local` in the BACKGROUND, and the URL it serves.""" + child = subprocess.Popen( + [sys.executable, str(DRIVER), "generate-and-open", config.LOCAL_FLAG, + "--repo-root", str(corpus), "--repository", "fixture", + "--run-dir", str(run_dir), "--no-open", "--no-validate", + "--port", "0"], + env=_clean_env(**{STATE: str(state)}), cwd=ROOT, + stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) + deadline = time.monotonic() + 90 + url = None + while time.monotonic() < deadline and url is None: + line = child.stdout.readline() + if not line: + break + if line.startswith("http://"): + url = line.strip() + if url is None: + child.kill() + out, err = child.communicate(timeout=30) + raise AssertionError(f"the entry point never served: {err[-2000:]}") + return child, url + + +def _status(state: Path) -> tuple[int, dict]: + """`OPENDOX_INSTALL_MODE=local opendox-runtime runtime status`, a SECOND process.""" + done = subprocess.run( + [sys.executable, "-m", "opendox.runtime.cli", "runtime", "status", + "--probe-timeout", "10"], + env=_clean_env(**{MODE: "local", STATE: str(state)}), cwd=ROOT, + capture_output=True, text=True, timeout=60) + return done.returncode, json.loads(done.stdout) + + +@pytest.fixture() +def corpus(tmp_path: Path) -> Path: + root = tmp_path / "plain-documents" + root.mkdir() + (root / "note.md").write_text("# A note\n\nPlain text.\n", encoding="utf-8") + return root + + +# -- (iii): the packaging ---------------------------------------------------- + + +def test_the_local_extra_carries_the_runtime_and_the_server_and_test_joins_it( +) -> None: + project = tomllib.loads((ROOT / "pyproject.toml").read_text()) + extras = project["project"]["optional-dependencies"] + assert "opendox[runtime]" in extras["local"] + assert any(req.startswith("pgserver") for req in extras["local"]) + assert "opendox[local]" in extras["test"], ( + "F9.1 installs `.[test]` alone; without the local extra there, this " + "suite could not start the server it tests") + lock = (ROOT / "constraints-cpython312-linux.txt").read_text() + assert re.search(r"(?m)^pgserver==", lock), "the lock does not pin pgserver" + files = project["tool"]["setuptools"]["data-files"] + assert files == {"share/opendox/migrations": ["migrations/*.sql"]} + + +# -- the layout, before anything starts ---------------------------------------- + + +def test_the_two_dsns_are_two_users_over_the_one_socket(state_dir: Path) -> None: + settings = config.load_settings({MODE: "local", STATE: str(state_dir)}) + bundle = config.database_bundle(state_dir) + assert settings.database_url == bundle.served_dsn + assert settings.migration_database_url == bundle.migration_dsn + assert settings.database_url != settings.migration_database_url # 13.3 + for dsn in (settings.database_url, settings.migration_database_url): + assert dsn.startswith("postgresql://") # 13.2 + assert f"host={bundle.socket_dir}" in dsn and "port=5432" in dsn + assert config.user_named_by(settings.database_url) == config.BUNDLE_SERVED_ROLE + assert config.user_named_by(settings.migration_database_url) == \ + config.BUNDLE_OWNER_ROLE + assert bundle.data_dir.parent == bundle.socket_dir.parent + assert bundle.data_dir.is_relative_to(state_dir) + assert bundle.socket_dir.is_relative_to(state_dir) + + +def test_a_state_dir_too_long_for_a_unix_socket_is_refused_naming_it() -> None: + long = "/tmp/" + "x" * 120 + with pytest.raises(config.ConfigurationError) as caught: + config.load_settings({MODE: "local", STATE: long}) + assert STATE in str(caught.value) and "socket" in str(caught.value) + + +def test_a_relative_state_dir_is_refused_naming_it() -> None: + with pytest.raises(config.ConfigurationError) as caught: + config.load_settings({MODE: "local", STATE: "var/state"}) + assert STATE in str(caught.value) + + +def test_the_default_state_dir_is_the_users_own(monkeypatch) -> None: + home = config.state_dir({"HOME": "/ignored"}) + assert home.name == "opendox" and home.is_absolute() + assert config.state_dir({"XDG_STATE_HOME": "/srv/state"}) == \ + Path("/srv/state/opendox") + # the XDG rule: a relative value is ignored, never joined onto the cwd + assert config.state_dir({"XDG_STATE_HOME": "relative"}) == \ + Path.home() / ".local" / "state" / "opendox" + + +# -- 13.1 and R1Q16 (i), (ii), (iv): F13.1's two blocks, on the real entry point + + +def test_the_entry_point_owns_a_migrated_server_with_no_tcp_listener( + corpus: Path, state_dir: Path, tmp_path: Path) -> None: + """F13.1's `runtime status` block and its TCP-listener block, against the + server `generate-and-open --local` started in the background; then + `kill "$SERVER"; wait`, and the server is gone with it.""" + server, url = _launch(corpus, state_dir, tmp_path / "run") + try: + with urllib.request.urlopen(url, timeout=10) as answer: # ready + assert answer.status == 200 + # -- F13.1's `runtime status` block -------------------------------- + code, status = _status(state_dir) + assert status.get("database") == "reachable", \ + f"no bundled database answered: {status}" + assert status.get("applied_migrations") and \ + not status.get("pending_migrations"), f"not migrated: {status}" + state = os.path.realpath(state_dir) + bundle = status.get("database_bundle") or {} + for key in ("data_dir", "socket_dir"): + got = os.path.realpath(bundle.get(key, "")) + assert got.startswith(state + os.sep), \ + f"{key} {got!r} is not under the install's state dir {state!r}" + # a local install has no broker, and that is not a fault (T070) + assert status["broker_keys"] == "not configured (local mode)" + assert code == 0 and status["ok"] is True, status + # -- F13.1's TCP-listener block, the pid from `database_bundle` ----- + pid = bundle.get("pid") + assert isinstance(pid, int), f"the bundle reports no server pid: {pid!r}" + assert not _tcp_listeners(pid), \ + f"the bundled server listens on TCP: {_tcp_listeners(pid)}" + # -- R1Q16 (i): the server is the document server's own child ------ + assert _parent_of(pid) == server.pid, ( + "the bundled server is not a child of the process serving the " + "document surface") + mode = os.stat(bundle["socket_dir"]).st_mode & 0o777 + assert mode == 0o700, f"the socket directory is {mode:o}" + finally: + server.send_signal(signal.SIGTERM) # `kill "$SERVER"` + server.communicate(timeout=60) # `wait "$SERVER"` + # -- R1Q16 (iv): it stopped with the entry point, and cleanly ---------- + assert server.returncode == 0, server.returncode + assert _wait_gone(pid), "the bundled server outlived its entry point" + assert config.DatabaseBundle(state_dir).data_dir.joinpath("PG_VERSION").is_file(), \ + "the data directory must survive a stop: it is the install's database" + + +@pytest.mark.skipif(not sys.platform.startswith("linux"), + reason="PR_SET_PDEATHSIG is Linux's") +def test_the_server_stops_even_when_the_entry_point_is_killed_outright( + corpus: Path, state_dir: Path, tmp_path: Path) -> None: + """(iv)'s backstop: SIGKILL gives the entry point no chance to stop + anything, and the parent-death signal stops the server all the same.""" + server, _url = _launch(corpus, state_dir, tmp_path / "run") + pid = bundle_mod.running_pid(config.DatabaseBundle(state_dir)) + assert isinstance(pid, int) + server.kill() + server.communicate(timeout=30) + assert _wait_gone(pid), "the bundled server outlived a SIGKILLed entry point" + + +def test_a_second_entry_point_on_the_same_state_dir_is_refused( + state_dir: Path) -> None: + settings = config.load_settings({MODE: "local", STATE: str(state_dir)}) + first = bundle_mod.BundledServer(settings).start() + try: + with pytest.raises(bundle_mod.BundleRefused) as caught: + bundle_mod.BundledServer(settings).start() + assert str(first.report()["pid"]) in str(caught.value) + # and a restart of the one that owns it re-migrates nothing + assert first.applied == ["0001", "0002"] + finally: + first.stop() + again = bundle_mod.BundledServer(settings).start() + try: + assert again.applied == [] + finally: + again.stop() + + +def test_migrate_under_the_local_mode_uses_the_bundle_and_refuses_a_dsn( + state_dir: Path) -> None: + """`runtime migrate` is part of the same install: it reads the bundle's + owner DSN, and an operator's migration DSN beside `local` is refused.""" + settings = config.load_settings({MODE: "local", STATE: str(state_dir)}) + with bundle_mod.BundledServer(settings): + done = subprocess.run( + [sys.executable, "-m", "opendox.runtime.cli", "runtime", "migrate"], + env=_clean_env(**{MODE: "local", STATE: str(state_dir)}), cwd=ROOT, + capture_output=True, text=True, timeout=60) + evidence = json.loads(done.stdout) + assert done.returncode == 0 and evidence["applied"] == [], evidence + done = subprocess.run( + [sys.executable, "-m", "opendox.runtime.cli", "runtime", "migrate"], + env=_clean_env(**{MODE: "local", STATE: str(state_dir), + PREFIX + "MIGRATION_DATABASE_URL": + "postgresql://m:hunter2@db.invalid/x"}), + cwd=ROOT, capture_output=True, text=True, timeout=60) + evidence = json.loads(done.stdout) + assert evidence["refusal"] == "configuration", evidence + assert PREFIX + "MIGRATION_DATABASE_URL" in evidence["message"] + assert "hunter2" not in done.stdout + + +# -- the migrations gap: a wheel, outside any checkout -------------------------- + + +def test_a_wheel_install_migrates_its_bundled_server_outside_a_checkout( + state_dir: Path, tmp_path: Path) -> None: + """Build this package's wheel, install it OUTSIDE the checkout, run it from + a directory with no `migrations/`, and migrate the bundled server from the + copy the wheel carries (the holder's assignment to T072).""" + source = tmp_path / "source" + source.mkdir() + for name in ("pyproject.toml", "src", "migrations"): + item = ROOT / name + (shutil.copytree if item.is_dir() else shutil.copy2)( + item, source / name, **({"ignore": shutil.ignore_patterns( + "__pycache__", "*.egg-info")} if item.is_dir() else {})) + wheels = tmp_path / "wheels" + built = subprocess.run( + [sys.executable, "-m", "pip", "wheel", "--no-deps", "--no-index", + "--no-build-isolation", "-q", "-w", str(wheels), str(source)], + capture_output=True, text=True, timeout=300) + assert built.returncode == 0, built.stderr[-3000:] + (wheel,) = wheels.glob("opendox-*.whl") + prefix = tmp_path / "prefix" + # `--ignore-installed` IS LOAD-BEARING: without it pip treats the suite's + # own (editable) `opendox` as the installed copy of the same project and + # UNINSTALLS it before writing the new one under `--prefix` — measured, it + # emptied the environment this very suite runs in. + installed = subprocess.run( + [sys.executable, "-m", "pip", "install", "--no-deps", "--no-index", + "--ignore-installed", "-q", "--prefix", str(prefix), str(wheel)], + capture_output=True, text=True, timeout=300) + assert installed.returncode == 0, installed.stderr[-3000:] + assert "uninstall" not in (installed.stdout + installed.stderr).lower() + # ...and the environment this suite runs in still has its own install + still = subprocess.run( + [sys.executable, "-c", "import importlib.metadata as m; " + "print(m.distribution('opendox').version)"], + env=_clean_env(PYTHONPATH=""), capture_output=True, text=True, timeout=60) + assert still.returncode == 0, "installing the wheel removed the suite's opendox" + site = Path(sysconfig.get_path("purelib", vars={"base": str(prefix), + "platbase": str(prefix)})) + elsewhere = tmp_path / "elsewhere" + elsewhere.mkdir() + program = f""" +import json, sys +from pathlib import Path +import opendox +from opendox.runtime import bundle, config +prefix = Path({str(prefix)!r}).resolve() +assert Path(opendox.__file__).resolve().is_relative_to(prefix), opendox.__file__ +assert not Path("migrations").exists() +settings = config.load_settings() +found = Path(settings.migrations_dir).resolve() +assert found.is_relative_to(prefix / "share" / "opendox"), found +with bundle.BundledServer(settings) as server: + print(json.dumps({{"applied": server.applied, "dir": str(found)}})) +""" + env = _clean_env(**{MODE: "local", STATE: str(state_dir)}) + env["PYTHONPATH"] = str(site) + done = subprocess.run([sys.executable, "-c", program], cwd=elsewhere, + env=env, capture_output=True, text=True, timeout=120) + assert done.returncode == 0, done.stderr[-3000:] + result = json.loads(done.stdout.strip().splitlines()[-1]) + assert result["applied"] == ["0001", "0002"], result diff --git a/tests_runtime/test_install_mode.py b/tests_runtime/test_install_mode.py index de9ca524..ad1225bf 100644 --- a/tests_runtime/test_install_mode.py +++ b/tests_runtime/test_install_mode.py @@ -62,6 +62,11 @@ HOSTED = {**DSNS, PREFIX + "OIDC_ISSUER": "https://issuer.example.invalid/realms/fixture", PREFIX + "OIDC_AUDIENCE": "fixture"} +#: A LOCAL install's whole environment: the selector and a state directory. +#: No DSN — the local install supplies both from the server it bundles +#: (plan 034 T072), and one given beside it is refused. Nothing is created +#: there: `load_settings` derives paths, and starts nothing. +LOCAL = {MODE: "local", PREFIX + "STATE_DIR": "/nonexistent/opendox-state"} def _refusal(env: dict, **kwargs) -> str: @@ -190,7 +195,7 @@ def test_the_hosted_mode_is_unchanged() -> None: @pytest.mark.parametrize("selection", ["setting", "flag"]) def test_a_local_install_needs_no_broker(selection: str) -> None: - env = dict(DSNS) + env = {PREFIX + "STATE_DIR": LOCAL[PREFIX + "STATE_DIR"]} kwargs = {} if selection == "setting": env[MODE] = "local" @@ -213,13 +218,19 @@ def test_a_broker_setting_beside_the_local_mode_is_refused_by_name( """A holder reading (#656, T070): a broker setting says hosted was meant.""" assert set(HOSTED_ONLY_SETTINGS) == {PREFIX + "OIDC_ISSUER", PREFIX + "OIDC_AUDIENCE", - PREFIX + "OIDC_JWKS_URL"} - secret = "https://svc:hunter2@broker.example.invalid/realms/x" - message = _refusal({**DSNS, MODE: "local", name: secret}) + PREFIX + "OIDC_JWKS_URL", + # and T072's two: the local install + # supplies both DSNs itself (13.1) + PREFIX + "DATABASE_URL", + PREFIX + "MIGRATION_DATABASE_URL"} + secret = ("https://svc:hunter2@broker.example.invalid/realms/x" + if "OIDC" in name else "postgresql://u:hunter2@db.invalid/x") + message = _refusal({**LOCAL, name: secret}) assert name in message, message assert "hunter2" not in message, "the value must not be repeated" # and the flag spelling of the same selection refuses it the same way - assert name in _refusal({**DSNS, name: secret}, local_flag=True) + assert name in _refusal({PREFIX + "STATE_DIR": LOCAL[PREFIX + "STATE_DIR"], + name: secret}, local_flag=True) def test_every_broker_setting_given_is_named_at_once() -> None: @@ -231,8 +242,7 @@ def test_every_broker_setting_given_is_named_at_once() -> None: @pytest.mark.parametrize("host", sorted(LOCAL_BIND_HOSTS)) def test_a_local_install_binds_each_loopback_spelling(host: str) -> None: - settings = load_settings({**DSNS, MODE: "local", - PREFIX + "BIND_HOST": host}) + settings = load_settings({**LOCAL, PREFIX + "BIND_HOST": host}) assert settings.bind_host == host @@ -243,7 +253,7 @@ def test_a_local_install_refuses_a_non_loopback_bind_naming_the_rule( """13.4: loopback ONLY, and no opt-in. `127.0.0.2` is refused too: the document server does not treat it as loopback (`serve.LOOPBACK_HOSTS`), and the mode makes the SAME judgement at its own boundary.""" - message = _refusal({**DSNS, MODE: "local", PREFIX + "BIND_HOST": host}) + message = _refusal({**LOCAL, PREFIX + "BIND_HOST": host}) assert PREFIX + "BIND_HOST" in message, message assert "loopback" in message.lower(), message assert "no opt-in" in message.lower(), message @@ -283,9 +293,8 @@ def run(self) -> None: stub.Config, stub.Server = _Config, _Server scrubbed.setitem(sys.modules, "uvicorn", stub) scrubbed.setattr(app_module, "create_app", lambda **kwargs: object()) - for name, value in DSNS.items(): + for name, value in LOCAL.items(): scrubbed.setenv(name, value) - scrubbed.setenv(MODE, "local") code, evidence = _run(["runtime", "serve"]) assert served == [], "the API was started for a LOCAL install" assert code == 1 @@ -306,9 +315,8 @@ def _no_broker(_settings): "install, which has no broker") monkeypatch.setattr(oidc, "build_verifier", _no_broker) - for name, value in DSNS.items(): + for name, value in LOCAL.items(): scrubbed.setenv(name, value) - scrubbed.setenv(MODE, "local") code, evidence = _run(["runtime", "status", "--probe-timeout", "0.2"]) assert evidence.get("refusal") is None, evidence assert evidence["broker_keys"] == "not configured (local mode)" @@ -316,8 +324,11 @@ def _no_broker(_settings): assert evidence["settings"][MODE] == INSTALL_MODE_LOCAL assert evidence["settings"][PREFIX + "OIDC_ISSUER"] == "" assert evidence["settings"][PREFIX + "OIDC_JWKS_URL"] == "" - # the database half (port 1, unreachable) is the ONLY reason `ok` is false + # the database half is the ONLY reason `ok` is false: no bundled server + # is running on this (nonexistent) state directory, and `status` reports + # that rather than starting one assert evidence["database"].startswith("unreachable"), evidence + assert evidence["database_bundle"]["pid"] is None, evidence assert code == 1 diff --git a/tests_runtime/test_runtime_surface.py b/tests_runtime/test_runtime_surface.py index ff6a73a3..5de1373c 100644 --- a/tests_runtime/test_runtime_surface.py +++ b/tests_runtime/test_runtime_surface.py @@ -50,6 +50,11 @@ # one would make openDox the only destination the corpus could not test. "opendox.runtime.local_git_adapter", "opendox.runtime.repository_act", + # THE LOCAL INSTALL'S BUNDLED SERVER (plan 034 T072). `opendox.cli` + # imports it at module level, and `opendox.cli` is what `opendox --help` + # runs on an install with no extra at all, so it connects through + # `psycopg` only inside the functions that start the server. + "opendox.runtime.bundle", ) #: The modules that legitimately need the `runtime` extra, and the only ones. From c3a70a2d1145a7fd7bcc56cbf851a4c7513bdd47 Mon Sep 17 00:00:00 2001 From: Brett Heap <1513478+brettheap@users.noreply.github.com> Date: Wed, 30 Sep 2026 00:47:47 +0000 Subject: [PATCH 07/39] T072: extend the dependency lock for the `local` extra and the test extra's setuptools The lock is extended under its own pins (`-c` this file), in a clean cpython 3.12.3 venv on linux x86_64, as its header asks. Five pins are new: - pgserver 0.1.4, with its own psutil, platformdirs and fasteners; - setuptools, for the wheel-install test's offline build. No earlier pin moved. Arc: neutral-product-standalone-operability Lane: openxfactory-4 (openXfactory-4-openDox_extraction) Co-Authored-By: Claude Opus 5.5 (1M context) --- constraints-cpython312-linux.txt | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/constraints-cpython312-linux.txt b/constraints-cpython312-linux.txt index 84ed0972..4772dbe1 100644 --- a/constraints-cpython312-linux.txt +++ b/constraints-cpython312-linux.txt @@ -30,6 +30,10 @@ # two installs, no second list of names to drift. # # Resolved 2026-09-18 on cpython 3.12.3 / linux x86_64. +# Extended 2026-09-30 on cpython 3.12.3 / linux x86_64, under the pins above +# (`-c` this file), for plan 034 T072's `local` extra and the `test` extra's +# `setuptools`: pgserver, its psutil, platformdirs and fasteners, and +# setuptools are new; no earlier pin moved. PyJWT==2.14.0 PyYAML==6.0.3 Pygments==2.21.0 @@ -41,6 +45,7 @@ cffi==2.1.1 click==8.5.0 cryptography==50.0.1 fastapi==0.141.1 +fasteners==0.20 h11==0.16.0 httpcore==1.0.9 httptools==0.8.0 @@ -48,7 +53,10 @@ httpx==0.28.1 idna==3.20 iniconfig==2.3.0 packaging==26.3 +pgserver==0.1.4 +platformdirs==4.12.2 pluggy==1.6.0 +psutil==7.2.2 psycopg-binary==3.3.6 psycopg-pool==3.3.2 psycopg==3.3.6 @@ -57,6 +65,7 @@ pydantic==2.13.5 pydantic_core==2.46.5 pytest==8.4.2 python-dotenv==1.2.3 +setuptools==84.0.0 starlette==1.6.0 typing-inspection==0.4.4 typing_extensions==4.16.0 From 32683e8f2ec5aa9443e572739ced835d4628161a Mon Sep 17 00:00:00 2001 From: Brett Heap <1513478+brettheap@users.noreply.github.com> Date: Wed, 30 Sep 2026 00:48:46 +0000 Subject: [PATCH 08/39] Fix round: the install-mode fixture's repository ignores the user's git config (Copilot review) `tests/test_install_mode_entrypoint.py`'s `corpus` fixture ran `git commit` under the caller's global and system git configuration. A global `commit.gpgsign=true` therefore failed the setup before any install-mode probe ran. Measured with a hostile global config (`commit.gpgsign = true`, `gpg.program = /bin/false`): 7 errors at b50e3b1, 14 passed here. The fixture now sets GIT_CONFIG_GLOBAL=/dev/null and GIT_CONFIG_NOSYSTEM=1, as tests/test_checkout_head.py does. Arc: neutral-product-standalone-operability Lane: openxfactory-4 (openXfactory-4-openDox_extraction) Co-Authored-By: Claude Opus 5.5 (1M context) --- tests/test_install_mode_entrypoint.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/tests/test_install_mode_entrypoint.py b/tests/test_install_mode_entrypoint.py index 2a121ef4..9c4c6203 100644 --- a/tests/test_install_mode_entrypoint.py +++ b/tests/test_install_mode_entrypoint.py @@ -49,11 +49,19 @@ @pytest.fixture() def corpus(tmp_path: Path) -> Path: - """A fresh repository, as F13.1's preamble makes one.""" + """A fresh repository, as F13.1's preamble makes one. + + Made APART FROM the user's own git configuration (`GIT_CONFIG_GLOBAL`, + `GIT_CONFIG_NOSYSTEM`), as `tests/test_checkout_head.py` makes its + repositories, so a global signing rule or hook cannot fail the setup + before the install-mode probe it exists for ever runs (Copilot review of + this PR). + """ root = tmp_path / "plain-documents" root.mkdir() (root / "note.md").write_text("# A note\n\nPlain text.\n", encoding="utf-8") env = {**os.environ, + "GIT_CONFIG_GLOBAL": os.devnull, "GIT_CONFIG_NOSYSTEM": "1", "GIT_AUTHOR_NAME": "fixture", "GIT_AUTHOR_EMAIL": "fixture@example.invalid", "GIT_COMMITTER_NAME": "fixture", "GIT_COMMITTER_EMAIL": "fixture@example.invalid"} From 525f61c31db5df8b5264bb6a4f485ea0325d76b4 Mon Sep 17 00:00:00 2001 From: Brett Heap <1513478+brettheap@users.noreply.github.com> Date: Wed, 30 Sep 2026 00:58:12 +0000 Subject: [PATCH 09/39] Fix round: runtime migrate and reset refuse what a local install cannot be (Copilot review) load_migration_settings recorded OPENDOX_INSTALL_MODE=local but never asked refuse_what_a_local_install_cannot_be. So `runtime migrate` and a confirmed `runtime reset` accepted OPENDOX_OIDC_ISSUER, OPENDOX_OIDC_AUDIENCE, OPENDOX_OIDC_JWKS_URL or a non-loopback OPENDOX_BIND_HOST beside `local`, which load_settings and generate-and-open both refuse. They now refuse them at configuration, before any database is reached. Seven new cases: - the three broker settings x {migrate, reset}; - the bind. All seven fail at 32683e8 and pass here. Full suite: 2538 selected, 2527 passed, 11 skipped, 0 failed. Arc: neutral-product-standalone-operability Lane: openxfactory-4 (openXfactory-4-openDox_extraction) Co-Authored-By: Claude Opus 5.5 (1M context) --- src/opendox/runtime/config.py | 19 +++++++++++++----- tests_runtime/test_install_mode.py | 31 ++++++++++++++++++++++++++++++ 2 files changed, 45 insertions(+), 5 deletions(-) diff --git a/src/opendox/runtime/config.py b/src/opendox/runtime/config.py index 6edd5c88..326144e4 100644 --- a/src/opendox/runtime/config.py +++ b/src/opendox/runtime/config.py @@ -1801,6 +1801,16 @@ def load_migration_settings(env: Mapping[str, str] | None = None) -> RuntimeSett not, because those are the served runtime and must have the real thing. """ env = os.environ if env is None else env + # THE ONE READING OF THE SELECTOR, AND OF WHAT A LOCAL INSTALL CANNOT BE + # (plan 034 T070; Copilot review of openDox-code#67). A migration run is + # part of the same install as the served one, so `runtime migrate` and + # `runtime reset` refuse what `load_settings` and `generate-and-open` + # refuse beside `local` — a broker setting, or a non-loopback + # `OPENDOX_BIND_HOST` — rather than accepting it in the one loader that + # never reads it. + mode = install_mode(env) + if mode == INSTALL_MODE_LOCAL: + refuse_what_a_local_install_cannot_be(env) dsn = env.get(PREFIX + "MIGRATION_DATABASE_URL", "").strip() if not dsn: raise ConfigurationError( @@ -1818,11 +1828,10 @@ def load_migration_settings(env: Mapping[str, str] | None = None) -> RuntimeSett return RuntimeSettings( database_url=dsn, migration_database_url=dsn, - # READ, SO AN UNRECOGNISED VALUE IS REFUSED HERE TOO (plan 034 T070): - # a migration run is part of the same install and one reading of the - # selector serves every verb. It changes nothing else a migration run - # does; the broker fields below are sentinels in either shape. - install_mode=install_mode(env), + # READ ABOVE, SO AN UNRECOGNISED VALUE IS REFUSED HERE TOO (plan 034 + # T070). It changes nothing else a migration run does; the broker + # fields below are sentinels in either shape. + install_mode=mode, oidc_issuer=MIGRATION_SENTINEL_ISSUER, oidc_audience=MIGRATION_SENTINEL_AUDIENCE, oidc_jwks_url=None, diff --git a/tests_runtime/test_install_mode.py b/tests_runtime/test_install_mode.py index de9ca524..28932ab3 100644 --- a/tests_runtime/test_install_mode.py +++ b/tests_runtime/test_install_mode.py @@ -321,6 +321,37 @@ def _no_broker(_settings): assert code == 1 +@pytest.mark.parametrize("verb", [["runtime", "migrate"], + ["runtime", "reset", "--confirm", + cli.RESET_CONFIRMATION]]) +@pytest.mark.parametrize("name", [PREFIX + "OIDC_ISSUER", + PREFIX + "OIDC_AUDIENCE", + PREFIX + "OIDC_JWKS_URL"]) +def test_migrate_and_reset_refuse_a_broker_setting_beside_the_local_mode( + scrubbed, verb: list[str], name: str) -> None: + """The migration loader asks what every other loader asks of `local` + (Copilot review of openDox-code#67): refused at CONFIGURATION, before any + database is reached — `reset` included, confirmation and all.""" + scrubbed.setenv(PREFIX + "MIGRATION_DATABASE_URL", DSNS[PREFIX + "MIGRATION_DATABASE_URL"]) + scrubbed.setenv(MODE, "local") + scrubbed.setenv(name, "https://issuer.example.invalid/realms/x" + if name != PREFIX + "OIDC_AUDIENCE" else "fixture") + code, evidence = _run(verb) + assert code == 1 + assert evidence["refusal"] == "configuration", evidence + assert name in evidence["message"], evidence + + +def test_migrate_refuses_a_non_loopback_bind_beside_the_local_mode( + scrubbed) -> None: + scrubbed.setenv(PREFIX + "MIGRATION_DATABASE_URL", DSNS[PREFIX + "MIGRATION_DATABASE_URL"]) + scrubbed.setenv(MODE, "local") + scrubbed.setenv(PREFIX + "BIND_HOST", "0.0.0.0") + code, evidence = _run(["runtime", "migrate"]) + assert code == 1 and evidence["refusal"] == "configuration", evidence + assert "loopback" in evidence["message"].lower(), evidence + + def test_runtime_status_reports_the_hosted_mode_it_loaded(scrubbed) -> None: for name, value in HOSTED.items(): scrubbed.setenv(name, value) From 02dadc5590dafe69a2ef156dc3d77bf275e24d67 Mon Sep 17 00:00:00 2001 From: Brett Heap <1513478+brettheap@users.noreply.github.com> Date: Wed, 30 Sep 2026 01:12:23 +0000 Subject: [PATCH 10/39] Fix round: status reports a local install's broker on its early return too (Copilot review) When the runtime extra is absent, `runtime status` returns early, and that return said `broker_keys: "not probed"` for every install. A local install's broker is not configured whether or not the extra is present. That answer comes from the configuration, not from a probe, so the early return now gives the local install the answer the full report gives: `"not configured (local mode)"`, with `broker_discovery: null`. Both returns write it through one helper, so the two cannot drift. A hosted install's early return still reads "not probed", as before (13.6). The branch is covered now, so its `pragma: no cover` goes. A new case runs both shapes with `opendox.runtime.db` absent from `sys.modules`. Before (`525f61c`'s runtime/cli.py): local 1 failed and hosted passed. After: both pass. Four mutants of the fix are killed. Full suite: 2540 selected, 2529 passed, 11 skipped, 0 failed. Lane: openxfactory-4 (openXfactory-4-openDox_extraction) Arc: neutral-product-standalone-operability Co-Authored-By: Claude Opus 5.5 (1M context) --- src/opendox/runtime/cli.py | 28 ++++++++++++++++++++++++---- tests_runtime/test_install_mode.py | 28 ++++++++++++++++++++++++++++ 2 files changed, 52 insertions(+), 4 deletions(-) diff --git a/src/opendox/runtime/cli.py b/src/opendox/runtime/cli.py index af0ac390..a69bfd41 100644 --- a/src/opendox/runtime/cli.py +++ b/src/opendox/runtime/cli.py @@ -664,6 +664,19 @@ def cmd_serve(args: argparse.Namespace) -> int: "exiting normally"}, ok=True) +def _report_the_local_broker(report: dict[str, Any]) -> None: + """What `status` says of a LOCAL install's broker, on every path. + + Its broker is NOT CONFIGURED (plan 034 T070; #1144 13.4): a statement + about the install's configuration rather than a probe's result, so there + is no discovery URL to report and nothing counts against `ok`. `status` + returns from two places, and both write it here, so the two answers + cannot drift apart (Copilot review of openDox-code#67). + """ + report["broker_keys"] = "not configured (local mode)" + report["broker_discovery"] = None + + def cmd_status(args: argparse.Namespace) -> int: """Report, never change: configuration, the schema pin, the ledger, the broker. @@ -693,10 +706,18 @@ def cmd_status(args: argparse.Namespace) -> int: try: from opendox.runtime.db import Database - except ImportError as exc: # pragma: no cover - the extra is absent + except ImportError as exc: report["runtime_extra"] = f"absent: {_safe_message(exc)}" report["database"] = "not probed" - report["broker_keys"] = "not probed" + # A LOCAL INSTALL'S BROKER IS NOT CONFIGURED WHETHER OR NOT THE EXTRA + # IS PRESENT (plan 034 T070; Copilot review of openDox-code#67). That + # answer comes from its configuration, not from a probe, so this early + # return gives the same one the full report gives below. A hosted + # install's broker was never probed, and says so, as before. + if settings.install_mode == INSTALL_MODE_LOCAL: + _report_the_local_broker(report) + else: + report["broker_keys"] = "not probed" return _emit(report, ok=False) report["runtime_extra"] = "present" @@ -778,8 +799,7 @@ def cmd_status(args: argparse.Namespace) -> int: # — F13.1 runs it under `set -e`, and a verdict of "unhealthy" for a # broker the install was never meant to have would be false. if settings.install_mode == INSTALL_MODE_LOCAL: - report["broker_keys"] = "not configured (local mode)" - report["broker_discovery"] = None + _report_the_local_broker(report) return _emit(report, ok=ok) try: from opendox.runtime.oidc import build_verifier diff --git a/tests_runtime/test_install_mode.py b/tests_runtime/test_install_mode.py index 28932ab3..f9c4bf9e 100644 --- a/tests_runtime/test_install_mode.py +++ b/tests_runtime/test_install_mode.py @@ -352,6 +352,34 @@ def test_migrate_refuses_a_non_loopback_bind_beside_the_local_mode( assert "loopback" in evidence["message"].lower(), evidence +@pytest.mark.parametrize("mode", [INSTALL_MODE_LOCAL, INSTALL_MODE_HOSTED]) +def test_runtime_status_without_the_runtime_extra_reports_the_broker_by_mode( + scrubbed, mode: str) -> None: + """`status` returns early when the runtime extra is absent, and that + return gives the local broker the same answer the full report gives + (Copilot review of openDox-code#67). A hosted install's broker reads + "not probed", unchanged (13.6). `None` in `sys.modules` is how an absent + module is simulated: the import raises `ImportError`, as it would without + the extra.""" + import sys + + scrubbed.setitem(sys.modules, "opendox.runtime.db", None) + for name, value in (HOSTED if mode == INSTALL_MODE_HOSTED else DSNS).items(): + scrubbed.setenv(name, value) + scrubbed.setenv(MODE, mode) + code, evidence = _run(["runtime", "status", "--probe-timeout", "0.2"]) + assert evidence["runtime_extra"].startswith("absent"), evidence + assert evidence["database"] == "not probed", evidence + assert code == 1 + if mode == INSTALL_MODE_LOCAL: + assert evidence["broker_keys"] == "not configured (local mode)", evidence + assert "broker_discovery" in evidence, evidence + assert evidence["broker_discovery"] is None + else: + assert evidence["broker_keys"] == "not probed", evidence + assert "broker_discovery" not in evidence, evidence + + def test_runtime_status_reports_the_hosted_mode_it_loaded(scrubbed) -> None: for name, value in HOSTED.items(): scrubbed.setenv(name, value) From 859b37b60a6c70818bcfdd8749ce232dcdfb7741 Mon Sep 17 00:00:00 2001 From: Brett Heap <1513478+brettheap@users.noreply.github.com> Date: Wed, 30 Sep 2026 15:04:37 +0000 Subject: [PATCH 11/39] Fix round: a healthy local status is proven to exit 0, and RuntimeSettings' broker invariants are scoped (Copilot review) A local `status` returns `ok` on the database's verdict alone. Both earlier local cases forced a database fault and asserted exit 1, so a regression that also counted the absent broker as a fault would still have passed. A DB-backed case now runs `status` for a local install against a migrated schema on the suite's own server (`database` and `postgres_dsn`, with the schema selected in the DSN). It asserts `ok` true, exit 0, the database reachable with nothing pending and no drift, and the broker reported as not configured and never probed. Measured: with the local return mutated to `ok=False`, this case fails and the other 40 in the module pass. `RuntimeSettings`' docstring said that a local install's issuer and audience are empty and that a hosted one always carries a real issuer. That is true of `load_settings` alone. `load_migration_settings` carries the migration sentinels in either shape. The docstring now scopes each statement to its loader. Full suite: 2541 selected, 2530 passed, 11 skipped, 0 failed. Lane: openxfactory-4 (openXfactory-4-openDox_extraction) Arc: neutral-product-standalone-operability Co-Authored-By: Claude Opus 5.5 (1M context) --- src/opendox/runtime/config.py | 22 +++++++++++++++------ tests_runtime/test_install_mode.py | 31 ++++++++++++++++++++++++++++++ 2 files changed, 47 insertions(+), 6 deletions(-) diff --git a/src/opendox/runtime/config.py b/src/opendox/runtime/config.py index 326144e4..ad8c518d 100644 --- a/src/opendox/runtime/config.py +++ b/src/opendox/runtime/config.py @@ -203,12 +203,22 @@ class RuntimeSettings: requires one (unaffected by this: it already refused to load without one). `install_mode` is `INSTALL_MODE_HOSTED` or `INSTALL_MODE_LOCAL` (plan 034 - T070; #1144 13.4). A LOCAL install has no broker, so its `oidc_issuer` and - `oidc_audience` are EMPTY and its `oidc_jwks_url` is `None` — never a - placeholder that looks like an endpoint — and `jwks_url()` and - `discovery_url()` answer the empty string for it rather than a path glued - onto nothing. A hosted install always carries a real issuer, because - `load_settings` refuses one without it. + T070; #1144 13.4), whichever loader built the object. + + THE BROKER FIELDS DEPEND ON THE LOADER, and what follows holds for + `load_settings` only (Copilot review of openDox-code#67): + * From `load_settings`, a LOCAL install has no broker. Its + `oidc_issuer` and `oidc_audience` are EMPTY and its `oidc_jwks_url` + is `None`, never a placeholder that looks like an endpoint, and + `jwks_url()` and `discovery_url()` answer the empty string rather + than a path glued onto nothing. A HOSTED one always carries a real + issuer, because `load_settings` refuses one without it. + * From `load_migration_settings`, in EITHER shape, the issuer and + audience are `MIGRATION_SENTINEL_ISSUER` and + `MIGRATION_SENTINEL_AUDIENCE`. A migration run reaches no broker at + all, and anything that tried to with those values would fail naming + them. So neither statement above applies to it. `install_mode` there + records the shape the run belongs to, and nothing else. """ database_url: str diff --git a/tests_runtime/test_install_mode.py b/tests_runtime/test_install_mode.py index f9c4bf9e..8fcc2a7b 100644 --- a/tests_runtime/test_install_mode.py +++ b/tests_runtime/test_install_mode.py @@ -352,6 +352,37 @@ def test_migrate_refuses_a_non_loopback_bind_beside_the_local_mode( assert "loopback" in evidence["message"].lower(), evidence +def test_runtime_status_of_a_healthy_local_install_exits_zero( + scrubbed, monkeypatch: pytest.MonkeyPatch, postgres_dsn: str, + database) -> None: + """THE EXIT CODE IS THE DATABASE'S VERDICT ALONE (Copilot review of + openDox-code#67). A migrated, reachable database is the only thing a + local install's `status` needs, and with it the verb exits 0. The broker + is reported as not configured and is never probed, so F13.1's `set -e` + survives. The other local cases force a database fault and exit 1, so + they cannot tell a broker counted as a fault from a database that failed. + """ + from opendox.runtime import oidc + + def _no_broker(_settings): + raise AssertionError("status built a broker verifier for a LOCAL " + "install, which has no broker") + + monkeypatch.setattr(oidc, "build_verifier", _no_broker) + joiner = "&" if "?" in postgres_dsn else "?" + scrubbed.setenv(PREFIX + "DATABASE_URL", + f"{postgres_dsn}{joiner}options=-c%20search_path%3D" + f"{database.schema}%2Cpublic") + scrubbed.setenv(MODE, "local") + code, evidence = _run(["runtime", "status", "--probe-timeout", "5"]) + assert evidence["database"] == "reachable", evidence + assert evidence["pending_migrations"] == [], evidence + assert not evidence["migration_drift"], evidence + assert evidence["broker_keys"] == "not configured (local mode)", evidence + assert evidence["broker_discovery"] is None + assert evidence["ok"] is True and code == 0, evidence + + @pytest.mark.parametrize("mode", [INSTALL_MODE_LOCAL, INSTALL_MODE_HOSTED]) def test_runtime_status_without_the_runtime_extra_reports_the_broker_by_mode( scrubbed, mode: str) -> None: From 5e528729aaa1097e4ff5009f7d2a50bfd78fc394 Mon Sep 17 00:00:00 2001 From: Brett Heap <1513478+brettheap@users.noreply.github.com> Date: Wed, 30 Sep 2026 15:11:40 +0000 Subject: [PATCH 12/39] Fix round: the local install's migrations, pid, initdb, start and interrupts hardened (Copilot review) Copilot's reviews at 95fe16f and 32db5d8 opened ten threads. Eight are fixed here. The two about the server package (PostgreSQL 16.2, no wheel for 3.13) wait on the holder. - Migrations (r4139811473, r4139880241). An explicit OPENDOX_MIGRATIONS_DIR is used as given. Unset, a LOCAL install uses only the copy its own installation carries, and never the working directory's: its entry point runs every migration as the bundle's owner, and the canonical gate pins 0001 alone. Where the installation carries none, it is refused, naming the setting. The installation's copy is the source tree the module was imported from (src/ beside a pyproject.toml naming opendox), then the RECORD of the distribution that holds the running module, and never another one found by name. A HOSTED install's unset default is unchanged (13.6). - The pid (r4139811555). A postmaster.pid is believed only for this data directory's postmaster, as the kernel reports it: an executable named postgres whose working directory is the data directory. Another user's process is never believed. A lock that /proc proves stale is removed before the launch, so a recycled pid no longer holds the bundle. - initdb (r4139880213). It runs into an attempt directory beside the data directory, which is renamed into place only on success. An attempt whose process is gone is removed. A non-empty data directory that holds no cluster is refused and left untouched. - start() (r4139880279). Directories, initialize, launch, wait, bootstrap and migrate are one guarded operation, and every failure is the one named refusal (phase and class name), with anything started stopped. - Interrupts (r4139880267). SIGTERM or Ctrl-C anywhere in the local lifecycle is a clean stop: no traceback, the bundle stopped, the handler restored first. Nothing was served, so the exit is 128 + the signal number. A served run ended by SIGTERM still exits 0. - Refusal wording (r4139880298). Broker settings and operator DSNs are two classes, and each is refused with its own reason. - The test helper (r4139811584). The launch helper is bounded by its deadline, through a selector. Measured with a silent 8 s child and a 1 s deadline: the old loop returned after 8.0 s, the new one after 1.0 s. tests_runtime/test_local_lifecycle.py (new, hermetic) holds these cases, plus a real-server stale-lock case and the helper's own case in test_bundled_postgres.py. Against ac61596's source, 17 of the module's first 18 cases fail. The one that passes is the hosted default, which is unchanged on purpose. All 23 mutants of the fixes are killed. Lane: openxfactory-4 (openXfactory-4-openDox_extraction) Arc: neutral-product-standalone-operability Co-Authored-By: Claude Opus 5.5 (1M context) --- src/opendox/cli.py | 34 +- src/opendox/runtime/bundle.py | 185 ++++++++-- src/opendox/runtime/config.py | 195 +++++++--- tests_runtime/test_bundled_postgres.py | 96 ++++- tests_runtime/test_local_lifecycle.py | 489 +++++++++++++++++++++++++ 5 files changed, 913 insertions(+), 86 deletions(-) create mode 100644 tests_runtime/test_local_lifecycle.py diff --git a/src/opendox/cli.py b/src/opendox/cli.py index 11c3b203..5e0073ee 100644 --- a/src/opendox/cli.py +++ b/src/opendox/cli.py @@ -459,9 +459,19 @@ def _resolve_install_shape(args: argparse.Namespace, return runtime_config.load_settings(env, local_flag=local_flag) -def _terminate_as_interrupt(signum, frame): # pragma: no cover - a signal +class _Terminated(KeyboardInterrupt): + """SIGTERM, raised as the interrupt the serve loop already stops cleanly on. + + A subclass, so the serve loop's own `except KeyboardInterrupt` still ends + a served run with 0 (F13.1's `kill "$SERVER"; wait "$SERVER"`). An + interrupt that arrives BEFORE the serve loop can still say which signal + it was. + """ + + +def _terminate_as_interrupt(signum, frame): """SIGTERM, read as the Ctrl-C the serve loop already stops cleanly on.""" - raise KeyboardInterrupt + raise _Terminated def cmd_generate_and_open(args: argparse.Namespace, *, opener=webbrowser.open) -> int: @@ -513,10 +523,28 @@ def cmd_generate_and_open(args: argparse.Namespace, *, opener=webbrowser.open) - print(f" database {report['socket_dir']} (bundled, pid {report['pid']}, " f"migrations applied now: {server.applied or 'none pending'})") return _generate_and_open(args, opener=opener) + except KeyboardInterrupt as interrupt: + # AN INTERRUPT ANYWHERE IN THE LOCAL LIFECYCLE IS A CLEAN STOP (Copilot + # review of openDox-code#69). SIGTERM, or Ctrl-C, can arrive while the + # server is initializing or migrating, or while the snapshot is being + # generated, all before the serve loop's own handler. It is a stop + # that was asked for, so it is not a traceback: the `finally` below + # stops the bundled server and restores the handler. Nothing was + # served, so the exit is the signal's conventional status (128 + its + # number) and not 0. + signum = (signal.SIGTERM if isinstance(interrupt, _Terminated) + else signal.SIGINT) + print(f"generate-and-open interrupted ({signum.name}) before it " + "served; its bundled PostgreSQL server stops with it", + file=sys.stderr) + return 128 + int(signum) finally: - server.stop() + # THE HANDLER FIRST, so a second SIGTERM during the stop takes the + # default action at once; the parent-death signal still stops the + # server if this process goes before `stop()` has finished. if previous is not None: signal.signal(signal.SIGTERM, previous) + server.stop() def _generate_and_open(args: argparse.Namespace, *, opener) -> int: diff --git a/src/opendox/runtime/bundle.py b/src/opendox/runtime/bundle.py index 701aa1bf..633dc3a3 100644 --- a/src/opendox/runtime/bundle.py +++ b/src/opendox/runtime/bundle.py @@ -29,7 +29,9 @@ THE LIFECYCLE, IN FULL: - * `initdb` once per data directory: the MIGRATION identity + * `initdb` once per data directory, into an attempt directory that is + renamed into place only when it has succeeded, so an interrupted first + start never leaves a half-built cluster: the MIGRATION identity (`config.BUNDLE_OWNER_ROLE`) is the bootstrap superuser, local connections are `trust` and host connections are `reject`, UTF-8 in the `C` locale. Trust is safe BECAUSE of the socket: its directory is 0700, owned by the @@ -60,9 +62,11 @@ import ctypes import importlib.util import os +import shutil import signal import subprocess import sys +import tempfile import time from pathlib import Path from typing import Any @@ -129,33 +133,84 @@ def server_binaries() -> Path: return binaries +def _lock_file_pid(bundle: DatabaseBundle) -> int | None: + """The pid on the first line of the server's `postmaster.pid`, or `None`.""" + try: + first = (bundle.data_dir / "postmaster.pid").read_text( + encoding="utf-8").splitlines()[0] + return int(first.strip()) + except (OSError, IndexError, ValueError): + return None + + +def _serves(pid: int, bundle: DatabaseBundle) -> bool | None: + """Whether `pid` is the postmaster of `bundle`'s data directory. + + Asked of the KERNEL, as the pair this module launches: an executable named + `postgres` whose working directory IS the data directory. The postmaster + changes into its data directory at startup, and neither of the two can be + rewritten by its process title, so a recycled pid given to anything else, + even another `postgres` serving another directory, is not it (Copilot + review of openDox-code#69). + + `False` also when the kernel will not say: another user's process cannot + be this bundle's server, because the server runs as the owner of a 0700 + data directory, the user this runs as. `None` only where there is no + `/proc` to ask at all. + """ + if not Path("/proc/self").exists(): + return None + try: + executable = os.readlink(f"/proc/{pid}/exe") + cwd = os.readlink(f"/proc/{pid}/cwd") + except OSError: + return False + if Path(executable.removesuffix(" (deleted)")).name != "postgres": + return False + try: + return Path(cwd).resolve() == bundle.data_dir.resolve() + except OSError: + return False + + def running_pid(bundle: DatabaseBundle) -> int | None: """The pid of a live server on `bundle`'s data directory, or `None`. Read from the server's own `postmaster.pid` (its first line), and only - believed while that process exists: a file left by a server that did not - stop cleanly names a pid that is gone, or that the kernel has since given - to something else, and PostgreSQL itself treats such a file as stale. + believed while that process exists and IS this bundle's server: a file + left by a server that did not stop cleanly names a pid that is gone, or + that the kernel has since given to something else (`_serves`). """ - try: - first = (bundle.data_dir / "postmaster.pid").read_text( - encoding="utf-8").splitlines()[0] - pid = int(first.strip()) - except (OSError, IndexError, ValueError): + pid = _lock_file_pid(bundle) + if pid is None: return None try: os.kill(pid, 0) - except ProcessLookupError: + except (ProcessLookupError, PermissionError): + # gone; or alive and another user's, which cannot be this server return None - except PermissionError: - return pid # alive, and not ours to signal - # A REUSED PID IS NOT A SERVER: where the kernel says what runs there, it - # must be a postgres, or the file is stale whatever its first line says. + return pid if _serves(pid, bundle) is not False else None + + +def _remove_a_proven_stale_lock(bundle: DatabaseBundle) -> None: + """Remove a `postmaster.pid` the kernel PROVES is not this server's. + + PostgreSQL removes a lock file whose pid is gone. It refuses to start, + however, over one whose pid the kernel has given to another live process + of the same user, and that refusal would last as long as the unrelated + process does. Where `/proc` shows that process is not this data + directory's postmaster, the lock is stale by proof and is removed. Where + nothing can be proven, it is left for PostgreSQL to judge. + """ + pid = _lock_file_pid(bundle) + if pid is None: + return try: - command = Path(f"/proc/{pid}/cmdline").read_bytes() - except OSError: - return pid # no /proc to ask; believe the live pid - return pid if b"postgres" in command else None + os.kill(pid, 0) + except (ProcessLookupError, PermissionError): + return # PostgreSQL's own rule covers both + if _serves(pid, bundle) is False: + (bundle.data_dir / "postmaster.pid").unlink(missing_ok=True) def report(bundle: DatabaseBundle) -> dict[str, Any]: @@ -245,25 +300,37 @@ def start(self) -> BundledServer: f"(pid {already}). One local install's database belongs to one " "entry point at a time: stop the other `generate-and-open " f"{LOCAL_FLAG}`, or give this one its own {PREFIX}STATE_DIR") - self._prepare_directories() - if not (self.bundle.data_dir / "PG_VERSION").is_file(): - self._initdb(binaries) - self._launch(binaries) + # THE WHOLE START IS ONE GUARDED OPERATION (Copilot review of + # openDox-code#69). The directories, `initdb` and the launch fail as + # plainly as the connection does: a timeout, a permission, a missing + # file. Each one comes out as the one named refusal the entry point + # prints, with whatever was started stopped, never as a traceback. + phase = "preparing its directories" try: + self._prepare_directories() + phase = "initializing its data directory" + self._initialize(binaries) + phase = "launching it" + _remove_a_proven_stale_lock(self.bundle) + self._launch(binaries) + phase = "waiting for it to accept a connection" self._wait_until_ready() + phase = "bootstrapping its database and served role" self._bootstrap() + phase = "migrating it" self._migrate() except (BundleRefused, migrations.MigrationError): self.stop() raise - except Exception as exc: # noqa: BLE001 - the driver's, named not quoted + except Exception as exc: # noqa: BLE001 - named, never quoted # THE DRIVER'S TEXT IS NOT REPEATED, for the reason `runtime/cli.py` - # gives: it quotes the connection string it was handed. The class - # names what went wrong and the server's own log says the rest. + # gives: it quotes the connection string it was handed. The phase + # and the class name say what went wrong, and the server's own log + # says the rest. self.stop() raise BundleRefused( - f"the bundled PostgreSQL server started but could not be " - f"prepared ({type(exc).__name__}); its log is " + f"the bundled PostgreSQL server could not be started " + f"({phase}: {type(exc).__name__}); its log is " f"{self.log_path}") from None except BaseException: self.stop() @@ -284,9 +351,69 @@ def _prepare_directories(self) -> None: self.bundle.socket_dir.mkdir(mode=0o700, parents=True, exist_ok=True) os.chmod(self.bundle.socket_dir, 0o700) - def _initdb(self, binaries: Path) -> None: + #: The prefix an initialization attempt's directory carries, beside the + #: data directory, followed by the pid of the process making it. + ATTEMPT_PREFIX = "data.initdb-" + + def _initialize(self, binaries: Path) -> None: + """A complete cluster at `data_dir`, or a refusal. Never a partial one. + + `initdb` runs into an ATTEMPT directory beside the data directory, and + the attempt is renamed into place only once `initdb` has succeeded. So + a data directory exists only as a finished cluster, and a first start + that dies midway (a kill, a full disk, a power cut) leaves an attempt + and not a half-built data directory that the next start would either + launch or fail to re-initialize for ever (Copilot review of + openDox-code#69). The next start removes an attempt whose process is + gone, and starts again. + + A data directory that exists and holds no cluster is NOT this + install's to remove, unless it is empty: it is refused, named, and + left as it is. + """ + data = self.bundle.data_dir + if (data / "PG_VERSION").is_file(): + return + if data.exists(): + try: + data.rmdir() # an empty directory holds nothing + except OSError: + raise BundleRefused( + f"{data} exists and holds no PostgreSQL cluster, so it is " + "not this install's database, and it is left untouched: " + "move it aside, or give this install its own " + f"{PREFIX}STATE_DIR") from None + self._remove_abandoned_attempts() + attempt = Path(tempfile.mkdtemp( + prefix=f"{self.ATTEMPT_PREFIX}{os.getpid()}-", dir=data.parent)) + try: + self._initdb(binaries, attempt) + try: + os.rename(attempt, data) + except OSError: + if not (data / "PG_VERSION").is_file(): + raise + # another start finished first; its cluster is the one used + shutil.rmtree(attempt, ignore_errors=True) + except BaseException: + shutil.rmtree(attempt, ignore_errors=True) + raise + + def _remove_abandoned_attempts(self) -> None: + """Every initialization attempt whose process no longer exists.""" + for candidate in self.bundle.data_dir.parent.glob( + f"{self.ATTEMPT_PREFIX}*"): + owner = candidate.name[len(self.ATTEMPT_PREFIX):].split("-", 1)[0] + try: + os.kill(int(owner), 0) + except ProcessLookupError: + shutil.rmtree(candidate, ignore_errors=True) + except (ValueError, OSError): + continue # alive, or not ours to judge + + def _initdb(self, binaries: Path, target: Path) -> None: done = subprocess.run( - [str(binaries / "initdb"), "-D", str(self.bundle.data_dir), + [str(binaries / "initdb"), "-D", str(target), "-U", BUNDLE_OWNER_ROLE, "--auth-local=trust", "--auth-host=reject", "--encoding=UTF8", "--locale=C", "--no-instructions"], env=_child_environment(), capture_output=True, text=True, diff --git a/src/opendox/runtime/config.py b/src/opendox/runtime/config.py index 229b3e75..f6257bbe 100644 --- a/src/opendox/runtime/config.py +++ b/src/opendox/runtime/config.py @@ -32,10 +32,12 @@ import re import shlex import sys +import tomllib import urllib.parse from collections.abc import Mapping from dataclasses import dataclass from pathlib import Path, PurePath +from typing import Any #: The environment prefix. One string, so a rename is one edit. PREFIX = "OPENDOX_" @@ -186,11 +188,13 @@ class Setting: ), Setting( PREFIX + "MIGRATIONS_DIR", "migrations", False, False, - "the ordered-SQL directory, repository-root-relative. Unset, it is " - "`migrations` where the working directory holds one (a checkout, or " - "the image's /app), and otherwise the copy the installed package " - "ships (plan 034 T072), so an install run from anywhere else still " - "has the migrations it applies", + "the ordered-SQL directory, repository-root-relative. Unset, a HOSTED " + "install uses `migrations` where the working directory holds one (a " + "checkout, or the image's /app), as it always has, and otherwise the " + "copy this installation carries; a LOCAL install uses ONLY the copy " + "this installation carries and never the working directory's (plan " + "034 T072), so launching it from a checkout of somebody else's " + "repository cannot run that repository's SQL", ), Setting( PREFIX + "PROJECT_REPOSITORY_ROOT", "var/projects", False, False, @@ -1579,13 +1583,21 @@ def _refuse_two_dsns_that_select_different_schemas( #: pre-existing service can stand in for it"), so an operator's DSN beside #: `local` is either about to be silently overridden or is another server #: trying to stand in for the bundled one. Neither is accepted. -HOSTED_ONLY_SETTINGS: tuple[str, ...] = ( +#: +#: TWO CLASSES, TWO REASONS (Copilot review of openDox-code#69): the broker +#: settings are refused because honouring `local` would drop an +#: authentication, and the DSNs because the local install supplies its own +#: database. Each refusal says its own reason. +BROKER_SETTINGS: tuple[str, ...] = ( PREFIX + "OIDC_ISSUER", PREFIX + "OIDC_AUDIENCE", PREFIX + "OIDC_JWKS_URL", +) +OPERATOR_DATABASE_SETTINGS: tuple[str, ...] = ( PREFIX + "DATABASE_URL", PREFIX + "MIGRATION_DATABASE_URL", ) +HOSTED_ONLY_SETTINGS: tuple[str, ...] = BROKER_SETTINGS + OPERATOR_DATABASE_SETTINGS #: THE BUNDLED SERVER'S IDENTITY (plan 034 T072; #1144 13.1, as T007 batch H's #: addendum reads). The data directory and the socket directory live under the @@ -1712,49 +1724,121 @@ def database_bundle(state: Path) -> DatabaseBundle: PACKAGED_MIGRATIONS = PurePath("share", "opendox", "migrations") -def packaged_migrations_dir() -> Path | None: - """The migrations this INSTALLATION carries, or `None` where it carries none. +def _source_tree_migrations(module_file: Path) -> Path | None: + """The `migrations/` of the SOURCE TREE `module_file` was imported from. - Two places, in order. The installed distribution's own data files — a - wheel install, whose `RECORD` lists them wherever the install scheme put - them. Then the source tree this module was imported from — an editable - install, which installs no data files, or `src/` on the path — whose - `migrations/` sits beside `src/`. + `module_file` is this module (`src/opendox/runtime/config.py`), so the + tree is three directories up: a checkout run with `src/` on the path, or + an editable install, neither of which installs data files. It counts only + where it really is that tree: `src/` is the directory the package sits + in, and the root's `pyproject.toml` names THIS project. A wheel's + `site-packages`, or a `--target` directory that happens to sit inside + some other checkout, is neither. """ + here = module_file.resolve() + package_parent, root = here.parents[2], here.parents[3] + if package_parent.name != "src": + return None try: - files = importlib.metadata.distribution("opendox").files or () - except importlib.metadata.PackageNotFoundError: - files = () + project = tomllib.loads( + (root / "pyproject.toml").read_text(encoding="utf-8")) + except (OSError, tomllib.TOMLDecodeError): + return None + if project.get("project", {}).get("name") != "opendox": + return None + candidate = root / "migrations" + return candidate if candidate.is_dir() else None + + +def _distribution_migrations(module_file: Path, + distribution: Any | None = None) -> Path | None: + """The migrations the INSTALLED DISTRIBUTION of `module_file` carries. + + From the distribution's own `RECORD`, wherever its install scheme put the + data files — and ONLY when that distribution is the one `module_file` was + loaded from. A name lookup alone is not: with a checkout's `src/` on the + path, `distribution("opendox")` can find an older wheel installed beside + it, and that wheel's migrations are another version's (Copilot review of + openDox-code#69). + """ + if distribution is None: + try: + distribution = importlib.metadata.distribution("opendox") + except importlib.metadata.PackageNotFoundError: + return None + files = list(distribution.files or ()) + here = module_file.resolve() + tail = here.parts[-3:] # ("opendox", "runtime", "config.py") + if not any(PurePath(entry).parts[-3:] == tail + and Path(entry.locate()).resolve() == here for entry in files): + return None for entry in files: parts = PurePath(entry).parts if (entry.name.endswith(".sql") and tuple(parts[-4:-1]) == PACKAGED_MIGRATIONS.parts): return Path(entry.locate()).resolve().parent - source = Path(__file__).resolve().parents[3] - if (source / "pyproject.toml").is_file() and (source / "migrations").is_dir(): - return source / "migrations" return None -def migrations_dir(env: Mapping[str, str] | None = None) -> Path: +def installation_migrations_dir() -> Path | None: + """The migrations THIS INSTALLATION carries, or `None` where it carries none. + + Tied to the code that is running, never to a name or to the working + directory. First the source tree this module was imported from (a + checkout, or an editable install). Then the installed distribution that + this module belongs to, from its `RECORD` (a wheel install). A real wheel + install has no `pyproject.toml` beside its package, so it falls through to + its own `RECORD` (Copilot review of openDox-code#69). + """ + module_file = Path(__file__) + return (_source_tree_migrations(module_file) + or _distribution_migrations(module_file)) + + +def migrations_dir(env: Mapping[str, str] | None = None, *, + local: bool = False) -> Path: """`OPENDOX_MIGRATIONS_DIR`, or where this install's migrations are. - Set, it is used as given, as it always was. Unset, it is `migrations` - wherever the working directory holds one — a checkout, or the image's - `/app` — which is today's default, unchanged; and OTHERWISE the copy the - installed package carries (`packaged_migrations_dir`), so `pip install - "opendox[local]"` run from a user's home directory migrates its bundled - server instead of refusing for a missing directory. Where neither exists - it is still `migrations`, and the canonical gate refuses it by name. + SET, it is used as given, in either shape: executing another directory's + SQL is something an operator says, not something a directory implies. + + UNSET, the shape decides. + * A LOCAL install uses ONLY the copy this installation carries + (`installation_migrations_dir`), never the working directory's. Its + entry point runs every migration it finds as the bundled server's + OWNER, and the canonical gate pins `0001` alone. So a `migrations/` + in whatever directory a user launches it from, holding the pinned + `0001` and SQL of its own, would otherwise be executed (Copilot + review of openDox-code#69). An installation that carries none is + refused, naming the setting. + * A HOSTED install keeps today's default, unchanged (13.6): `migrations` + where the working directory holds one (the image's `/app`, a + checkout), and otherwise the copy this installation carries. Where + neither exists, it is still `migrations`, and the canonical gate + refuses it by name. The compose file and the Kubernetes manifests + set the variable explicitly, so they read nothing implicit. """ env = os.environ if env is None else env - raw = env.get(PREFIX + "MIGRATIONS_DIR", "").strip() + setting = PREFIX + "MIGRATIONS_DIR" + raw = env.get(setting, "").strip() if raw: return Path(raw) + if local: + found = installation_migrations_dir() + if found is None: + raise ConfigurationError( + f"{setting} is unset, and this installation carries no " + "migrations of its own: a wheel install has them under " + f"`{PACKAGED_MIGRATIONS}` in its data directory, and a " + "checkout has `migrations/` beside `src/`. A LOCAL install " + "never reads the working directory's `migrations/`, because " + "its entry point runs them as the bundled server's owner. " + f"Reinstall the package, or name the directory in {setting}") + return found here = Path("migrations") if here.is_dir(): return here - return packaged_migrations_dir() or here + return installation_migrations_dir() or here def install_mode(env: Mapping[str, str] | None = None, *, @@ -1837,21 +1921,46 @@ def refuse_what_a_local_install_cannot_be(env: Mapping[str, str]) -> None: _optional(env, _by_name(PREFIX + "BIND_HOST")) or "127.0.0.1") +def _named(given: list[str]) -> str: + return f"{' and '.join(given)} {'are' if len(given) > 1 else 'is'} set" + + def _refuse_hosted_only_settings(env: Mapping[str, str]) -> None: - """Every setting in `HOSTED_ONLY_SETTINGS` given beside `local`, named.""" - given = [name for name in HOSTED_ONLY_SETTINGS - if env.get(name, "").strip()] - if given: - raise ConfigurationError( - f"{' and '.join(given)} {'are' if len(given) > 1 else 'is'} set, " - "and this is a LOCAL install, which has no broker and reads " - f"{'none of them' if len(given) > 1 else 'none'}. A broker " + """Every setting in `HOSTED_ONLY_SETTINGS` given beside `local`, named. + + Each CLASS carries its own reason (Copilot review of openDox-code#69). + A broker setting says a hosted install was meant, and a DSN says another + database was meant. A local run with only `OPENDOX_DATABASE_URL` is told + about its database, not about an authentication it never configured. + The values are never repeated: a DSN carries a password. + """ + def given(names: tuple[str, ...]) -> list[str]: + return [name for name in names if env.get(name, "").strip()] + + brokers, databases = given(BROKER_SETTINGS), given(OPERATOR_DATABASE_SETTINGS) + if not (brokers or databases): + return + reasons = [] + if brokers: + reasons.append( + f"{_named(brokers)}, and a LOCAL install has no broker and reads " + f"{'none of them' if len(brokers) > 1 else 'none'}. A broker " "setting beside the local mode says a HOSTED install was meant, " "and honouring `local` over it would silently drop that " - f"authentication: unset {'them' if len(given) > 1 else 'it'} for " - f"a local install, or drop {LOCAL_FLAG} / " - f"{PREFIX}INSTALL_MODE=local for a hosted one (the values are not " - "repeated here)") + "authentication") + if databases: + reasons.append( + f"{_named(databases)}, and a LOCAL install supplies BOTH of its " + "DSNs itself, from the PostgreSQL server it bundles under " + f"{PREFIX}STATE_DIR (13.1). An operator's DSN beside the local " + "mode would either be silently overridden or be another server " + "standing in for the bundled one, and neither is accepted") + every = brokers + databases + raise ConfigurationError( + "; and ".join(reasons) + f". Unset {'them' if len(every) > 1 else 'it'} " + f"for a local install, or drop {LOCAL_FLAG} / " + f"{PREFIX}INSTALL_MODE=local for a hosted one (the values are not " + "repeated here)") def require_the_hosted_issuer(env: Mapping[str, str] | None = None) -> None: @@ -1998,7 +2107,7 @@ def load_settings(env: Mapping[str, str] | None = None, *, served_schema=_served_schema(env), served_database=_served_database(env) or (BUNDLE_DATABASE if local else None), publish_openapi=_boolean(env, _by_name(PREFIX + "PUBLISH_OPENAPI")), - migrations_dir=migrations_dir(env), + migrations_dir=migrations_dir(env, local=local), project_repository_root=Path( _optional(env, _by_name(PREFIX + "PROJECT_REPOSITORY_ROOT")) or "var/projects" ), @@ -2081,7 +2190,7 @@ def load_migration_settings(env: Mapping[str, str] | None = None) -> RuntimeSett served_schema=_served_schema(env), served_database=_served_database(env) or (BUNDLE_DATABASE if local else None), publish_openapi=False, - migrations_dir=migrations_dir(env), + migrations_dir=migrations_dir(env, local=local), project_repository_root=Path( _optional(env, _by_name(PREFIX + "PROJECT_REPOSITORY_ROOT")) or "var/projects"), diff --git a/tests_runtime/test_bundled_postgres.py b/tests_runtime/test_bundled_postgres.py index 895bcfb9..acf6e369 100644 --- a/tests_runtime/test_bundled_postgres.py +++ b/tests_runtime/test_bundled_postgres.py @@ -33,6 +33,7 @@ import json import os import re +import selectors import shutil import signal import subprocess @@ -141,6 +142,33 @@ def _wait_gone(pid: int, seconds: float = 30.0) -> bool: return False +def _first_url(child: subprocess.Popen, seconds: float) -> str | None: + """The first `http://` line `child` prints within `seconds`, or `None`. + + BOUNDED BY THE DEADLINE, not by the child: a blocking `readline()` waits + for as long as a child that is alive and silent stays so, which is exactly + the startup failure this has to diagnose (Copilot review of + openDox-code#69). So the pipe is polled with a selector, only up to the + time left, and read in whatever pieces arrive. + """ + deadline = time.monotonic() + seconds + pending = b"" + with selectors.DefaultSelector() as selector: + selector.register(child.stdout, selectors.EVENT_READ) + while (remaining := deadline - time.monotonic()) > 0: + if not selector.select(timeout=remaining): + return None # the deadline + chunk = os.read(child.stdout.fileno(), 65536) + if not chunk: + return None # the child closed it + pending += chunk + *lines, pending = pending.split(b"\n") + for line in lines: + if line.startswith(b"http://"): + return line.strip().decode() + return None + + def _launch(corpus: Path, state: Path, run_dir: Path) -> tuple[subprocess.Popen, str]: """`generate-and-open --local` in the BACKGROUND, and the URL it serves.""" child = subprocess.Popen( @@ -149,22 +177,40 @@ def _launch(corpus: Path, state: Path, run_dir: Path) -> tuple[subprocess.Popen, "--run-dir", str(run_dir), "--no-open", "--no-validate", "--port", "0"], env=_clean_env(**{STATE: str(state)}), cwd=ROOT, - stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) - deadline = time.monotonic() + 90 - url = None - while time.monotonic() < deadline and url is None: - line = child.stdout.readline() - if not line: - break - if line.startswith("http://"): - url = line.strip() + stdout=subprocess.PIPE, stderr=subprocess.PIPE) + url = _first_url(child, 90) if url is None: child.kill() - out, err = child.communicate(timeout=30) - raise AssertionError(f"the entry point never served: {err[-2000:]}") + _out, err = child.communicate(timeout=30) + raise AssertionError( + f"the entry point never served: {err.decode(errors='replace')[-2000:]}") return child, url +def test_the_launch_helper_is_bounded_by_its_deadline_not_by_the_child() -> None: + """A child that is alive and silent does not hold the helper past its + deadline; one that prints the URL is read, however the pieces arrive.""" + silent = subprocess.Popen([sys.executable, "-c", "import time; time.sleep(60)"], + stdout=subprocess.PIPE) + try: + began = time.monotonic() + assert _first_url(silent, 1.0) is None + assert time.monotonic() - began < 10 + finally: + silent.kill() + silent.communicate(timeout=10) + talker = subprocess.Popen( + [sys.executable, "-c", "import sys, time; sys.stdout.write(' serving '); " + "sys.stdout.flush(); time.sleep(0.2); print('x'); " + "print('http://127.0.0.1:1/index.html', flush=True); time.sleep(60)"], + stdout=subprocess.PIPE) + try: + assert _first_url(talker, 30) == "http://127.0.0.1:1/index.html" + finally: + talker.kill() + talker.communicate(timeout=10) + + def _status(state: Path) -> tuple[int, dict]: """`OPENDOX_INSTALL_MODE=local opendox-runtime runtime status`, a SECOND process.""" done = subprocess.run( @@ -325,6 +371,34 @@ def test_a_second_entry_point_on_the_same_state_dir_is_refused( again.stop() +@pytest.mark.skipif(not Path("/proc/self").exists(), reason="asks /proc") +def test_a_stale_lock_naming_a_recycled_pid_does_not_hold_the_bundle( + state_dir: Path) -> None: + """A server that did not stop cleanly leaves its `postmaster.pid`, and the + kernel gives its pid to something else, here a process with `postgres` in + its argv. `status` does not report that process, and the next start + starts, with the proven-stale lock removed (Copilot review of + openDox-code#69).""" + settings = config.load_settings({MODE: "local", STATE: str(state_dir)}) + bundle_mod.BundledServer(settings).start().stop() # a real cluster + bundle = config.DatabaseBundle(state_dir) + decoy = subprocess.Popen( + [sys.executable, "-c", "import time; time.sleep(60)", "postgres"], + cwd=bundle.data_dir, stdout=subprocess.DEVNULL) + try: + (bundle.data_dir / "postmaster.pid").write_text( + f"{decoy.pid}\n{bundle.data_dir}\n", encoding="utf-8") + assert bundle_mod.report(bundle)["pid"] is None + server = bundle_mod.BundledServer(settings).start() + try: + assert bundle_mod.running_pid(bundle) == server.report()["pid"] + finally: + server.stop() + finally: + decoy.kill() + decoy.wait(timeout=10) + + def test_migrate_under_the_local_mode_uses_the_bundle_and_refuses_a_dsn( state_dir: Path) -> None: """`runtime migrate` is part of the same install: it reads the bundle's diff --git a/tests_runtime/test_local_lifecycle.py b/tests_runtime/test_local_lifecycle.py new file mode 100644 index 00000000..334deb49 --- /dev/null +++ b/tests_runtime/test_local_lifecycle.py @@ -0,0 +1,489 @@ +"""The LOCAL install's lifecycle, hardened (plan 034 T072; Copilot review of +openDox-code#69). + +Hermetic: nothing here starts a real PostgreSQL server. The bundle's binaries +are stood in by small scripts where a phase must fail, and the entry point's +bundle is stood in where the case is the entry point's own handling. The real +server's cases are `test_bundled_postgres.py`'s. + + * which migrations a local install runs: this installation's own, never the + working directory's, and never another installation's found by name; + * a stale `postmaster.pid` is believed only for this data directory's + postmaster, asked of the kernel; + * `initdb` never leaves a half-built data directory; + * every phase of a start fails as the one named refusal, never a traceback; + * an interrupt anywhere in the local lifecycle is a clean stop. +""" + +from __future__ import annotations + +import os +import shutil +import signal +import subprocess +import sys +import tempfile +import time +from pathlib import Path + +import pytest + +from opendox import cli as cli_mod +from opendox.runtime import bundle as bundle_mod +from opendox.runtime import config +from opendox.runtime.config import PREFIX + +ROOT = Path(__file__).resolve().parents[1] +MODE = PREFIX + "INSTALL_MODE" +STATE = PREFIX + "STATE_DIR" +MIGRATIONS = PREFIX + "MIGRATIONS_DIR" + + +@pytest.fixture() +def scrubbed(monkeypatch: pytest.MonkeyPatch) -> pytest.MonkeyPatch: + for name in config.SETTING_NAMES: + monkeypatch.delenv(name, raising=False) + return monkeypatch + + +@pytest.fixture() +def short_state(): + """A state directory with a SHORT path (the socket's path is bounded).""" + path = Path(tempfile.mkdtemp(prefix="odx-l-", dir="/tmp")) + yield path + for directory in path.rglob("*"): + if directory.is_dir(): + directory.chmod(0o700) + shutil.rmtree(path, ignore_errors=True) + + +def _local(state: Path) -> config.RuntimeSettings: + return config.load_settings({MODE: "local", STATE: str(state)}) + + +# -- which migrations a local install runs ------------------------------------ + + +@pytest.fixture() +def hostile_cwd(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Path: + """A working directory whose `migrations/` holds the pinned `0001` AND SQL + of its own: what a checkout of somebody else's repository can hold.""" + here = tmp_path / "somebody-elses-checkout" + (here / "migrations").mkdir(parents=True) + for sql in (ROOT / "migrations").glob("0001*.sql"): + shutil.copy2(sql, here / "migrations" / sql.name) + (here / "migrations" / "0099_not_this_products.sql").write_text( + "create table stolen (secret text);\n", encoding="utf-8") + monkeypatch.chdir(here) + return here + + +def test_a_local_install_never_runs_the_working_directorys_migrations( + hostile_cwd: Path, short_state: Path) -> None: + settings = _local(short_state) + found = Path(settings.migrations_dir).resolve() + assert not found.is_relative_to(hostile_cwd.resolve()), found + assert found == (ROOT / "migrations").resolve(), found + migrate = config.load_migration_settings({MODE: "local", STATE: str(short_state)}) + assert Path(migrate.migrations_dir).resolve() == found + + +def test_a_hosted_install_keeps_its_working_directory_default( + hostile_cwd: Path) -> None: + """13.6: the hosted default is unchanged, and the deploy files set the + variable explicitly.""" + assert config.migrations_dir({}) == Path("migrations") + + +def test_an_explicit_migrations_dir_is_used_as_given_in_either_shape( + tmp_path: Path) -> None: + chosen = str(tmp_path / "chosen") + assert config.migrations_dir({MIGRATIONS: chosen}, local=True) == Path(chosen) + assert config.migrations_dir({MIGRATIONS: chosen}) == Path(chosen) + + +def test_a_local_installation_that_carries_none_is_refused_naming_the_setting( + monkeypatch: pytest.MonkeyPatch, hostile_cwd: Path) -> None: + monkeypatch.setattr(config, "installation_migrations_dir", lambda: None) + with pytest.raises(config.ConfigurationError) as caught: + config.migrations_dir({}, local=True) + assert MIGRATIONS in str(caught.value), caught.value + + +class _FakeFile: + """One `importlib.metadata` RECORD entry, located under `base`.""" + + def __init__(self, base: Path, relative: str) -> None: + self.base, self.relative = base, relative + self.name = Path(relative).name + self.parts = Path(relative).parts + + def __fspath__(self) -> str: + return self.relative + + def locate(self) -> Path: + return self.base / self.relative + + +class _FakeDistribution: + def __init__(self, base: Path) -> None: + site = "lib/python3/site-packages" + self.files = [ + _FakeFile(base, f"{site}/opendox/runtime/config.py"), + _FakeFile(base, "share/opendox/migrations/0001_older.sql"), + ] + + +def test_another_installations_record_is_not_this_ones(tmp_path: Path) -> None: + """A distribution found by NAME whose RECORD does not hold the running + module (an older wheel beside a checkout) is not this installation.""" + older = _FakeDistribution(tmp_path / "older-wheel") + assert config._distribution_migrations(Path(config.__file__), older) is None + + +def test_the_record_of_the_distribution_that_holds_the_module_is_used( + tmp_path: Path) -> None: + base = tmp_path / "wheel" + dist = _FakeDistribution(base) + module = dist.files[0].locate() + module.parent.mkdir(parents=True) + module.write_text("# stand-in\n", encoding="utf-8") + assert config._distribution_migrations(module, dist) == \ + (base / "share/opendox/migrations").resolve() + + +def test_the_source_tree_wins_over_a_wheel_found_by_name( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + """A checkout run with `src/` on the path, beside an older wheel: the + checkout's own migrations, not the wheel's (Copilot review of + openDox-code#69).""" + older = _FakeDistribution(tmp_path / "older-wheel") + monkeypatch.setattr(config.importlib.metadata, "distribution", + lambda name: older) + assert config.installation_migrations_dir() == ROOT / "migrations" + + +def test_the_source_tree_is_asked_before_any_record( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + """The documented ORDER, pinned: even a RECORD that does hold the running + module is asked only after the source tree it was imported from.""" + class _Holding: + files = [_FakeFile(ROOT / "src", "opendox/runtime/config.py"), + _FakeFile(tmp_path, "share/opendox/migrations/0001_other.sql")] + + assert config._distribution_migrations(Path(config.__file__), _Holding()) == \ + (tmp_path / "share/opendox/migrations").resolve() + monkeypatch.setattr(config.importlib.metadata, "distribution", + lambda name: _Holding()) + assert config.installation_migrations_dir() == ROOT / "migrations" + + +def test_a_package_outside_src_is_not_a_source_tree(tmp_path: Path) -> None: + """A `--target` install inside some other checkout: its parent has a + `pyproject.toml` and a `migrations/`, and it is still not this tree.""" + target = tmp_path / "checkout" / "vendor" / "opendox" / "runtime" + target.mkdir(parents=True) + (tmp_path / "checkout" / "pyproject.toml").write_text( + '[project]\nname = "opendox"\n', encoding="utf-8") + (tmp_path / "checkout" / "migrations").mkdir() + assert config._source_tree_migrations(target / "config.py") is None + + +# -- a stale postmaster.pid --------------------------------------------------- + + +def _decoy(executable: str, cwd: Path, *argv: str) -> subprocess.Popen: + return subprocess.Popen([executable, *argv], cwd=cwd, + stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + + +def _lock(bundle: config.DatabaseBundle, pid: int) -> None: + bundle.data_dir.mkdir(parents=True, exist_ok=True) + (bundle.data_dir / "postmaster.pid").write_text( + f"{pid}\n{bundle.data_dir}\n", encoding="utf-8") + + +@pytest.mark.skipif(not Path("/proc/self").exists(), reason="asks /proc") +def test_a_recycled_pid_is_not_believed_even_with_postgres_in_its_argv( + short_state: Path) -> None: + """Even IN the data directory, and with `postgres -D ` in its + argv: the executable is not `postgres`, so it is not this server.""" + bundle = config.DatabaseBundle(short_state) + bundle.data_dir.mkdir(parents=True) + decoy = _decoy(sys.executable, bundle.data_dir, "-c", + "import time; time.sleep(60)", "postgres", "-D", + str(bundle.data_dir)) + try: + _lock(bundle, decoy.pid) + assert bundle_mod.running_pid(bundle) is None + assert bundle_mod.report(bundle)["pid"] is None + # and the start is not refused over it: the stale lock is removed + bundle_mod._remove_a_proven_stale_lock(bundle) + assert not (bundle.data_dir / "postmaster.pid").exists() + finally: + decoy.kill() + decoy.wait(timeout=10) + + +@pytest.mark.skipif(not Path("/proc/self").exists(), reason="asks /proc") +def test_a_postgres_serving_another_directory_is_not_this_server( + short_state: Path, tmp_path: Path) -> None: + """The exact pair: an executable NAMED `postgres` is not enough, it must + be running in THIS data directory. And the positive control: the same + decoy, run in the data directory, is believed.""" + sleeper = shutil.which("sleep") + assert sleeper + named = tmp_path / "bin" / "postgres" + named.parent.mkdir() + shutil.copy2(sleeper, named) + bundle = config.DatabaseBundle(short_state) + bundle.data_dir.mkdir(parents=True) + elsewhere = tmp_path / "another-data-dir" + elsewhere.mkdir() + for cwd, believed in ((elsewhere, False), (bundle.data_dir, True)): + decoy = _decoy(str(named), cwd, "60") + try: + _lock(bundle, decoy.pid) + deadline = time.monotonic() + 10 + while (Path(os.readlink(f"/proc/{decoy.pid}/exe")).name != "postgres" + and time.monotonic() < deadline): + time.sleep(0.02) # the exec is complete + got = bundle_mod.running_pid(bundle) + assert got == (decoy.pid if believed else None), (cwd, got) + finally: + decoy.kill() + decoy.wait(timeout=10) + + +def test_another_users_process_is_not_this_server(short_state: Path) -> None: + """A pid the kernel will not let this user signal belongs to another user, + and this bundle's server runs as the owner of its 0700 data directory. + Pid 1 is another user's on an ordinary host and in CI. Where the suite + runs as pid 1's owner, the pid is still not a postgres in this directory. + Either way it is not believed.""" + bundle = config.DatabaseBundle(short_state) + _lock(bundle, 1) + assert bundle_mod.running_pid(bundle) is None + + +# -- the two refusal classes, each with its own reason -------------------------- + + +def test_a_dsn_beside_local_is_refused_for_the_database_not_a_broker() -> None: + with pytest.raises(config.ConfigurationError) as caught: + config.load_settings({MODE: "local", + PREFIX + "DATABASE_URL": "postgresql://s@h.invalid/x"}) + message = str(caught.value) + assert PREFIX + "DATABASE_URL" in message and "bundles" in message, message + assert "broker" not in message and "authentication" not in message, message + + +def test_a_broker_setting_beside_local_is_refused_for_the_broker() -> None: + with pytest.raises(config.ConfigurationError) as caught: + config.load_settings({MODE: "local", + PREFIX + "OIDC_AUDIENCE": "fixture"}) + message = str(caught.value) + assert PREFIX + "OIDC_AUDIENCE" in message, message + assert "broker" in message and "authentication" in message, message + assert "bundles" not in message, message + + +def test_both_classes_together_name_both_reasons() -> None: + with pytest.raises(config.ConfigurationError) as caught: + config.load_settings({ + MODE: "local", PREFIX + "OIDC_ISSUER": "https://i.invalid/r/x", + PREFIX + "MIGRATION_DATABASE_URL": "postgresql://m:hunter2@h.invalid/x"}) + message = str(caught.value) + for fragment in (PREFIX + "OIDC_ISSUER", PREFIX + "MIGRATION_DATABASE_URL", + "authentication", "bundles"): + assert fragment in message, (fragment, message) + assert "hunter2" not in message + + +# -- initdb, and every phase of a start --------------------------------------- + + +def _binaries(tmp_path: Path, *, initdb: str, postgres: str = "exit 0") -> Path: + """Stand-in `initdb` and `postgres` scripts, executable.""" + directory = tmp_path / "fake-bin" + directory.mkdir() + for name, body in (("initdb", initdb), ("postgres", postgres)): + script = directory / name + script.write_text(body if body.startswith("#!") else + f"#!/bin/sh\n{body}\n", encoding="utf-8") + script.chmod(0o755) + return directory + + +def _server(monkeypatch, tmp_path: Path, state: Path, **scripts) -> bundle_mod.BundledServer: + binaries = _binaries(tmp_path, **scripts) + monkeypatch.setattr(bundle_mod, "server_binaries", lambda: binaries) + return bundle_mod.BundledServer(_local(state)) + + +def _target_of_initdb() -> str: + """Shell: the directory `initdb -D ` was told to initialize.""" + return 'while [ "$1" != "-D" ]; do shift; done; T="$2"' + + +def test_an_initdb_that_dies_midway_leaves_no_data_directory( + monkeypatch, tmp_path: Path, short_state: Path) -> None: + """The half-built cluster: `PG_VERSION` written, then the run fails. The + data directory must not exist afterwards, so the next start initializes + again instead of launching the remains (Copilot review of #69).""" + server = _server(monkeypatch, tmp_path, short_state, initdb=( + f'{_target_of_initdb()}\necho 16 > "$T/PG_VERSION"\n' + 'mkdir -p "$T/base"\necho "initdb: killed" >&2\nexit 1')) + with pytest.raises(bundle_mod.BundleRefused) as caught: + server.start() + assert "initdb" in str(caught.value) + data = server.bundle.data_dir + assert not data.exists(), sorted(p.name for p in data.iterdir()) + leftovers = list(data.parent.glob(f"{bundle_mod.BundledServer.ATTEMPT_PREFIX}*")) + assert leftovers == [], leftovers + + +def test_an_abandoned_attempt_is_removed_and_a_live_ones_is_kept( + monkeypatch, tmp_path: Path, short_state: Path) -> None: + server = _server(monkeypatch, tmp_path, short_state, initdb="exit 1") + parent = server.bundle.data_dir.parent + parent.mkdir(parents=True) + dead = subprocess.Popen(["true"]) + dead.wait(timeout=10) + abandoned = parent / f"{server.ATTEMPT_PREFIX}{dead.pid}-x" + live = parent / f"{server.ATTEMPT_PREFIX}{os.getpid()}-y" + for attempt in (abandoned, live): + attempt.mkdir() + (attempt / "PG_VERSION").write_text("16\n", encoding="utf-8") + with pytest.raises(bundle_mod.BundleRefused): + server.start() + assert not abandoned.exists() + assert live.exists(), "a concurrent start's attempt was removed" + + +def test_a_data_directory_that_is_not_a_cluster_is_refused_and_left_alone( + monkeypatch, tmp_path: Path, short_state: Path) -> None: + server = _server(monkeypatch, tmp_path, short_state, initdb="exit 0") + data = server.bundle.data_dir + data.mkdir(parents=True) + (data / "somebody-elses-file").write_text("keep me\n", encoding="utf-8") + with pytest.raises(bundle_mod.BundleRefused) as caught: + server.start() + assert str(data) in str(caught.value) and STATE in str(caught.value) + assert (data / "somebody-elses-file").read_text(encoding="utf-8") == "keep me\n" + + +def test_an_initdb_that_hangs_is_the_named_refusal_not_a_traceback( + monkeypatch, tmp_path: Path, short_state: Path) -> None: + monkeypatch.setattr(bundle_mod, "START_TIMEOUT_SECONDS", 0.5) + server = _server(monkeypatch, tmp_path, short_state, initdb="exec sleep 30") + with pytest.raises(bundle_mod.BundleRefused) as caught: + server.start() + assert "initializing its data directory: TimeoutExpired" in str(caught.value) + assert not server.bundle.data_dir.exists() + + +def test_a_launch_that_cannot_exec_is_the_named_refusal( + monkeypatch, tmp_path: Path, short_state: Path) -> None: + server = _server(monkeypatch, tmp_path, short_state, + initdb=f'{_target_of_initdb()}\necho 16 > "$T/PG_VERSION"', + postgres="#!/nonexistent/interpreter\n") + with pytest.raises(bundle_mod.BundleRefused) as caught: + server.start() + assert "launching it: FileNotFoundError" in str(caught.value), caught.value + assert server.process is None + + +def test_directories_it_cannot_make_are_the_named_refusal( + monkeypatch, tmp_path: Path, short_state: Path) -> None: + if hasattr(os, "geteuid") and os.geteuid() == 0: # pragma: no cover + pytest.skip("root ignores directory modes") + locked = short_state / "locked" + locked.mkdir(mode=0o500) + binaries = _binaries(tmp_path, initdb="exit 0") + monkeypatch.setattr(bundle_mod, "server_binaries", lambda: binaries) + server = bundle_mod.BundledServer(_local(locked / "state")) + with pytest.raises(bundle_mod.BundleRefused) as caught: + server.start() + assert "preparing its directories: PermissionError" in str(caught.value) + + +# -- an interrupt anywhere in the local lifecycle ------------------------------- + + +@pytest.fixture() +def local_entry(scrubbed, monkeypatch, tmp_path: Path): + """`cmd_generate_and_open --local` with its bundle stood in. The corpus + checks are not under test here and pass whatever the root.""" + scrubbed.setenv(MODE, "local") + scrubbed.setenv(STATE, "/tmp/odx-never-created") + monkeypatch.setattr(cli_mod, "_refuse_non_corpus_repo_root", lambda args: None) + monkeypatch.setattr(cli_mod, "_refuse_malformed_generated_at", lambda args: None) + stopped: list[int] = [] + behaviour: dict = {} + + class _StandIn: + applied: list = [] + + def __init__(self, settings) -> None: + pass + + def start(self): + behaviour.get("start", lambda: None)() + return self + + def stop(self) -> None: + stopped.append(1) + + def report(self) -> dict: + return {"data_dir": None, "socket_dir": "(stood in)", "pid": None} + + monkeypatch.setattr(cli_mod.bundle_mod, "BundledServer", _StandIn) + args = cli_mod.build_parser().parse_args( + ["generate-and-open", config.LOCAL_FLAG, "--repo-root", str(tmp_path), + "--repository", "fixture", "--no-open", "--no-serve"]) + return args, behaviour, stopped + + +def _entry(args) -> int: + """The entry point, with an ESCAPING interrupt turned into a failure: one + that reached pytest would abort the whole session, not fail this case.""" + try: + return cli_mod.cmd_generate_and_open(args, opener=lambda url: None) + except KeyboardInterrupt as escaped: + raise AssertionError( + f"the interrupt escaped the entry point ({type(escaped).__name__}), " + "which is a traceback and not a clean stop") from None + + +def _sigterm_to_self() -> None: + os.kill(os.getpid(), signal.SIGTERM) + for _ in range(500): # the handler runs between two bytecodes + time.sleep(0.01) + raise AssertionError("SIGTERM was not delivered") # pragma: no cover + + +def test_a_sigterm_while_the_bundle_starts_is_a_clean_stop( + local_entry, capsys) -> None: + args, behaviour, stopped = local_entry + before = signal.getsignal(signal.SIGTERM) + behaviour["start"] = _sigterm_to_self + rc = _entry(args) + assert rc == 128 + signal.SIGTERM + assert "interrupted (SIGTERM)" in capsys.readouterr().err + assert stopped == [1], "the bundle was not stopped" + assert signal.getsignal(signal.SIGTERM) == before, "the handler was not restored" + + +def test_a_ctrl_c_while_the_snapshot_is_generated_is_a_clean_stop( + local_entry, monkeypatch, capsys) -> None: + args, _behaviour, stopped = local_entry + + def _interrupted(args, *, opener): + raise KeyboardInterrupt + + monkeypatch.setattr(cli_mod, "_generate_and_open", _interrupted) + rc = _entry(args) + assert rc == 128 + signal.SIGINT + assert "interrupted (SIGINT)" in capsys.readouterr().err + assert stopped == [1] From 96b2699fd511b53b0a4b8fd89c200ac35c711490 Mon Sep 17 00:00:00 2001 From: Brett Heap <1513478+brettheap@users.noreply.github.com> Date: Wed, 30 Sep 2026 15:48:41 +0000 Subject: [PATCH 13/39] Fix round: an unprovable pid is not believed, and the state dir refuses by name (Copilot review) Copilot's review at ac61596a opened two more threads, both real. - r4139938402: the old fallback believed any pid it could not inspect. So a process that exited between the signal check and the /proc read, or any pid on a platform without /proc, counted as the server. Round 4 already treated a vanished process as gone on the /proc path. This round makes the rule total. `running_pid` believes a pid only when the kernel proves it is this data directory's postmaster. Where nothing can be asked (no /proc: macOS, the BSDs), it believes nothing, and this module does not refuse a start over it. PostgreSQL's own interlocks, the lock file's live-pid check and the shared-memory check, still refuse a second postmaster, so this never yields two servers, and never a refusal over a process that is not one. A lock that cannot be proven stale is left for PostgreSQL to judge. The price on such a platform is a `status` with no pid. That is recorded, not hidden: the standard library has no portable way to ask, and a third-party module here would be an undeclared runtime dependency (test_consumer_reach). Measured before: round 4's source with no /proc, and a python decoy in the data dir, reported the decoy's pid. ac61596's source reported a pid that had already exited. - r4139938444: `Path.expanduser()` raises RuntimeError for an unknown `~user`, and `Path.home()` does the same where there is no home. Both now refuse by name, as ConfigurationError naming OPENDOX_STATE_DIR. A hosted install still never reads the setting and is not refused over it (13.6). Five new cases fail against 28bdccd's source and pass here. Five mutants of the fixes are killed. Full suite: 2584 selected, 2573 passed, 11 skipped, 0 failed. Lane: openxfactory-4 (openXfactory-4-openDox_extraction) Arc: neutral-product-standalone-operability Co-Authored-By: Claude Opus 5.5 (1M context) --- src/opendox/runtime/bundle.py | 68 +++++++++++++++++------ src/opendox/runtime/config.py | 24 ++++++-- tests_runtime/test_local_lifecycle.py | 80 +++++++++++++++++++++++++-- 3 files changed, 144 insertions(+), 28 deletions(-) diff --git a/src/opendox/runtime/bundle.py b/src/opendox/runtime/bundle.py index 633dc3a3..9014332e 100644 --- a/src/opendox/runtime/bundle.py +++ b/src/opendox/runtime/bundle.py @@ -143,29 +143,54 @@ def _lock_file_pid(bundle: DatabaseBundle) -> int | None: return None +#: Where the kernel answers what a pid is, on Linux. A module constant so a +#: case can take it away and exercise a platform without it. +PROC = Path("/proc") + + +def _identity(pid: int) -> tuple[str, str] | None: + """`(executable, working directory)` of `pid`, from the kernel's `/proc`. + + `None` where the process is gone or is not this user's to inspect. A + process that exits between the lock file's read and this one is GONE, + never proof of anything (Copilot review of openDox-code#69). Raises + `LookupError` where there is no `/proc` at all (macOS, the BSDs): the + standard library has no portable way to ask, and `running_pid` then + believes nothing it cannot prove. + """ + if not PROC.joinpath("self").exists(): + raise LookupError("no /proc to ask") + try: + return (os.readlink(PROC / str(pid) / "exe"), + os.readlink(PROC / str(pid) / "cwd")) + except OSError: # gone, or another user's: not inspectable + return None + + def _serves(pid: int, bundle: DatabaseBundle) -> bool | None: """Whether `pid` is the postmaster of `bundle`'s data directory. - Asked of the KERNEL, as the pair this module launches: an executable named - `postgres` whose working directory IS the data directory. The postmaster - changes into its data directory at startup, and neither of the two can be - rewritten by its process title, so a recycled pid given to anything else, - even another `postgres` serving another directory, is not it (Copilot - review of openDox-code#69). - - `False` also when the kernel will not say: another user's process cannot - be this bundle's server, because the server runs as the owner of a 0700 - data directory, the user this runs as. `None` only where there is no - `/proc` to ask at all. + Asked of the platform, as the pair this module launches: an executable + named `postgres` whose working directory IS the data directory. The + postmaster changes into its data directory at startup, and neither of the + two can be rewritten by its process title. So a recycled pid given to + anything else is not it, even another `postgres` serving another directory + (Copilot review of openDox-code#69). + + `False` also for a process that is gone, or that the platform will not + describe: another user's process cannot be this bundle's server, because + the server runs as the owner of a 0700 data directory, which is the user + this runs as. `None` only where nothing can be asked at all. """ - if not Path("/proc/self").exists(): - return None try: - executable = os.readlink(f"/proc/{pid}/exe") - cwd = os.readlink(f"/proc/{pid}/cwd") - except OSError: + identity = _identity(pid) + except LookupError: + return None + if identity is None: return False - if Path(executable.removesuffix(" (deleted)")).name != "postgres": + executable, cwd = identity + if Path(executable.removesuffix(" (deleted)")).name not in {"postgres", + "postgres.exe"}: return False try: return Path(cwd).resolve() == bundle.data_dir.resolve() @@ -189,7 +214,14 @@ def running_pid(bundle: DatabaseBundle) -> int | None: except (ProcessLookupError, PermissionError): # gone; or alive and another user's, which cannot be this server return None - return pid if _serves(pid, bundle) is not False else None + # BELIEVED ONLY WHEN PROVEN. Where nothing can say what the pid is (no + # `/proc`), it is not reported as this server, and this module does not + # refuse a start over it. PostgreSQL's own interlocks, the lock file's + # live-pid check and the shared-memory check, still refuse a second + # postmaster on one data directory. So an unverifiable pid never yields + # two servers, and never a refusal over a process that is not one. The + # price on such a platform is a `status` that reports no pid. + return pid if _serves(pid, bundle) is True else None def _remove_a_proven_stale_lock(bundle: DatabaseBundle) -> None: diff --git a/src/opendox/runtime/config.py b/src/opendox/runtime/config.py index ce84233b..c9d7b73b 100644 --- a/src/opendox/runtime/config.py +++ b/src/opendox/runtime/config.py @@ -1696,7 +1696,16 @@ def state_dir(env: Mapping[str, str] | None = None) -> Path: setting = PREFIX + "STATE_DIR" raw = env.get(setting, "").strip() if raw: - path = Path(raw).expanduser() + try: + path = Path(raw).expanduser() + except RuntimeError: + # `~nosuchuser/...`: `expanduser` raises rather than answering, and + # a setting is refused by name, never by a traceback (Copilot + # review of openDox-code#69). + raise ConfigurationError( + f"{setting} is {raw!r}, whose `~` names no user this system " + "knows, so it expands to no directory. Name the state " + "directory absolutely") from None if not path.is_absolute(): raise ConfigurationError( f"{setting} is {raw!r}, which is not an absolute path. The " @@ -1705,9 +1714,16 @@ def state_dir(env: Mapping[str, str] | None = None) -> Path: "the same socket, so the state directory is named absolutely") return path xdg = env.get("XDG_STATE_HOME", "").strip() - base = Path(xdg) if xdg and Path(xdg).is_absolute() else ( - Path.home() / ".local" / "state") - return base / "opendox" + if xdg and Path(xdg).is_absolute(): + return Path(xdg) / "opendox" + try: + home = Path.home() + except RuntimeError: + raise ConfigurationError( + f"{setting} is unset and this process has no home directory to " + "put the default under (no HOME, and no password entry for the " + f"user). Set {setting} to an absolute path") from None + return home / ".local" / "state" / "opendox" def database_bundle(state: Path) -> DatabaseBundle: diff --git a/tests_runtime/test_local_lifecycle.py b/tests_runtime/test_local_lifecycle.py index 334deb49..6abb41c7 100644 --- a/tests_runtime/test_local_lifecycle.py +++ b/tests_runtime/test_local_lifecycle.py @@ -203,7 +203,11 @@ def _lock(bundle: config.DatabaseBundle, pid: int) -> None: f"{pid}\n{bundle.data_dir}\n", encoding="utf-8") -@pytest.mark.skipif(not Path("/proc/self").exists(), reason="asks /proc") +needs_proc = pytest.mark.skipif(not bundle_mod.PROC.joinpath("self").exists(), + reason="asks /proc") + + +@needs_proc def test_a_recycled_pid_is_not_believed_even_with_postgres_in_its_argv( short_state: Path) -> None: """Even IN the data directory, and with `postgres -D ` in its @@ -225,7 +229,7 @@ def test_a_recycled_pid_is_not_believed_even_with_postgres_in_its_argv( decoy.wait(timeout=10) -@pytest.mark.skipif(not Path("/proc/self").exists(), reason="asks /proc") +@needs_proc def test_a_postgres_serving_another_directory_is_not_this_server( short_state: Path, tmp_path: Path) -> None: """The exact pair: an executable NAMED `postgres` is not enough, it must @@ -244,10 +248,6 @@ def test_a_postgres_serving_another_directory_is_not_this_server( decoy = _decoy(str(named), cwd, "60") try: _lock(bundle, decoy.pid) - deadline = time.monotonic() + 10 - while (Path(os.readlink(f"/proc/{decoy.pid}/exe")).name != "postgres" - and time.monotonic() < deadline): - time.sleep(0.02) # the exec is complete got = bundle_mod.running_pid(bundle) assert got == (decoy.pid if believed else None), (cwd, got) finally: @@ -255,6 +255,45 @@ def test_a_postgres_serving_another_directory_is_not_this_server( decoy.wait(timeout=10) +def test_a_pid_nothing_can_describe_is_not_believed_and_its_lock_is_kept( + short_state: Path, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + """No `/proc` (macOS, the BSDs): nothing can say what the pid is, so it + is not reported as this server, and its lock is NOT removed, since + nothing proved it stale. PostgreSQL's own interlock is left to judge a + start. The decoy would be believed if it could be described (the case + above, where it is).""" + monkeypatch.setattr(bundle_mod, "PROC", tmp_path / "no-proc") + sleeper = shutil.which("sleep") + assert sleeper + named = tmp_path / "bin" / "postgres" + named.parent.mkdir() + shutil.copy2(sleeper, named) + bundle = config.DatabaseBundle(short_state) + bundle.data_dir.mkdir(parents=True) + decoy = _decoy(str(named), bundle.data_dir, "60") + try: + _lock(bundle, decoy.pid) + assert bundle_mod.running_pid(bundle) is None + bundle_mod._remove_a_proven_stale_lock(bundle) + assert (bundle.data_dir / "postmaster.pid").exists() + finally: + decoy.kill() + decoy.wait(timeout=10) + + +@needs_proc +def test_a_process_that_exits_before_it_is_described_is_not_believed( + short_state: Path, monkeypatch: pytest.MonkeyPatch) -> None: + """Alive at the signal check, gone by the time it is described: that is + a stale lock, never proof of a server.""" + bundle = config.DatabaseBundle(short_state) + gone = subprocess.Popen(["true"]) + gone.wait(timeout=10) + _lock(bundle, gone.pid) + monkeypatch.setattr(bundle_mod.os, "kill", lambda pid, sig: None) + assert bundle_mod.running_pid(bundle) is None + + def test_another_users_process_is_not_this_server(short_state: Path) -> None: """A pid the kernel will not let this user signal belongs to another user, and this bundle's server runs as the owner of its 0700 data directory. @@ -266,6 +305,35 @@ def test_another_users_process_is_not_this_server(short_state: Path) -> None: assert bundle_mod.running_pid(bundle) is None +# -- the state directory, refused by name -------------------------------------- + + +def test_a_state_dir_naming_an_unknown_user_is_refused_by_name() -> None: + raw = "~no-such-user-odx-8f3a/state" + with pytest.raises(config.ConfigurationError) as caught: + config.state_dir({STATE: raw}) + assert STATE in str(caught.value), caught.value + with pytest.raises(config.ConfigurationError): + config.load_settings({MODE: "local", STATE: raw}) + # a HOSTED install never reads it, and is not refused over it (13.6) + hosted = config.load_settings({ + STATE: raw, PREFIX + "DATABASE_URL": "postgresql://s@127.0.0.1:1/x", + PREFIX + "OIDC_ISSUER": "https://issuer.example.invalid/realms/x", + PREFIX + "OIDC_AUDIENCE": "fixture"}) + assert hosted.install_mode == config.INSTALL_MODE_HOSTED + + +def test_no_home_for_the_default_state_dir_is_refused_by_name( + monkeypatch: pytest.MonkeyPatch) -> None: + def _no_home(): + raise RuntimeError("Could not determine home directory.") + + monkeypatch.setattr(config.Path, "home", staticmethod(_no_home)) + with pytest.raises(config.ConfigurationError) as caught: + config.state_dir({}) + assert STATE in str(caught.value), caught.value + + # -- the two refusal classes, each with its own reason -------------------------- From 026f00ea7ce8eda3dfc513e3cff848c9a08a472d Mon Sep 17 00:00:00 2001 From: Brett Heap <1513478+brettheap@users.noreply.github.com> Date: Wed, 30 Sep 2026 15:51:48 +0000 Subject: [PATCH 14/39] Fix round: the install-mode module says which of its cases is DB-backed (Copilot review) The module docstring called every case hermetic. Since the fourth fix round, one is not: `test_runtime_status_of_a_healthy_local_install_exits_zero` takes the suite's `postgres_dsn` and `database` fixtures, because a healthy local `status` exits 0 only against a database that answers. The docstring now names that case and says it is skipped without Postgres and fails under CI, like every DB-backed case. It says the rest stay hermetic. Docstring only: the module runs 41 passed. Lane: openxfactory-4 (openXfactory-4-openDox_extraction) Arc: neutral-product-standalone-operability Co-Authored-By: Claude Opus 5.5 (1M context) --- tests_runtime/test_install_mode.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/tests_runtime/test_install_mode.py b/tests_runtime/test_install_mode.py index 8fcc2a7b..24d44eb6 100644 --- a/tests_runtime/test_install_mode.py +++ b/tests_runtime/test_install_mode.py @@ -1,12 +1,20 @@ """`OPENDOX_INSTALL_MODE`: the local single-user install, and the hosted one it cannot be reached from by omission (plan 034 T070; #1144 13.4, 13.5, 13.6). -HERMETIC: standard library plus `opendox.runtime.config` and the runtime CLI, -both stdlib-only at import. No database is reached: every DSN below is a -well-formed PostgreSQL URI aimed at port 1 of the loopback, so a refusal here -is always a CONFIGURATION refusal, which is the whole of what 13.4-13.6 ask of +HERMETIC, WITH ONE EXCEPTION. Every case but one uses the standard library +plus `opendox.runtime.config` and the runtime CLI, both stdlib-only at import, +and reaches no database. Each DSN in those cases is a well-formed PostgreSQL +URI aimed at port 1 of the loopback, so a refusal there is always a +CONFIGURATION refusal, which is the whole of what 13.4-13.6 ask of `load_settings`. +The exception is `test_runtime_status_of_a_healthy_local_install_exits_zero`, +which is DB-BACKED (Copilot review of openDox-code#67). A healthy local +`status` exits 0 only against a database that answers, so that case takes the +suite's `postgres_dsn` and `database` fixtures (`tests_runtime/conftest.py`). +Like every DB-backed case, it is skipped where there is no Postgres, and it +fails under CI. + WHAT IS RULED AND WHAT IS READ, so a reviewer can tell them apart: * RULED: the selector, its two values and its hosted default (#1144 13.4); From a0fb7c8d4320108a408f6ece93a09adc69c4e408 Mon Sep 17 00:00:00 2001 From: Brett Heap <1513478+brettheap@users.noreply.github.com> Date: Wed, 30 Sep 2026 15:58:55 +0000 Subject: [PATCH 15/39] Fix round: pyproject's packaging note names the resolver that exists (Copilot review) The data-files note pointed readers at `opendox.runtime.config.packaged_migrations_dir`, which fix round 4 replaced with `installation_migrations_dir`, the source tree first and then the RECORD of the distribution that holds the running module. The note now names that function and says what it asks. The packaging case now also checks that every `opendox.runtime.config.` pyproject.toml names exists, so a stale pointer cannot come back. Against 4aed6278's pyproject.toml it fails, naming `packaged_migrations_dir`. Here it passes. Full suite: 2584 selected, 2573 passed, 11 skipped, 0 failed. Lane: openxfactory-4 (openXfactory-4-openDox_extraction) Arc: neutral-product-standalone-operability Co-Authored-By: Claude Opus 5.5 (1M context) --- pyproject.toml | 7 ++++--- tests_runtime/test_bundled_postgres.py | 5 +++++ 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 7be64df0..15f8ca53 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -246,9 +246,10 @@ where = ["src"] # `COPY migrations ./migrations`, run from `/app`) and every checkout run finds # it; this maps the same files into the wheel's data directory, so an install # run OUTSIDE a checkout still has the migrations its bundled server applies. -# `opendox.runtime.config.packaged_migrations_dir` finds them through the -# installed distribution's own record, and the canonical digest gate is what -# proves the copy found is the pinned one. A data directory and not package +# `opendox.runtime.config.installation_migrations_dir` finds them through the +# own record of the installed distribution that holds the running module +# (after the source tree, for a checkout), and the canonical digest gate is +# what proves the copy found is the pinned one. A data directory and not package # data because package data must live inside the package, and the one tree is # not moved. [tool.setuptools.data-files] diff --git a/tests_runtime/test_bundled_postgres.py b/tests_runtime/test_bundled_postgres.py index acf6e369..1062c809 100644 --- a/tests_runtime/test_bundled_postgres.py +++ b/tests_runtime/test_bundled_postgres.py @@ -245,6 +245,11 @@ def test_the_local_extra_carries_the_runtime_and_the_server_and_test_joins_it( assert re.search(r"(?m)^pgserver==", lock), "the lock does not pin pgserver" files = project["tool"]["setuptools"]["data-files"] assert files == {"share/opendox/migrations": ["migrations/*.sql"]} + # and the packaging notes name functions that exist (Copilot review of #69) + for name in re.findall(r"opendox\.runtime\.config\.(\w+)", + (ROOT / "pyproject.toml").read_text()): + assert hasattr(config, name), ( + f"pyproject.toml points readers at config.{name}, which does not exist") # -- the layout, before anything starts ---------------------------------------- From 0f77d5c1a6dc68ea8314b2b2177c90f1147f569c Mon Sep 17 00:00:00 2001 From: Brett Heap <1513478+brettheap@users.noreply.github.com> Date: Wed, 30 Sep 2026 16:33:22 +0000 Subject: [PATCH 16/39] Fix round: libpq's environment, the socket's path and a relative HOME (Copilot and SonarCloud review) Copilot's review at a0fb7c8d opened three threads, and SonarCloud raised a reliability finding. All four are fixed here. - PG* defaults (r4146787926). libpq fills every parameter a DSN leaves unset from the environment. PGHOSTADDR outranks the socket `host` and sends the connection to TCP, PGSERVICE fills parameters from a service file, and PGOPTIONS sets the session's parameters. No DSN can name every parameter, and an explicitly empty `service` is itself an error. So `bundle.isolated_from_libpq_environment` lifts every PG* variable out of os.environ for the duration and puts it back afterwards. It wraps `generate-and-open --local`'s whole lifecycle and the runtime CLI's verbs under `local`. A hosted install's libpq is untouched (13.6). With PGHOSTADDR=192.0.2.1, PGSERVICE=no-such-service and PGOPTIONS=-c search_path=nowhere set, the real entry point still starts, migrates and serves its own server, and `runtime status` still finds it. - The socket's path (r4146787852). Before the socket directory is chmodded (a chmod follows a symlink), the resolved path is checked. The state dir, postgres/ and run/ must be real directories owned by this user and writable by no one else. Every ancestor must be owned by this user or by root, and must be sticky if every user can write it, or if a group other than this user's own can write it. Anything else is refused by name. - A relative HOME (r4146659876). It is refused for the default state directory, which would otherwise depend on the working directory. - SonarCloud S6466. server_binaries no longer indexes a list. It takes the first search location or none, and both refusal shapes have a case. Against a0fb7c8d's source, 8 of the new cases fail and the positive control passes. 11 mutants are killed. Full suite: 2595 selected, 2584 passed, 11 skipped, 0 failed. Lane: openxfactory-4 (openXfactory-4-openDox_extraction) Arc: neutral-product-standalone-operability Co-Authored-By: Claude Opus 5.5 (1M context) --- src/opendox/cli.py | 11 ++ src/opendox/runtime/bundle.py | 112 ++++++++++++++++++- src/opendox/runtime/cli.py | 26 ++++- src/opendox/runtime/config.py | 9 ++ tests_runtime/test_bundled_postgres.py | 36 ++++++- tests_runtime/test_local_lifecycle.py | 142 +++++++++++++++++++++++++ 6 files changed, 326 insertions(+), 10 deletions(-) diff --git a/src/opendox/cli.py b/src/opendox/cli.py index 5e0073ee..fd2da4e9 100644 --- a/src/opendox/cli.py +++ b/src/opendox/cli.py @@ -508,6 +508,17 @@ def cmd_generate_and_open(args: argparse.Namespace, *, opener=webbrowser.open) - _refuse_malformed_generated_at(args) server = bundle_mod.BundledServer(settings) args.database_bundle = server + # NO `PG*` DEFAULT REACHES THE BUNDLE'S CONNECTIONS while this process + # runs its database (Copilot review of openDox-code#69): see + # `bundle.isolated_from_libpq_environment`. + with bundle_mod.isolated_from_libpq_environment(): + return _run_the_local_lifecycle(args, server, opener=opener) + + +def _run_the_local_lifecycle(args: argparse.Namespace, server, *, opener) -> int: + """Start the bundled server, generate and serve, and stop it, however + this ends: a served run ended by Ctrl-C or SIGTERM, a `--no-serve` run, a + refusal, a failure, or an interrupt before anything was served.""" try: previous = signal.signal(signal.SIGTERM, _terminate_as_interrupt) except ValueError: # not the main thread: no handler to own diff --git a/src/opendox/runtime/bundle.py b/src/opendox/runtime/bundle.py index 9014332e..0994f446 100644 --- a/src/opendox/runtime/bundle.py +++ b/src/opendox/runtime/bundle.py @@ -59,15 +59,18 @@ from __future__ import annotations +import contextlib import ctypes import importlib.util import os import shutil import signal +import stat import subprocess import sys import tempfile import time +from collections.abc import Iterator from pathlib import Path from typing import Any @@ -115,14 +118,17 @@ def server_binaries() -> Path: object under the user's runtime directory as a side effect. """ spec = importlib.util.find_spec(SERVER_DISTRIBUTION) - locations = list(spec.submodule_search_locations or ()) if spec else [] - if not locations: + # THE FIRST LOCATION, OR NONE, read without an index, so no path reaches + # a subscript that could raise (SonarCloud S6466 on openDox-code#69). + location = next(iter(spec.submodule_search_locations or ()), None) \ + if spec else None + if location is None: raise BundleRefused( "the local install's PostgreSQL server is not installed: it " "arrives with the `local` extra, `pip install \"opendox[local]\"` " "(R1Q16 (iii)). A local install brings its own database and never " "borrows one") - binaries = Path(locations[0]) / "pginstall" / "bin" + binaries = Path(location) / "pginstall" / "bin" missing = [name for name in ("initdb", "postgres") if not os.access(binaries / name, os.X_OK)] if missing: @@ -263,6 +269,35 @@ def _child_environment() -> dict[str, str]: if not name.startswith("PG")} +@contextlib.contextmanager +def isolated_from_libpq_environment() -> Iterator[None]: + """Run a LOCAL install's client side with libpq's `PG*` defaults out of reach. + + libpq fills every connection parameter a DSN leaves unset from the + process environment. Some of those parameters move the connection + somewhere else. `PGHOSTADDR` outranks the DSN's socket `host` and sends + it to a TCP server. `PGSERVICE` fills parameters from a service file. + `PGOPTIONS` sets session parameters, a `search_path` among them. The + bundle's DSNs name their socket, port, user and database, but they + cannot name every parameter libpq has, and an explicitly empty `service` + is itself an error. So a local install's process reads NONE of them while + it runs its database: they are lifted out of `os.environ` for the + duration and put back afterwards (Copilot review of openDox-code#69). + `_child_environment` already does the same for `initdb` and the server. + + For the process's own entry points only: `generate-and-open --local` + around its whole lifecycle, and the runtime CLI's verbs under `local`. + Nothing else in those processes speaks libpq. + """ + lifted = {name: os.environ.pop(name) for name in + [name for name in os.environ if name.startswith("PG")]} + try: + yield + finally: + for name, value in lifted.items(): + os.environ.setdefault(name, value) + + def _die_with_parent(): """A `preexec_fn` that signals the server when its parent goes away (iv). @@ -291,6 +326,33 @@ def _preexec() -> None: # pragma: no cover - runs in the child return _preexec +def _unsafe_because(info: os.stat_result, *, uid: int, gid: int, + own: bool) -> str | None: + """Why one directory of the socket's path is unsafe, or `None`.""" + mode = info.st_mode + if stat.S_ISLNK(mode): + return "is a symbolic link" + if not stat.S_ISDIR(mode): + return "is not a directory" + if own: + if info.st_uid != uid: + return f"is owned by uid {info.st_uid}, not by this user" + if mode & 0o022: + return (f"is writable by {'every user' if mode & 0o002 else 'its group'}" + f" (mode {stat.S_IMODE(mode):o})") + return None + if info.st_uid not in (uid, 0): + return f"is owned by uid {info.st_uid}, neither this user nor root" + sticky = bool(mode & stat.S_ISVTX) + if mode & 0o002 and not sticky: + return (f"is writable by every user and is not sticky " + f"(mode {stat.S_IMODE(mode):o})") + if mode & 0o020 and info.st_gid != gid and not sticky: + return (f"is writable by group {info.st_gid}, which is not this " + f"user's own, and is not sticky (mode {stat.S_IMODE(mode):o})") + return None + + class BundledServer: """One local install's PostgreSQL server: started as this process's child. @@ -381,8 +443,47 @@ def _prepare_directories(self) -> None: for directory in (self.bundle.state_dir, self.bundle.data_dir.parent): directory.mkdir(mode=0o700, parents=True, exist_ok=True) self.bundle.socket_dir.mkdir(mode=0o700, parents=True, exist_ok=True) + # CHECKED BEFORE THE CHMOD, which follows a symbolic link: a `run` + # placed there as a link would otherwise have its TARGET re-moded. + self._refuse_an_unsafe_tree() os.chmod(self.bundle.socket_dir, 0o700) + def _refuse_an_unsafe_tree(self) -> None: + """The socket's whole path is this user's to change, or it is refused. + + `trust` makes reaching the socket the credential, so the 0700 on the + socket directory is worth only what the directories above it are + worth. A directory entry is controlled by its PARENT. A parent that + another user can write lets them rename `run` away, or put a symbolic + link in its place, after the mode is set (Copilot review of + openDox-code#69). So, over the RESOLVED path, which a user's own + symbolic link in `OPENDOX_STATE_DIR` may lead to: + + * this install's own tree, the state directory, `postgres/` and + `run/`, must be real directories, owned by this user and writable + by no one else; + * every directory above it must be owned by this user or by root. + One that every user can write must be sticky, as `/tmp` is, so + nobody can rename what is not theirs. One that its group can + write must be sticky too, unless the group is this user's own, + which is how a umask-002 system creates the user's directories. + """ + uid, gid = os.getuid(), os.getgid() + state = self.bundle.state_dir.resolve() + own = [state, state / self.bundle.socket_dir.parent.name, + state / self.bundle.socket_dir.parent.name / self.bundle.socket_dir.name] + for directory, mine in [(path, True) for path in own] + \ + [(path, False) for path in state.parents]: + info = os.lstat(directory) + reason = _unsafe_because(info, uid=uid, gid=gid, own=mine) + if reason is not None: + raise BundleRefused( + f"{directory} {reason}, so another user could replace the " + "bundled server's socket directory, and reaching that " + "socket is the only credential the server asks for. Use a " + f"state directory only this user can change ({PREFIX}" + "STATE_DIR)") + #: The prefix an initialization attempt's directory carries, beside the #: data directory, followed by the pid of the process making it. ATTEMPT_PREFIX = "data.initdb-" @@ -599,5 +700,6 @@ def __exit__(self, *exc: object) -> None: self.stop() -__all__ = ["BUNDLE_PORT", "BundleRefused", "BundledServer", "report", - "running_pid", "server_binaries"] +__all__ = ["BUNDLE_PORT", "BundleRefused", "BundledServer", + "isolated_from_libpq_environment", "report", "running_pid", + "server_binaries"] diff --git a/src/opendox/runtime/cli.py b/src/opendox/runtime/cli.py index d8b5fc20..b5d22207 100644 --- a/src/opendox/runtime/cli.py +++ b/src/opendox/runtime/cli.py @@ -80,6 +80,7 @@ import contextlib import json import logging +import os import re import stat import sys @@ -97,6 +98,7 @@ ConfigurationError, redacted_url, RuntimeSettings, + install_mode, load_migration_settings, load_settings, migration_database_url, @@ -1225,6 +1227,27 @@ def build_parser() -> argparse.ArgumentParser: return parser +def _isolated_when_local() -> contextlib.AbstractContextManager[None]: + """A LOCAL install's verbs run with libpq's `PG*` defaults out of reach. + + The bundle's DSNs name their socket, but libpq fills everything else from + the environment, and `PGHOSTADDR` alone would send `status` or `migrate` + to a TCP server instead (Copilot review of openDox-code#69; see + `bundle.isolated_from_libpq_environment`). A hosted install's operator + configures libpq as they please, as before (13.6). A selector that cannot + be read isolates nothing; the verb refuses it by name. + """ + try: + local = install_mode(os.environ) == INSTALL_MODE_LOCAL + except ConfigurationError: + local = False + if not local: + return contextlib.nullcontext() + from opendox.runtime import bundle as bundle_mod + + return bundle_mod.isolated_from_libpq_environment() + + def main(argv: list[str] | None = None) -> int: """Parse and dispatch, and NEVER let a traceback be the whole answer. @@ -1240,7 +1263,8 @@ def main(argv: list[str] | None = None) -> int: """ args = build_parser().parse_args(argv) try: - return int(args.func(args)) + with _isolated_when_local(): + return int(args.func(args)) except SystemExit: raise except Exception as exc: # noqa: BLE001 diff --git a/src/opendox/runtime/config.py b/src/opendox/runtime/config.py index c9d7b73b..56725995 100644 --- a/src/opendox/runtime/config.py +++ b/src/opendox/runtime/config.py @@ -1723,6 +1723,15 @@ def state_dir(env: Mapping[str, str] | None = None) -> Path: f"{setting} is unset and this process has no home directory to " "put the default under (no HOME, and no password entry for the " f"user). Set {setting} to an absolute path") from None + if not home.is_absolute(): + # `Path.home()` returns HOME as given, and a relative one would give + # the serving process and a `runtime status` run from another + # directory two different sockets (Copilot review of openDox-code#69). + raise ConfigurationError( + f"{setting} is unset and HOME is {str(home)!r}, which is not an " + "absolute path, so the default state directory would depend on " + f"the working directory. Set {setting} to an absolute path, or " + "HOME to one") return home / ".local" / "state" / "opendox" diff --git a/tests_runtime/test_bundled_postgres.py b/tests_runtime/test_bundled_postgres.py index 1062c809..56e1f844 100644 --- a/tests_runtime/test_bundled_postgres.py +++ b/tests_runtime/test_bundled_postgres.py @@ -169,14 +169,15 @@ def _first_url(child: subprocess.Popen, seconds: float) -> str | None: return None -def _launch(corpus: Path, state: Path, run_dir: Path) -> tuple[subprocess.Popen, str]: +def _launch(corpus: Path, state: Path, run_dir: Path, + **extra: str) -> tuple[subprocess.Popen, str]: """`generate-and-open --local` in the BACKGROUND, and the URL it serves.""" child = subprocess.Popen( [sys.executable, str(DRIVER), "generate-and-open", config.LOCAL_FLAG, "--repo-root", str(corpus), "--repository", "fixture", "--run-dir", str(run_dir), "--no-open", "--no-validate", "--port", "0"], - env=_clean_env(**{STATE: str(state)}), cwd=ROOT, + env=_clean_env(**{STATE: str(state)}, **extra), cwd=ROOT, stdout=subprocess.PIPE, stderr=subprocess.PIPE) url = _first_url(child, 90) if url is None: @@ -211,12 +212,12 @@ def test_the_launch_helper_is_bounded_by_its_deadline_not_by_the_child() -> None talker.communicate(timeout=10) -def _status(state: Path) -> tuple[int, dict]: +def _status(state: Path, **extra: str) -> tuple[int, dict]: """`OPENDOX_INSTALL_MODE=local opendox-runtime runtime status`, a SECOND process.""" done = subprocess.run( [sys.executable, "-m", "opendox.runtime.cli", "runtime", "status", "--probe-timeout", "10"], - env=_clean_env(**{MODE: "local", STATE: str(state)}), cwd=ROOT, + env=_clean_env(**{MODE: "local", STATE: str(state)}, **extra), cwd=ROOT, capture_output=True, text=True, timeout=60) return done.returncode, json.loads(done.stdout) @@ -343,6 +344,33 @@ def test_the_entry_point_owns_a_migrated_server_with_no_tcp_listener( "the data directory must survive a stop: it is the install's database" +#: libpq defaults that, READ, would move the bundle's connections: to an +#: unroutable TCP address (TEST-NET-1, RFC 5737), through a service that does +#: not exist, and into a schema that does not either. +HOSTILE_LIBPQ = {"PGHOSTADDR": "192.0.2.1", "PGSERVICE": "no-such-service-odx", + "PGOPTIONS": "-c search_path=nowhere"} + + +def test_libpq_defaults_in_the_environment_never_reach_the_bundle( + corpus: Path, state_dir: Path, tmp_path: Path) -> None: + """`PGHOSTADDR` outranks a DSN's socket `host`, `PGSERVICE` fills + parameters from a service file, and `PGOPTIONS` sets the session's + parameters. With all three set, the entry point still starts, migrates + and serves ITS OWN server, and `runtime status` in a second process still + finds it (Copilot review of openDox-code#69).""" + server, url = _launch(corpus, state_dir, tmp_path / "run", **HOSTILE_LIBPQ) + try: + code, status = _status(state_dir, **HOSTILE_LIBPQ) + assert status.get("database") == "reachable", status + assert status.get("applied_migrations") and \ + not status.get("pending_migrations"), status + assert code == 0 and status["ok"] is True, status + finally: + server.send_signal(signal.SIGTERM) + server.communicate(timeout=60) + assert server.returncode == 0, server.returncode + + @pytest.mark.skipif(not sys.platform.startswith("linux"), reason="PR_SET_PDEATHSIG is Linux's") def test_the_server_stops_even_when_the_entry_point_is_killed_outright( diff --git a/tests_runtime/test_local_lifecycle.py b/tests_runtime/test_local_lifecycle.py index 6abb41c7..5bec3e8b 100644 --- a/tests_runtime/test_local_lifecycle.py +++ b/tests_runtime/test_local_lifecycle.py @@ -20,6 +20,7 @@ import os import shutil import signal +import stat import subprocess import sys import tempfile @@ -334,6 +335,129 @@ def _no_home(): assert STATE in str(caught.value), caught.value +def test_a_relative_home_for_the_default_state_dir_is_refused_by_name( + monkeypatch: pytest.MonkeyPatch) -> None: + """`Path.home()` returns HOME as given; a relative one would put the + socket wherever each process happens to run (Copilot review of #69).""" + monkeypatch.setenv("HOME", "relative-home") + with pytest.raises(config.ConfigurationError) as caught: + config.state_dir({}) + assert STATE in str(caught.value) and "HOME" in str(caught.value) + # an absolute XDG_STATE_HOME still answers without HOME at all + assert config.state_dir({"XDG_STATE_HOME": "/srv/state"}) == \ + Path("/srv/state/opendox") + + +# -- libpq's environment, out of reach --------------------------------------- + + +def test_libpq_defaults_are_lifted_for_the_duration_and_put_back( + monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("PGHOSTADDR", "192.0.2.1") + monkeypatch.setenv("PGSERVICE", "no-such-service-odx") + monkeypatch.setenv("OPENDOX_NOT_LIBPQ", "kept") + with bundle_mod.isolated_from_libpq_environment(): + assert not [name for name in os.environ if name.startswith("PG")] + assert os.environ["OPENDOX_NOT_LIBPQ"] == "kept" + assert os.environ["PGHOSTADDR"] == "192.0.2.1" + assert os.environ["PGSERVICE"] == "no-such-service-odx" + + +def test_the_runtime_cli_isolates_only_a_local_install( + monkeypatch: pytest.MonkeyPatch, scrubbed) -> None: + from opendox.runtime import cli as runtime_cli + + seen: dict = {} + + def _verb(args) -> int: + seen["PGHOSTADDR"] = os.environ.get("PGHOSTADDR") + return 0 + + monkeypatch.setenv("PGHOSTADDR", "192.0.2.1") + parser = runtime_cli.build_parser() + monkeypatch.setattr(runtime_cli, "build_parser", lambda: parser) + real_parse = parser.parse_args + + def _parse(argv=None): + args = real_parse(argv) + args.func = _verb + return args + + monkeypatch.setattr(parser, "parse_args", _parse) + scrubbed.setenv(MODE, "local") + assert runtime_cli.main(["runtime", "status"]) == 0 + assert seen["PGHOSTADDR"] is None, "a local verb saw PGHOSTADDR" + scrubbed.setenv(MODE, "hosted") + assert runtime_cli.main(["runtime", "status"]) == 0 + assert seen["PGHOSTADDR"] == "192.0.2.1", "a hosted verb lost its libpq setting" + assert os.environ["PGHOSTADDR"] == "192.0.2.1" + + +# -- the socket's path, this user's to change ----------------------------------- + + +def _prepared(monkeypatch, tmp_path: Path, state: Path) -> bundle_mod.BundledServer: + binaries = _binaries(tmp_path, initdb='echo "initdb reached" >&2; exit 1') + monkeypatch.setattr(bundle_mod, "server_binaries", lambda: binaries) + return bundle_mod.BundledServer(_local(state)) + + +def test_a_state_dir_others_can_write_is_refused( + monkeypatch, tmp_path: Path, short_state: Path) -> None: + short_state.chmod(0o777) + server = _prepared(monkeypatch, tmp_path, short_state) + with pytest.raises(bundle_mod.BundleRefused) as caught: + server.start() + assert "writable by every user" in str(caught.value), caught.value + assert str(short_state) in str(caught.value) + + +@pytest.mark.parametrize("which", ["postgres", "run"]) +def test_a_symlink_inside_the_state_tree_is_refused_and_its_target_untouched( + monkeypatch, tmp_path: Path, short_state: Path, which: str) -> None: + target = tmp_path / "somewhere-else" + target.mkdir(mode=0o755) + target.chmod(0o755) + if which == "postgres": + (short_state / "postgres").symlink_to(target) + else: + (short_state / "postgres").mkdir(mode=0o700) + (short_state / "postgres" / "run").symlink_to(target) + server = _prepared(monkeypatch, tmp_path, short_state) + with pytest.raises(bundle_mod.BundleRefused) as caught: + server.start() + assert "symbolic link" in str(caught.value), caught.value + assert stat.S_IMODE(target.stat().st_mode) == 0o755, "the link's target was re-moded" + + +def test_an_ancestor_every_user_can_write_without_the_sticky_bit_is_refused( + monkeypatch, tmp_path: Path, short_state: Path) -> None: + open_dir = short_state / "open" + open_dir.mkdir() + open_dir.chmod(0o777) + server = _prepared(monkeypatch, tmp_path, open_dir / "state") + with pytest.raises(bundle_mod.BundleRefused) as caught: + server.start() + assert str(open_dir) in str(caught.value) and "not sticky" in str(caught.value) + + +def test_a_sticky_or_own_group_ancestor_is_accepted( + monkeypatch, tmp_path: Path, short_state: Path) -> None: + """The positive controls: `/tmp`'s shape (every user, sticky) and a + umask-002 system's shape (this user's own group). The start gets past + the tree and reaches `initdb`, which the stand-in fails on purpose.""" + sticky = short_state / "sticky" + sticky.mkdir() + sticky.chmod(0o1777) + group = sticky / "group" + group.mkdir() + group.chmod(0o775) + server = _prepared(monkeypatch, tmp_path, group / "state") + with pytest.raises(bundle_mod.BundleRefused) as caught: + server.start() + assert "initdb" in str(caught.value), caught.value + + # -- the two refusal classes, each with its own reason -------------------------- @@ -368,6 +492,24 @@ def test_both_classes_together_name_both_reasons() -> None: assert "hunter2" not in message +@pytest.mark.parametrize("found", ["absent", "no-locations"]) +def test_a_missing_server_package_is_the_named_refusal( + monkeypatch: pytest.MonkeyPatch, found: str) -> None: + """No `pgserver` at all, or a spec with no location: both are the one + refusal naming the `local` extra, never an `IndexError`.""" + import importlib.machinery + + spec = None + if found == "no-locations": + spec = importlib.machinery.ModuleSpec("pgserver", None, is_package=True) + spec.submodule_search_locations = [] + monkeypatch.setattr(bundle_mod.importlib.util, "find_spec", + lambda name: spec) + with pytest.raises(bundle_mod.BundleRefused) as caught: + bundle_mod.server_binaries() + assert 'opendox[local]' in str(caught.value), caught.value + + # -- initdb, and every phase of a start --------------------------------------- From 379fbb14539c9069dc1fd737403ef46f8d1ea950 Mon Sep 17 00:00:00 2001 From: Brett Heap <1513478+brettheap@users.noreply.github.com> Date: Wed, 30 Sep 2026 16:48:28 +0000 Subject: [PATCH 17/39] Fix round: the socket's path trusts no group and no link it cannot vouch for (Copilot review) Copilot's review at 0f77d5c1 opened two threads. Both were real. - r4147004990: a user's primary group can have other members, so a 0775 ancestor is not private. Every ancestor that anyone else can write, a group included, must now be sticky. The round-7 allowance for the user's own group is gone, and its positive control is now a refusal case. The sticky shape (/tmp) is still the control. - r4147005063: resolving the configured path before checking it discarded the path that was actually configured. A link on that path could be repointed afterwards, while the bundle kept using the unresolved paths. Now: - the ancestors of BOTH the configured path and the resolved one are checked; - every symbolic link on the configured path must be owned by this user or by root; - `..` is refused in OPENDOX_STATE_DIR and XDG_STATE_HOME (and in a derived HOME), so the configured components are the ones the kernel walks. A user's own link to a private directory is still accepted. Against 0f77d5c1's source, 5 of the new cases fail and the 4 controls pass. 5 mutants are killed. Full suite: 2601 selected, 2590 passed, 11 skipped, 0 failed. Lane: openxfactory-4 (openXfactory-4-openDox_extraction) Arc: neutral-product-standalone-operability Co-Authored-By: Claude Opus 5.5 (1M context) --- src/opendox/runtime/bundle.py | 94 ++++++++++++++++----------- src/opendox/runtime/config.py | 17 +++++ tests_runtime/test_local_lifecycle.py | 84 ++++++++++++++++++++++-- 3 files changed, 152 insertions(+), 43 deletions(-) diff --git a/src/opendox/runtime/bundle.py b/src/opendox/runtime/bundle.py index 0994f446..d38338b4 100644 --- a/src/opendox/runtime/bundle.py +++ b/src/opendox/runtime/bundle.py @@ -80,6 +80,7 @@ BUNDLE_OWNER_ROLE, BUNDLE_PORT, BUNDLE_SERVED_ROLE, + BUNDLE_SOCKET_DIR, LOCAL_FLAG, PREFIX, DatabaseBundle, @@ -326,8 +327,12 @@ def _preexec() -> None: # pragma: no cover - runs in the child return _preexec -def _unsafe_because(info: os.stat_result, *, uid: int, gid: int, - own: bool) -> str | None: +#: The install's own two directories under its state directory, the socket's +#: parent and the socket directory (`config.BUNDLE_SOCKET_DIR`). +BUNDLE_TREE = BUNDLE_SOCKET_DIR.parts + + +def _unsafe_because(info: os.stat_result, *, uid: int, own: bool) -> str | None: """Why one directory of the socket's path is unsafe, or `None`.""" mode = info.st_mode if stat.S_ISLNK(mode): @@ -343,13 +348,11 @@ def _unsafe_because(info: os.stat_result, *, uid: int, gid: int, return None if info.st_uid not in (uid, 0): return f"is owned by uid {info.st_uid}, neither this user nor root" - sticky = bool(mode & stat.S_ISVTX) - if mode & 0o002 and not sticky: - return (f"is writable by every user and is not sticky " - f"(mode {stat.S_IMODE(mode):o})") - if mode & 0o020 and info.st_gid != gid and not sticky: - return (f"is writable by group {info.st_gid}, which is not this " - f"user's own, and is not sticky (mode {stat.S_IMODE(mode):o})") + # A GROUP IS OTHER USERS, the user's own primary group included: it can + # have other members (Copilot review of openDox-code#69). + if mode & 0o022 and not mode & stat.S_ISVTX: + return (f"is writable by {'every user' if mode & 0o002 else 'its group'}" + f" and is not sticky (mode {stat.S_IMODE(mode):o})") return None @@ -452,37 +455,52 @@ def _refuse_an_unsafe_tree(self) -> None: """The socket's whole path is this user's to change, or it is refused. `trust` makes reaching the socket the credential, so the 0700 on the - socket directory is worth only what the directories above it are - worth. A directory entry is controlled by its PARENT. A parent that - another user can write lets them rename `run` away, or put a symbolic - link in its place, after the mode is set (Copilot review of - openDox-code#69). So, over the RESOLVED path, which a user's own - symbolic link in `OPENDOX_STATE_DIR` may lead to: - - * this install's own tree, the state directory, `postgres/` and - `run/`, must be real directories, owned by this user and writable - by no one else; - * every directory above it must be owned by this user or by root. - One that every user can write must be sticky, as `/tmp` is, so - nobody can rename what is not theirs. One that its group can - write must be sticky too, unless the group is this user's own, - which is how a umask-002 system creates the user's directories. + socket directory is worth only what the path above it is worth. A + directory entry is controlled by its PARENT: a parent that another + user can write lets them rename `run` away, or put a symbolic link in + its place, after the mode is set. A symbolic link on the way there + can be pointed elsewhere by whoever owns it, or by whoever can write + the directory it sits in (Copilot review of openDox-code#69). So: + + * THE INSTALL'S OWN TREE, as the configured path resolves: the + state directory, `postgres/` and `run/` must be real directories, + owned by this user and writable by no one else; + * EVERY DIRECTORY ABOVE IT, on the configured path and on the path + it resolves to, must be owned by this user or by root. One that + anyone else can write, a group included, must be sticky, as + `/tmp` is, so nobody can rename what is not theirs; + * EVERY SYMBOLIC LINK on the configured path must be this user's or + root's. + + `OPENDOX_STATE_DIR` never holds `..` (`config.state_dir` refuses it), + so the configured path's components are the ones the kernel walks. """ - uid, gid = os.getuid(), os.getgid() - state = self.bundle.state_dir.resolve() - own = [state, state / self.bundle.socket_dir.parent.name, - state / self.bundle.socket_dir.parent.name / self.bundle.socket_dir.name] - for directory, mine in [(path, True) for path in own] + \ - [(path, False) for path in state.parents]: - info = os.lstat(directory) - reason = _unsafe_because(info, uid=uid, gid=gid, own=mine) + uid = os.getuid() + configured = self.bundle.state_dir + state = configured.resolve() + tree = [state, state / BUNDLE_TREE[0], state / BUNDLE_TREE[0] / BUNDLE_TREE[1]] + checks = [(path, True) for path in tree] + [ + (path, False) for path in dict.fromkeys( + [*state.parents, *configured.parents])] + for directory, mine in checks: + info = os.lstat(directory) if mine else os.stat(directory) + reason = _unsafe_because(info, uid=uid, own=mine) if reason is not None: - raise BundleRefused( - f"{directory} {reason}, so another user could replace the " - "bundled server's socket directory, and reaching that " - "socket is the only credential the server asks for. Use a " - f"state directory only this user can change ({PREFIX}" - "STATE_DIR)") + raise self._unsafe(directory, reason) + for component in (configured, *configured.parents): + info = os.lstat(component) + if stat.S_ISLNK(info.st_mode) and info.st_uid not in (uid, 0): + raise self._unsafe( + component, f"is a symbolic link owned by uid {info.st_uid}, " + "neither this user nor root, who could point it elsewhere") + + @staticmethod + def _unsafe(directory: Path, reason: str) -> BundleRefused: + return BundleRefused( + f"{directory} {reason}, so another user could replace the bundled " + "server's socket directory, and reaching that socket is the only " + "credential the server asks for. Use a state directory only this " + f"user can change ({PREFIX}STATE_DIR)") #: The prefix an initialization attempt's directory carries, beside the #: data directory, followed by the pid of the process making it. diff --git a/src/opendox/runtime/config.py b/src/opendox/runtime/config.py index 56725995..36cd6272 100644 --- a/src/opendox/runtime/config.py +++ b/src/opendox/runtime/config.py @@ -1706,6 +1706,14 @@ def state_dir(env: Mapping[str, str] | None = None) -> Path: f"{setting} is {raw!r}, whose `~` names no user this system " "knows, so it expands to no directory. Name the state " "directory absolutely") from None + if ".." in path.parts: + # PARENT TRAVERSAL IS REFUSED, so the path the bundle checks is + # the one the kernel walks: `a/../b` names `b` lexically and + # something else wherever `a` is a symbolic link (Copilot review + # of openDox-code#69). + raise ConfigurationError( + f"{setting} is {raw!r}, which climbs out through `..`. Name " + "the state directory directly") if not path.is_absolute(): raise ConfigurationError( f"{setting} is {raw!r}, which is not an absolute path. The " @@ -1715,6 +1723,11 @@ def state_dir(env: Mapping[str, str] | None = None) -> Path: return path xdg = env.get("XDG_STATE_HOME", "").strip() if xdg and Path(xdg).is_absolute(): + if ".." in Path(xdg).parts: + raise ConfigurationError( + f"{setting} is unset and XDG_STATE_HOME is {xdg!r}, which " + f"climbs out through `..`. Set {setting}, or XDG_STATE_HOME, " + "to the directory itself") return Path(xdg) / "opendox" try: home = Path.home() @@ -1732,6 +1745,10 @@ def state_dir(env: Mapping[str, str] | None = None) -> Path: "absolute path, so the default state directory would depend on " f"the working directory. Set {setting} to an absolute path, or " "HOME to one") + if ".." in home.parts: + raise ConfigurationError( + f"{setting} is unset and HOME is {str(home)!r}, which climbs out " + f"through `..`. Set {setting} to the directory itself") return home / ".local" / "state" / "opendox" diff --git a/tests_runtime/test_local_lifecycle.py b/tests_runtime/test_local_lifecycle.py index 5bec3e8b..9285016d 100644 --- a/tests_runtime/test_local_lifecycle.py +++ b/tests_runtime/test_local_lifecycle.py @@ -441,23 +441,97 @@ def test_an_ancestor_every_user_can_write_without_the_sticky_bit_is_refused( assert str(open_dir) in str(caught.value) and "not sticky" in str(caught.value) -def test_a_sticky_or_own_group_ancestor_is_accepted( +def test_a_sticky_ancestor_is_accepted( monkeypatch, tmp_path: Path, short_state: Path) -> None: - """The positive controls: `/tmp`'s shape (every user, sticky) and a - umask-002 system's shape (this user's own group). The start gets past - the tree and reaches `initdb`, which the stand-in fails on purpose.""" + """The positive control, `/tmp`'s shape: every user can write it, and it + is sticky. The start gets past the tree and reaches `initdb`, which the + stand-in fails on purpose.""" sticky = short_state / "sticky" sticky.mkdir() sticky.chmod(0o1777) - group = sticky / "group" + server = _prepared(monkeypatch, tmp_path, sticky / "state") + with pytest.raises(bundle_mod.BundleRefused) as caught: + server.start() + assert "initdb" in str(caught.value), caught.value + + +def test_a_group_writable_ancestor_is_refused_even_for_this_users_group( + monkeypatch, tmp_path: Path, short_state: Path) -> None: + """A group is other users, the user's own primary group included (Copilot + review of #69): a 0775 ancestor is refused whatever its group.""" + group = short_state / "group" group.mkdir() group.chmod(0o775) + assert group.stat().st_gid == os.getgid() # this user's own group server = _prepared(monkeypatch, tmp_path, group / "state") + with pytest.raises(bundle_mod.BundleRefused) as caught: + server.start() + assert str(group) in str(caught.value) and "its group" in str(caught.value) + + +def test_a_link_in_a_directory_others_can_write_is_refused( + monkeypatch, tmp_path: Path, short_state: Path) -> None: + """The configured path goes through a symbolic link, and the link sits + in a directory every user can write. The link's TARGET is private, and + it is still refused, because anyone could replace the link (Copilot + review of #69).""" + private = short_state / "private" + private.mkdir(mode=0o700) + open_dir = short_state / "open" + open_dir.mkdir() + open_dir.chmod(0o777) + (open_dir / "link").symlink_to(private) + server = _prepared(monkeypatch, tmp_path, open_dir / "link" / "state") + with pytest.raises(bundle_mod.BundleRefused) as caught: + server.start() + assert str(open_dir) in str(caught.value) and "not sticky" in str(caught.value) + + +def test_this_users_own_link_to_a_private_directory_is_accepted( + monkeypatch, tmp_path: Path, short_state: Path) -> None: + private = short_state / "private" + private.mkdir(mode=0o700) + (short_state / "link").symlink_to(private) + server = _prepared(monkeypatch, tmp_path, short_state / "link" / "state") with pytest.raises(bundle_mod.BundleRefused) as caught: server.start() assert "initdb" in str(caught.value), caught.value +def test_a_link_another_user_owns_is_refused( + monkeypatch, tmp_path: Path, short_state: Path) -> None: + """Another user's link could be pointed elsewhere after the check. A + non-root suite cannot create one, so its `lstat` is stood in, for that + one path only.""" + private = short_state / "private" + private.mkdir(mode=0o700) + link = short_state / "link" + link.symlink_to(private) + real_lstat = os.lstat + + def _lstat(path, *args, **kwargs): + info = real_lstat(path, *args, **kwargs) + if Path(path) == link: + fields = list(info) + fields[4] = os.getuid() + 4242 # st_uid + return os.stat_result(fields) + return info + + monkeypatch.setattr(bundle_mod.os, "lstat", _lstat) + server = _prepared(monkeypatch, tmp_path, link / "state") + with pytest.raises(bundle_mod.BundleRefused) as caught: + server.start() + assert str(link) in str(caught.value) and "symbolic link owned by" in str(caught.value) + + +@pytest.mark.parametrize("variable", [STATE, "XDG_STATE_HOME"]) +def test_parent_traversal_in_the_state_path_is_refused(variable: str) -> None: + value = "/tmp/odx-a/../odx-b" + with pytest.raises(config.ConfigurationError) as caught: + config.state_dir({variable: value}) + assert "`..`" in str(caught.value), caught.value + + # -- the two refusal classes, each with its own reason -------------------------- From 84a6c04158e0d34b9f9d844a16aca4a691e9dfcc Mon Sep 17 00:00:00 2001 From: Brett Heap <1513478+brettheap@users.noreply.github.com> Date: Wed, 30 Sep 2026 17:47:27 +0000 Subject: [PATCH 18/39] T072: pixeltable-pgserver carries the server, and it authenticates by peer (RULED 5916000030 items 2, 3) Brett's rulings on openxFactory#656 (comment 5916000030) cover two things. Item 2, "pixeltable-pgserver (Recommended)". The `local` extra's carrier is now pixeltable-pgserver>=0.6.0, the maintained fork of pgserver. Only its binaries are used, found under pixeltable_pgserver/pginstall/bin. Measured on the installed 0.6.0 wheel: - initdb and postgres report PostgreSQL 16.14; - postgres links libz, libpthread, librt, libdl, libm and libc only, and initdb links the wheel's own vendored libpq through $ORIGIN; - the highest GLIBC symbol any binary or server module needs is 2.25, and the wheels are tagged manylinux_2_27/2_28; - the licence is Apache-2.0 (dist-info LICENSE and classifier); - the cp312 x86_64 wheel is 24,704,230 bytes; - wheels exist for cp310 to cp314. The lock was re-resolved in a clean environment under the existing pins less pgserver. The only line that moved is pgserver==0.1.4 -> pixeltable-pgserver==0.6.0. Item 3, "Peer auth + accept (Recommended)". - initdb now runs with --auth-local=peer --auth-host=reject. - Before every launch the bundle writes pg_hba.conf and pg_ident.conf atomically, mode 0600. pg_hba.conf holds one local rule, peer map=opendox, and host reject for IPv4 and IPv6. pg_ident.conf maps the running OS user (from the password database), and nobody else, to opendox and opendox_runtime. - listen_addresses stays empty. - An OS user name the map cannot hold plainly is refused, as is a uid with no password entry. - A cluster that an older build left as trust is put back to peer on its next start. The server's own reading proves it. pg_hba_file_rules has exactly those three rules and pg_ident_file_mappings exactly those two mappings, and system_user is peer: for both roles. The same OS user asking for a role outside the map is refused ("peer authentication failed"). Against 379fbb14's source and packaging, 13 of the new cases fail. Nine mutants of the carrier and the authentication are killed. The auth mutants are also killed by the real-server cases alone. Full suite: 2613 selected, 2602 passed, 11 skipped, 0 failed. Lane: openxfactory-4 (openXfactory-4-openDox_extraction) Arc: neutral-product-standalone-operability Co-Authored-By: Claude Opus 5.5 (1M context) --- constraints-cpython312-linux.txt | 7 +- pyproject.toml | 31 +++-- src/opendox/runtime/bundle.py | 158 +++++++++++++++++++------ src/opendox/runtime/config.py | 13 +- tests_runtime/test_bundled_postgres.py | 93 ++++++++++++++- tests_runtime/test_local_lifecycle.py | 75 +++++++++++- 6 files changed, 323 insertions(+), 54 deletions(-) diff --git a/constraints-cpython312-linux.txt b/constraints-cpython312-linux.txt index 4772dbe1..36437c08 100644 --- a/constraints-cpython312-linux.txt +++ b/constraints-cpython312-linux.txt @@ -34,6 +34,11 @@ # (`-c` this file), for plan 034 T072's `local` extra and the `test` extra's # `setuptools`: pgserver, its psutil, platformdirs and fasteners, and # setuptools are new; no earlier pin moved. +# Re-resolved 2026-09-30 on cpython 3.12.3 / linux x86_64, in a clean +# environment under the pins above less `pgserver`, when the carrier became +# `pixeltable-pgserver` (RULED, openxFactory#656 `5916000030` item 2): the one +# line that moved is `pgserver==0.1.4` -> `pixeltable-pgserver==0.6.0`, whose +# own requirements were all pinned already. PyJWT==2.14.0 PyYAML==6.0.3 Pygments==2.21.0 @@ -53,7 +58,7 @@ httpx==0.28.1 idna==3.20 iniconfig==2.3.0 packaging==26.3 -pgserver==0.1.4 +pixeltable-pgserver==0.6.0 platformdirs==4.12.2 pluggy==1.6.0 psutil==7.2.2 diff --git a/pyproject.toml b/pyproject.toml index 15f8ca53..3a870b31 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -119,18 +119,29 @@ test = [ # `opendox generate-and-open --local …`. It carries the runtime's packages and # the bundled server's own, and nothing else. # -# `pgserver` IS THE SERVER'S CARRIER, and only its binaries are used (see -# `opendox/runtime/bundle.py` for why its own manager is not): it ships -# PostgreSQL 16 as `initdb` and `postgres` inside the wheel, built to link only -# libc and libz, so a manylinux2014 host needs nothing else installed — no -# system PostgreSQL, no ICU. Apache-2.0; the server it carries is under the -# PostgreSQL License. `>=0.1.4` is the release this package has been exercised -# against (measured 2026-09-30, python 3.12.3: PostgreSQL 16.2), the same rule -# every floor in this file is set by. Its own three requirements (psutil, -# platformdirs, fasteners) arrive with it and are imported by nothing here. +# `pixeltable-pgserver` IS THE SERVER'S CARRIER (RULED, openxFactory#656 +# comment `5916000030` item 2, Brett Heap 2026-09-30: "pixeltable-pgserver +# (Recommended)"), and only its binaries are used (see `opendox/runtime/ +# bundle.py` for why its own manager is not). It is the maintained fork of +# `pgserver`, and it ships PostgreSQL 16 as `initdb` and `postgres` inside the +# wheel, built to link only libc and libz, so a host needs no system +# PostgreSQL and no ICU. Measured 2026-09-30 on its 0.6.0 cp312 manylinux +# wheel: +# * PostgreSQL 16.14 (`pginstall/bin/postgres --version`); the wheel also +# carries an 18.4 under `pginstall18/`, which this package does not use; +# * wheels for cp310 to cp314, so every interpreter `requires-python` +# admits today has one; `pgserver` 0.1.4 stopped at cp312 and 16.2; +# * linux wheels tagged manylinux_2_27/2_28 (glibc 2.27 and later), on +# x86_64 and aarch64, plus macOS and Windows; +# * about 24.7 MB per wheel, because it carries the two server majors; +# * Apache-2.0 for the package; the PostgreSQL License for the server. +# `>=0.6.0` is the release this package has been exercised against, the same +# rule every floor in this file is set by. Its own requirements (fasteners, +# platformdirs, psutil, typing-extensions) arrive with it and are imported by +# nothing here. local = [ "opendox[runtime]", - "pgserver>=0.1.4", + "pixeltable-pgserver>=0.6.0", ] # THE RUNTIME EXTRA — `split-opendox-two-layer-product` § 3.5, RULED Q2 diff --git a/src/opendox/runtime/bundle.py b/src/opendox/runtime/bundle.py index d38338b4..53a73e06 100644 --- a/src/opendox/runtime/bundle.py +++ b/src/opendox/runtime/bundle.py @@ -16,16 +16,18 @@ packages and the server's own; (iv) it stops with the entry point. -THE SERVER'S OWN PACKAGE IS `pgserver` (pyproject.toml's `local` extra), and -only its BINARIES are used: `initdb` and `postgres` from the wheel's -`pginstall/bin`, found by `importlib.util.find_spec` without importing -`pgserver` at all. Its Python manager is deliberately not used — it daemonizes -the server through `pg_ctl`, which re-parents it away from this process -(against (i)), shares one server between processes by reference count and -stops it from `atexit` (which a SIGTERM never runs, against (iv)), and may put -the socket under the user's runtime directory instead of the state directory -(against 13.1). The binaries themselves link only libc and libz, so they run on -any manylinux2014 host, which a wheel that links the system's ICU does not. +THE SERVER'S OWN PACKAGE IS `pixeltable-pgserver` (pyproject.toml's `local` +extra; RULED openxFactory#656 `5916000030` item 2), and only its BINARIES are +used: PostgreSQL 16's `initdb` and `postgres` from the wheel's `pginstall/bin`, +found by `importlib.util.find_spec` without importing `pixeltable_pgserver` at +all. Its Python manager is deliberately not used. It daemonizes the server +through `pg_ctl`, which re-parents it away from this process (against (i)). It +shares one server between processes by reference count and stops it from +`atexit`, which a SIGTERM never runs (against (iv)). And it may put the socket +under the user's runtime directory, opened to 0777, instead of the state +directory (against 13.1). The binaries link only the C library (libc, libm, +libpthread, librt, libdl) and libz from the system, plus the wheel's own +vendored libpq, so they need no system PostgreSQL and no ICU. THE LIFECYCLE, IN FULL: @@ -33,10 +35,15 @@ renamed into place only when it has succeeded, so an interrupted first start never leaves a half-built cluster: the MIGRATION identity (`config.BUNDLE_OWNER_ROLE`) is the bootstrap superuser, local connections - are `trust` and host connections are `reject`, UTF-8 in the `C` locale. - Trust is safe BECAUSE of the socket: its directory is 0700, owned by the - user running the install, and the server opens no TCP port at all, so - reaching the socket is the credential and nothing else can. + are `peer` and host connections are `reject`, UTF-8 in the `C` locale. + * PEER AUTHENTICATION, re-asserted before every launch (RULED + openxFactory#656 `5916000030` item 3). `pg_hba.conf` admits Unix-socket + connections through the `opendox` map only, and `pg_ident.conf`'s map + admits THIS install's OS user as the two roles and nobody else. The kernel + reports the connecting process's uid (`SO_PEERCRED`), so no password + exists to leak or to store, and a process of any other user is refused + even where it could reach the socket. The socket's directory is 0700 + besides, and the server opens no TCP port at all. * `postgres` started as a DIRECT CHILD of this process (`subprocess.Popen`, never `pg_ctl`), with `listen_addresses` empty and the socket directory given. On Linux it also carries `PR_SET_PDEATHSIG`, so an entry point @@ -88,8 +95,13 @@ database_bundle, ) -#: The distribution the `local` extra installs for the server's binaries. -SERVER_DISTRIBUTION = "pgserver" +#: The distribution the `local` extra installs for the server's binaries, and +#: the package it installs them under. +SERVER_DISTRIBUTION = "pixeltable-pgserver" +SERVER_PACKAGE = "pixeltable_pgserver" + +#: The `pg_ident.conf` map `pg_hba.conf`'s one local line authenticates through. +IDENT_MAP = "opendox" #: How long a start may take before it is a failure: `initdb` on a slow disk, #: plus the server's own recovery on a data directory an earlier run did not @@ -114,11 +126,11 @@ class BundleRefused(Exception): def server_binaries() -> Path: """The directory holding the bundled `initdb` and `postgres`, or a refusal. - Found WITHOUT importing `pgserver`: its package initializer imports its - manager, which this module does not use and whose import creates a lock - object under the user's runtime directory as a side effect. + Found WITHOUT importing `pixeltable_pgserver`: its package initializer + imports its manager, which this module does not use and which registers an + `atexit` handler and reaches for the user's runtime directory. """ - spec = importlib.util.find_spec(SERVER_DISTRIBUTION) + spec = importlib.util.find_spec(SERVER_PACKAGE) # THE FIRST LOCATION, OR NONE, read without an index, so no path reaches # a subscript that could raise (SonarCloud S6466 on openDox-code#69). location = next(iter(spec.submodule_search_locations or ()), None) \ @@ -332,6 +344,78 @@ def _preexec() -> None: # pragma: no cover - runs in the child BUNDLE_TREE = BUNDLE_SOCKET_DIR.parts +def os_user() -> str: + """The name of the OS user this runs as, which peer authentication maps. + + The SERVER resolves a connecting uid to a name through the same password + database, so a uid with no entry could not be authenticated at all, and is + refused here, by name. So is a name that `pg_ident.conf` could read as more + than a name: a regular expression (a leading `/`), a quote, a comment + mark, or white space. + """ + import pwd + + uid = os.getuid() + try: + name = pwd.getpwuid(uid).pw_name + except KeyError: + raise BundleRefused( + f"uid {uid} has no entry in the password database, and the bundled " + "server's peer authentication maps the connecting user BY NAME, so " + "it could never admit this one. Run the local install as a user " + "the system knows") from None + if not name or name.startswith("/") or any( + ch in name for ch in '"#') or any(ch.isspace() for ch in name): + raise BundleRefused( + f"the OS user name {name!r} cannot be written into the bundled " + "server's pg_ident.conf as a plain name (it holds a quote, a `#`, " + "white space, or starts with `/`)") + return name + + +def authentication_files(user: str) -> dict[str, str]: + """`pg_hba.conf` and `pg_ident.conf` for a local install run by `user`. + + RULED openxFactory#656 `5916000030` item 3 ("Peer auth + accept"): + * ONE local line, PEER through the `opendox` map. The kernel reports the + connecting process's uid, and the map admits `user` as the owner role + and as the served role, and nobody else as anybody. + * Every HOST connection is rejected. The server also listens on no TCP + address at all (`listen_addresses` is empty), so these lines never + match. They are written so that the file says what the install is. + * No replication line, so a replication connection is refused. + """ + header = ("# Written by opendox.runtime.bundle before every start of this local\n" + "# install's bundled server (plan 034 T072). Changes here are replaced.\n") + hba = (header + + "# TYPE DATABASE USER ADDRESS METHOD\n" + f"local all all peer map={IDENT_MAP}\n" + "host all all 0.0.0.0/0 reject\n" + "host all all ::/0 reject\n") + ident = (header + + "# MAPNAME SYSTEM-USERNAME PG-USERNAME\n" + f'{IDENT_MAP} "{user}" {BUNDLE_OWNER_ROLE}\n' + f'{IDENT_MAP} "{user}" {BUNDLE_SERVED_ROLE}\n') + return {"pg_hba.conf": hba, "pg_ident.conf": ident} + + +def write_authentication(data_dir: Path, user: str) -> None: + """Write both files into `data_dir`, each replaced atomically, mode 0600. + + Before EVERY launch, not only after `initdb`: a data directory an earlier + build initialized, or a file edited by hand, is brought back to the one + configuration this install runs with. + """ + for name, content in authentication_files(user).items(): + target = data_dir / name + temporary = data_dir / f".{name}.opendox-{os.getpid()}" + descriptor = os.open(temporary, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, + 0o600) + with os.fdopen(descriptor, "w", encoding="utf-8") as handle: + handle.write(content) + os.replace(temporary, target) + + def _unsafe_because(info: os.stat_result, *, uid: int, own: bool) -> str | None: """Why one directory of the socket's path is unsafe, or `None`.""" mode = info.st_mode @@ -407,6 +491,8 @@ def start(self) -> BundledServer: self._prepare_directories() phase = "initializing its data directory" self._initialize(binaries) + phase = "configuring its authentication" + write_authentication(self.bundle.data_dir, os_user()) phase = "launching it" _remove_a_proven_stale_lock(self.bundle) self._launch(binaries) @@ -439,9 +525,9 @@ def _prepare_directories(self) -> None: A directory that already exists is NOT re-moded, on the rule the runtime's `init` keeps for an operator's own path — except the socket - directory, whose mode IS the access control of a `trust` server: it is - this install's own, under its own state directory, and it is narrowed - to 0700 whatever it was. + directory, which guards the only way in: it is this install's own, + under its own state directory, and it is narrowed to 0700 whatever it + was. """ for directory in (self.bundle.state_dir, self.bundle.data_dir.parent): directory.mkdir(mode=0o700, parents=True, exist_ok=True) @@ -454,11 +540,14 @@ def _prepare_directories(self) -> None: def _refuse_an_unsafe_tree(self) -> None: """The socket's whole path is this user's to change, or it is refused. - `trust` makes reaching the socket the credential, so the 0700 on the - socket directory is worth only what the path above it is worth. A - directory entry is controlled by its PARENT: a parent that another - user can write lets them rename `run` away, or put a symbolic link in - its place, after the mode is set. A symbolic link on the way there + The socket's directory is how this install's clients find ITS + server, so the 0700 on it is worth only what the path above it is + worth. Peer authentication keeps other users out of the server, but + not a substitute socket out of the path: whoever could replace `run` + could stand up a server of their own for this install's clients to + talk to. A directory entry is controlled by its PARENT: a parent that + another user can write lets them rename `run` away, or put a symbolic + link in its place, after the mode is set. A symbolic link on the way there can be pointed elsewhere by whoever owns it, or by whoever can write the directory it sits in (Copilot review of openDox-code#69). So: @@ -498,9 +587,9 @@ def _refuse_an_unsafe_tree(self) -> None: def _unsafe(directory: Path, reason: str) -> BundleRefused: return BundleRefused( f"{directory} {reason}, so another user could replace the bundled " - "server's socket directory, and reaching that socket is the only " - "credential the server asks for. Use a state directory only this " - f"user can change ({PREFIX}STATE_DIR)") + "server's socket directory and put a server of their own where " + "this install's clients look for it. Use a state directory only " + f"this user can change ({PREFIX}STATE_DIR)") #: The prefix an initialization attempt's directory carries, beside the #: data directory, followed by the pid of the process making it. @@ -565,7 +654,7 @@ def _remove_abandoned_attempts(self) -> None: def _initdb(self, binaries: Path, target: Path) -> None: done = subprocess.run( [str(binaries / "initdb"), "-D", str(target), - "-U", BUNDLE_OWNER_ROLE, "--auth-local=trust", "--auth-host=reject", + "-U", BUNDLE_OWNER_ROLE, "--auth-local=peer", "--auth-host=reject", "--encoding=UTF8", "--locale=C", "--no-instructions"], env=_child_environment(), capture_output=True, text=True, timeout=START_TIMEOUT_SECONDS) @@ -719,5 +808,6 @@ def __exit__(self, *exc: object) -> None: __all__ = ["BUNDLE_PORT", "BundleRefused", "BundledServer", - "isolated_from_libpq_environment", "report", "running_pid", - "server_binaries"] + "authentication_files", "isolated_from_libpq_environment", + "os_user", "report", "running_pid", "server_binaries", + "write_authentication"] diff --git a/src/opendox/runtime/config.py b/src/opendox/runtime/config.py index 36cd6272..42d99eb5 100644 --- a/src/opendox/runtime/config.py +++ b/src/opendox/runtime/config.py @@ -1665,10 +1665,15 @@ def dsn(self, role: str) -> str: `host` is the socket DIRECTORY (libpq's rule for a value that starts with `/`), percent-encoded so a state directory holding a space or a `&` is still one value; `port` is spelled so a `PGPORT` in the - environment cannot send libpq to a different socket file. No password: - the socket directory is 0700 and the server's own `pg_hba.conf` - trusts local connections only, so reaching the socket IS the - credential, and a host connection is rejected outright. + environment cannot send libpq to a different socket file. NO + PASSWORD, because there is none to give: the server authenticates a + Unix-socket connection by PEER (RULED openxFactory#656 `5916000030` + item 3). The kernel reports the connecting process's uid, and + `pg_ident.conf` maps this install's OS user, and nobody else, to the + two roles. The socket's directory is 0700, the server opens no TCP + port, and a host connection is rejected outright. SonarCloud's S2115 + ("add password protection") is ACCEPTED on this line for that reason, + with the same ruling as its authority. """ host = urllib.parse.quote(str(self.socket_dir), safe="/") return (f"postgresql://{role}@/{BUNDLE_DATABASE}" diff --git a/tests_runtime/test_bundled_postgres.py b/tests_runtime/test_bundled_postgres.py index 56e1f844..5fb6f5d2 100644 --- a/tests_runtime/test_bundled_postgres.py +++ b/tests_runtime/test_bundled_postgres.py @@ -36,6 +36,7 @@ import selectors import shutil import signal +import stat import subprocess import sys import sysconfig @@ -238,12 +239,16 @@ def test_the_local_extra_carries_the_runtime_and_the_server_and_test_joins_it( project = tomllib.loads((ROOT / "pyproject.toml").read_text()) extras = project["project"]["optional-dependencies"] assert "opendox[runtime]" in extras["local"] - assert any(req.startswith("pgserver") for req in extras["local"]) + # the carrier RULED on openxFactory#656 `5916000030` item 2 + assert any(req.startswith("pixeltable-pgserver") for req in extras["local"]) + assert not any(req.startswith("pgserver") for req in extras["local"]) assert "opendox[local]" in extras["test"], ( "F9.1 installs `.[test]` alone; without the local extra there, this " "suite could not start the server it tests") lock = (ROOT / "constraints-cpython312-linux.txt").read_text() - assert re.search(r"(?m)^pgserver==", lock), "the lock does not pin pgserver" + assert re.search(r"(?m)^pixeltable-pgserver==", lock), \ + "the lock does not pin pixeltable-pgserver" + assert not re.search(r"(?m)^pgserver==", lock), "the lock still pins pgserver" files = project["tool"]["setuptools"]["data-files"] assert files == {"share/opendox/migrations": ["migrations/*.sql"]} # and the packaging notes name functions that exist (Copilot review of #69) @@ -432,6 +437,90 @@ def test_a_stale_lock_naming_a_recycled_pid_does_not_hold_the_bundle( decoy.wait(timeout=10) +# -- peer authentication (RULED openxFactory#656 `5916000030` item 3) ---------- + + +def _owner(server) -> "object": + import psycopg + + return psycopg.connect(server.bundle.migration_dsn, autocommit=True) + + +def test_the_bundle_authenticates_by_peer_through_the_one_map( + state_dir: Path) -> None: + """The server's OWN reading of its two files, from `pg_hba_file_rules` + and `pg_ident_file_mappings`, and the method each connection really + used, from `system_user`. There is one local rule, peer through the + `opendox` map. Host is rejected. There is no `trust` anywhere. The map + admits this OS user as the two roles and names no other OS user.""" + user = bundle_mod.os_user() + settings = config.load_settings({MODE: "local", STATE: str(state_dir)}) + with bundle_mod.BundledServer(settings) as server: + with _owner(server) as conn: + rules = conn.execute( + "select type, database, user_name, auth_method, options, error " + "from pg_hba_file_rules order by rule_number").fetchall() + mappings = conn.execute( + "select map_name, sys_name, pg_username, error " + "from pg_ident_file_mappings order by map_number").fetchall() + for dsn, role in ((server.bundle.migration_dsn, config.BUNDLE_OWNER_ROLE), + (server.bundle.served_dsn, config.BUNDLE_SERVED_ROLE)): + import psycopg + + with psycopg.connect(dsn) as conn: + assert conn.execute("select current_user, system_user").fetchone() \ + == (role, f"peer:{user}") + assert rules == [ + ("local", ["all"], ["all"], "peer", [f"map={bundle_mod.IDENT_MAP}"], None), + ("host", ["all"], ["all"], "reject", None, None), + ("host", ["all"], ["all"], "reject", None, None)], rules + assert mappings == [ + (bundle_mod.IDENT_MAP, user, config.BUNDLE_OWNER_ROLE, None), + (bundle_mod.IDENT_MAP, user, config.BUNDLE_SERVED_ROLE, None)], mappings + + +def test_a_role_outside_the_map_is_refused_even_for_this_os_user( + state_dir: Path) -> None: + """The MAP decides, not the socket. The same OS user, over the same + 0700 socket, asking for a role the map does not name, is refused by + peer authentication. A suite that does not run as root cannot connect + as a second OS user. What stands for that case is the map itself, read + back above, which names this user and no other.""" + import psycopg + from psycopg import sql + + settings = config.load_settings({MODE: "local", STATE: str(state_dir)}) + with bundle_mod.BundledServer(settings) as server: + with _owner(server) as conn: + conn.execute(sql.SQL("create role {} login").format( + sql.Identifier("odx_stranger"))) + with pytest.raises(psycopg.OperationalError) as caught: + psycopg.connect(server.bundle.dsn("odx_stranger")).close() + assert "peer authentication failed" in str(caught.value).lower(), caught.value + + +def test_an_older_trust_cluster_is_brought_back_to_peer_on_start( + state_dir: Path) -> None: + """A data directory an earlier build initialized with `trust` (or a file + edited by hand) is put back to the one configuration before the next + launch, so it never serves as trust.""" + settings = config.load_settings({MODE: "local", STATE: str(state_dir)}) + bundle_mod.BundledServer(settings).start().stop() + data = config.DatabaseBundle(state_dir).data_dir + (data / "pg_hba.conf").write_text("local all all trust\n", encoding="utf-8") + (data / "pg_ident.conf").write_text("", encoding="utf-8") + with bundle_mod.BundledServer(settings) as server: + import psycopg + + with psycopg.connect(server.bundle.served_dsn) as conn: + method = conn.execute("select system_user").fetchone()[0] + expected = bundle_mod.authentication_files(bundle_mod.os_user()) + for name, content in expected.items(): + assert (data / name).read_text(encoding="utf-8") == content, name + assert stat.S_IMODE((data / name).stat().st_mode) == 0o600, name + assert method == f"peer:{bundle_mod.os_user()}", method + + def test_migrate_under_the_local_mode_uses_the_bundle_and_refuses_a_dsn( state_dir: Path) -> None: """`runtime migrate` is part of the same install: it reads the bundle's diff --git a/tests_runtime/test_local_lifecycle.py b/tests_runtime/test_local_lifecycle.py index 9285016d..baee0695 100644 --- a/tests_runtime/test_local_lifecycle.py +++ b/tests_runtime/test_local_lifecycle.py @@ -569,13 +569,14 @@ def test_both_classes_together_name_both_reasons() -> None: @pytest.mark.parametrize("found", ["absent", "no-locations"]) def test_a_missing_server_package_is_the_named_refusal( monkeypatch: pytest.MonkeyPatch, found: str) -> None: - """No `pgserver` at all, or a spec with no location: both are the one - refusal naming the `local` extra, never an `IndexError`.""" + """No `pixeltable_pgserver` at all, or a spec with no location: both are + the one refusal naming the `local` extra, never an `IndexError`.""" import importlib.machinery spec = None if found == "no-locations": - spec = importlib.machinery.ModuleSpec("pgserver", None, is_package=True) + spec = importlib.machinery.ModuleSpec(bundle_mod.SERVER_PACKAGE, None, + is_package=True) spec.submodule_search_locations = [] monkeypatch.setattr(bundle_mod.importlib.util, "find_spec", lambda name: spec) @@ -584,6 +585,74 @@ def test_a_missing_server_package_is_the_named_refusal( assert 'opendox[local]' in str(caught.value), caught.value +# -- peer authentication, hermetic ---------------------------------------------- + + +def test_the_authentication_files_admit_one_os_user_as_the_two_roles() -> None: + files = bundle_mod.authentication_files("alice") + active = {name: [line.split() for line in text.splitlines() + if line.strip() and not line.startswith("#")] + for name, text in files.items()} + assert active["pg_hba.conf"] == [ + ["local", "all", "all", "peer", f"map={bundle_mod.IDENT_MAP}"], + ["host", "all", "all", "0.0.0.0/0", "reject"], + ["host", "all", "all", "::/0", "reject"]] + assert active["pg_ident.conf"] == [ + [bundle_mod.IDENT_MAP, '"alice"', config.BUNDLE_OWNER_ROLE], + [bundle_mod.IDENT_MAP, '"alice"', config.BUNDLE_SERVED_ROLE]] + assert "trust" not in " ".join(" ".join(r) for rows in active.values() for r in rows) + + +def test_the_files_are_written_0600_and_replace_what_was_there( + tmp_path: Path) -> None: + (tmp_path / "pg_hba.conf").write_text("local all all trust\n") + bundle_mod.write_authentication(tmp_path, "alice") + for name, content in bundle_mod.authentication_files("alice").items(): + assert (tmp_path / name).read_text(encoding="utf-8") == content + assert stat.S_IMODE((tmp_path / name).stat().st_mode) == 0o600 + assert sorted(p.name for p in tmp_path.iterdir()) == ["pg_hba.conf", "pg_ident.conf"] + + +@pytest.mark.parametrize("name", ["/regex", 'quo"te', "has space", "hash#tag", ""]) +def test_an_os_user_name_the_map_cannot_hold_plainly_is_refused( + monkeypatch: pytest.MonkeyPatch, name: str) -> None: + import pwd + + class _Entry: + pw_name = name + + monkeypatch.setattr(pwd, "getpwuid", lambda uid: _Entry()) + with pytest.raises(bundle_mod.BundleRefused): + bundle_mod.os_user() + + +def test_a_uid_with_no_password_entry_is_refused_by_name( + monkeypatch: pytest.MonkeyPatch) -> None: + import pwd + + def _missing(uid): + raise KeyError(uid) + + monkeypatch.setattr(pwd, "getpwuid", _missing) + with pytest.raises(bundle_mod.BundleRefused) as caught: + bundle_mod.os_user() + assert "password database" in str(caught.value) + + +def test_initdb_is_asked_for_peer_and_host_reject( + monkeypatch, tmp_path: Path, short_state: Path) -> None: + """The flags `initdb` receives, recorded by a stand-in: local is `peer`, + host is `reject`, and `trust` is not asked for anywhere.""" + record = tmp_path / "initdb-argv" + server = _server(monkeypatch, tmp_path, short_state, initdb=( + f'echo "$@" > "{record}"\nexit 1')) + with pytest.raises(bundle_mod.BundleRefused): + server.start() + argv = record.read_text(encoding="utf-8").split() + assert "--auth-local=peer" in argv and "--auth-host=reject" in argv, argv + assert not [a for a in argv if "trust" in a], argv + + # -- initdb, and every phase of a start --------------------------------------- From f8e6e9e9ef23566cbf6046c4167a272307581827 Mon Sep 17 00:00:00 2001 From: Brett Heap <1513478+brettheap@users.noreply.github.com> Date: Wed, 30 Sep 2026 18:08:40 +0000 Subject: [PATCH 19/39] Fix round: the data path, fresh directories and readiness belong to this install (Copilot review) Copilot's review at 84a6c041 made three points, all real. - r4147680113: the tree check left out postgres/data. An existing data directory, a broken link included, now joins the own-tree check: a real directory, owned by this user, writable by no one else, not a link. A link to a cluster elsewhere would otherwise have been given this install's authentication files and launched outside the state tree. A fresh data directory needs no check, because _initialize renames it into place. - Fresh directories and the umask (overview, previously missed). mkdir(parents=True) creates intermediate directories with the default mode less the umask. Under umask 0002, a fresh ~/.local/state/opendox would create group-writable parents, which the tree check then refused. Each missing component is now created on its own and set to exactly 0700, whatever the umask. - Readiness (overview, previously missed). A successful connection proves only that some server answered. Two entry points racing from an idle state both launch, and the loser's postgres lives a moment while the winner's socket answers. So readiness now also needs the data directory's lock file to name this child. Otherwise the wait goes on until this child exits and is refused. One check after the connection is enough, since the lock admits one postmaster per data directory and the socket directory belongs to exactly one data directory. A before-check was tried and dropped: no mutant distinguishes it. Against 84a6c041's bundle.py, all 6 new cases fail. 4 mutants are killed. Full suite: 2619 selected, 2608 passed, 11 skipped, 0 failed. Lane: openxfactory-4 (openXfactory-4-openDox_extraction) Arc: neutral-product-standalone-operability Co-Authored-By: Claude Opus 5.5 (1M context) --- src/opendox/runtime/bundle.py | 60 ++++++++++++++++++++++++-- tests_runtime/test_bundled_postgres.py | 24 +++++++++++ tests_runtime/test_local_lifecycle.py | 52 ++++++++++++++++++++++ 3 files changed, 132 insertions(+), 4 deletions(-) diff --git a/src/opendox/runtime/bundle.py b/src/opendox/runtime/bundle.py index 53a73e06..f5ff03c0 100644 --- a/src/opendox/runtime/bundle.py +++ b/src/opendox/runtime/bundle.py @@ -84,6 +84,7 @@ from opendox.runtime import migrations from opendox.runtime.config import ( BUNDLE_DATABASE, + BUNDLE_DATA_DIR, BUNDLE_OWNER_ROLE, BUNDLE_PORT, BUNDLE_SERVED_ROLE, @@ -344,6 +345,31 @@ def _preexec() -> None: # pragma: no cover - runs in the child BUNDLE_TREE = BUNDLE_SOCKET_DIR.parts +def _make_private_directories(leaf: Path) -> None: + """`leaf` and every missing directory above it, each created 0700. + + `Path.mkdir(parents=True)` gives the directories it creates on the way + the default mode less the umask, whatever mode the leaf is given. So + under a common umask of 0002 a fresh `~/.local/state/opendox/...` would + create `.local` and `state` group-writable, and the tree check would + then refuse the directories this install had just made (Copilot review + of openDox-code#69). Each missing component is created here, one at a + time, and set to exactly 0700, whatever the umask is. A directory that + already exists is left as it is, and the tree check judges it. + """ + missing = [] + for directory in (leaf, *leaf.parents): + if os.path.lexists(directory): + break + missing.append(directory) + for directory in reversed(missing): + try: + directory.mkdir(mode=0o700) + except FileExistsError: + continue # made by a concurrent start; judged below + os.chmod(directory, 0o700) + + def os_user() -> str: """The name of the OS user this runs as, which peer authentication maps. @@ -529,9 +555,7 @@ def _prepare_directories(self) -> None: under its own state directory, and it is narrowed to 0700 whatever it was. """ - for directory in (self.bundle.state_dir, self.bundle.data_dir.parent): - directory.mkdir(mode=0o700, parents=True, exist_ok=True) - self.bundle.socket_dir.mkdir(mode=0o700, parents=True, exist_ok=True) + _make_private_directories(self.bundle.socket_dir) # CHECKED BEFORE THE CHMOD, which follows a symbolic link: a `run` # placed there as a link would otherwise have its TARGET re-moded. self._refuse_an_unsafe_tree() @@ -568,6 +592,14 @@ def _refuse_an_unsafe_tree(self) -> None: configured = self.bundle.state_dir state = configured.resolve() tree = [state, state / BUNDLE_TREE[0], state / BUNDLE_TREE[0] / BUNDLE_TREE[1]] + # AND THE DATA DIRECTORY, where one exists already, a broken link + # included (Copilot review of openDox-code#69). A `data` placed there + # as a link to a cluster elsewhere would otherwise be launched, and + # given this install's authentication files, outside the state tree. + # A fresh one needs no check: `_initialize` renames it into place. + data = state / BUNDLE_DATA_DIR + if os.path.lexists(data): + tree.append(data) checks = [(path, True) for path in tree] + [ (path, False) for path in dict.fromkeys( [*state.parents, *configured.parents])] @@ -701,7 +733,22 @@ def _wait_until_ready(self) -> None: with psycopg.connect(self._dsn(BUNDLE_OWNER_ROLE, "postgres"), connect_timeout=2, autocommit=True) as conn: conn.execute("select 1") - return + # READY MEANS THIS CHILD IS SERVING, not merely that the socket + # answered (Copilot review of openDox-code#69). Two entry points + # racing from an idle state both launch. The loser's `postgres` + # lives a moment before it refuses the winner's lock, and the + # winner's socket already answers. So once a connection has + # answered, the data directory's lock file must name THIS + # child. Otherwise the wait goes on until this child exits and + # is refused. One check, AFTER the connection, is enough. The + # lock admits one postmaster per data directory, the socket + # directory belongs to exactly one data directory, and a + # lock naming this child therefore means the socket that + # answered is this child's. + if self._serving_is_this_child(): + return + last = "the socket answered, but not from this child" + time.sleep(0.1) except psycopg.OperationalError as exc: # THE CLASS NAME ONLY: a driver's message quotes the DSN it # could not reach, and this package never repeats one @@ -713,6 +760,11 @@ def _wait_until_ready(self) -> None: f"{START_TIMEOUT_SECONDS:.0f}s ({last}); its log is " f"{self.log_path}") + def _serving_is_this_child(self) -> bool: + """Whether the data directory's lock file names the child just launched.""" + return (self.process is not None + and _lock_file_pid(self.bundle) == self.process.pid) + def _dsn(self, role: str, database: str) -> str: """`DatabaseBundle.dsn`, aimed at a database other than the served one.""" return self.bundle.dsn(role).replace( diff --git a/tests_runtime/test_bundled_postgres.py b/tests_runtime/test_bundled_postgres.py index 5fb6f5d2..62b6b3b8 100644 --- a/tests_runtime/test_bundled_postgres.py +++ b/tests_runtime/test_bundled_postgres.py @@ -437,6 +437,30 @@ def test_a_stale_lock_naming_a_recycled_pid_does_not_hold_the_bundle( decoy.wait(timeout=10) +def test_readiness_is_this_childs_server_not_a_winners_socket( + state_dir: Path) -> None: + """Two entry points racing from an idle state both launch. The loser's + `postgres` lives a moment before it refuses the winner's lock, while the + winner's socket already answers. The loser must wait for ITS child, and + be refused when that child exits, not connect to the winner and carry on + as if it owned a database (Copilot review of openDox-code#69). The + loser's child is stood in by a process that lives three seconds.""" + settings = config.load_settings({MODE: "local", STATE: str(state_dir)}) + with bundle_mod.BundledServer(settings): + loser = bundle_mod.BundledServer(settings) + loser.process = subprocess.Popen( + [sys.executable, "-c", "import time; time.sleep(3)"]) + began = time.monotonic() + try: + with pytest.raises(bundle_mod.BundleRefused) as caught: + loser._wait_until_ready() + finally: + loser.process.kill() + loser.process.wait(timeout=10) + assert "exited during start" in str(caught.value), caught.value + assert time.monotonic() - began < bundle_mod.START_TIMEOUT_SECONDS + + # -- peer authentication (RULED openxFactory#656 `5916000030` item 3) ---------- diff --git a/tests_runtime/test_local_lifecycle.py b/tests_runtime/test_local_lifecycle.py index baee0695..58696b0b 100644 --- a/tests_runtime/test_local_lifecycle.py +++ b/tests_runtime/test_local_lifecycle.py @@ -524,6 +524,58 @@ def _lstat(path, *args, **kwargs): assert str(link) in str(caught.value) and "symbolic link owned by" in str(caught.value) +@pytest.mark.parametrize("umask", [0o002, 0o200]) +def test_missing_directories_are_created_0700_whatever_the_umask( + monkeypatch, tmp_path: Path, short_state: Path, umask: int) -> None: + """A fresh default path creates the directories above the state tree + too. Under umask 0002 `mkdir(parents=True)` would make them 0775, and + the tree check would then refuse what this install had just made + (Copilot review of #69). A umask that takes the owner's own bits would + leave them unusable. Each is created exactly 0700, and the start reaches + `initdb`, which the stand-in fails on purpose.""" + state = short_state / "a" / "b" / "state" + server = _prepared(monkeypatch, tmp_path, state) + previous = os.umask(umask) + try: + with pytest.raises(bundle_mod.BundleRefused) as caught: + server.start() + finally: + os.umask(previous) + assert "initdb" in str(caught.value), caught.value + for directory in (short_state / "a", short_state / "a" / "b", state, + state / "postgres", state / "postgres" / "run"): + assert stat.S_IMODE(directory.stat().st_mode) == 0o700, directory + + +@pytest.mark.parametrize("shape", ["link", "broken-link", "open"]) +def test_a_data_directory_that_is_not_this_installs_own_is_refused( + monkeypatch, tmp_path: Path, short_state: Path, shape: str) -> None: + """An existing `data` joins the tree check (Copilot review of #69): as a + link to a cluster elsewhere, as a broken link, or as a directory others + can write. It is refused before anything is written into it or launched + on it.""" + elsewhere = tmp_path / "cluster-elsewhere" + (short_state / "postgres").mkdir(mode=0o700) + data = short_state / "postgres" / "data" + if shape in {"link", "open"}: + target = elsewhere if shape == "link" else data + target.mkdir(mode=0o700) + (target / "PG_VERSION").write_text("16\n", encoding="utf-8") + if shape in {"link", "broken-link"}: + data.symlink_to(elsewhere) + if shape == "open": + data.chmod(0o777) + server = _prepared(monkeypatch, tmp_path, short_state) + with pytest.raises(bundle_mod.BundleRefused) as caught: + server.start() + message = str(caught.value) + assert str(data) in message, message + assert ("symbolic link" if shape != "open" else "writable by every user") \ + in message, message + if shape == "link": + assert not (elsewhere / "pg_hba.conf").exists(), "wrote into the link's target" + + @pytest.mark.parametrize("variable", [STATE, "XDG_STATE_HOME"]) def test_parent_traversal_in_the_state_path_is_refused(variable: str) -> None: value = "/tmp/odx-a/../odx-b" From c8fac05e328bd879a220e43bd8c985a81a43b653 Mon Sep 17 00:00:00 2001 From: Brett Heap <1513478+brettheap@users.noreply.github.com> Date: Fri, 2 Oct 2026 20:04:39 +0000 Subject: [PATCH 20/39] T070, owed at the merge round: the standalone generate-and-open runs say --local Since this PR, generate-and-open with neither --local nor OPENDOX_INSTALL_MODE=local is a HOSTED install, which refuses without its broker's issuer (#1144 13.4, 13.5). Four cases that landed on main with phase 2 run generate-and-open as the single-user install and relied on the old default, so each now says --local: - tests/test_standalone_generate_path.py (T056), case 3: the server starts, answers and stops. The module docstring names the change and moves F10.1's plain-install run to T077. - tests/test_post_render_validator.py (T058), test_generate_and_open_gives_the_same_verdicts, both fixtures. - tests/test_projection_seams.py (T055), test_generate_and_open_refuses_an_empty_source_option_before_its_run_dir. No case means hosted, so none takes a hosted fixture. Before this commit, all four fail on the merged tree with the hosted issuer refusal; after it they pass. Three mutants of the local path are killed, each failing all four cases: --local ignored, local refusing its own loopback default, and local also asking for the hosted issuer. Full suite: 3117 selected, 3106 passed, 11 skipped, 0 failed. Lane: openxfactory-4 (openXfactory-4-openDox_extraction) Arc: neutral-product-standalone-operability Co-Authored-By: Claude Opus 5.5 (1M context) --- tests/test_post_render_validator.py | 10 ++++++---- tests/test_projection_seams.py | 8 +++++--- tests/test_standalone_generate_path.py | 19 +++++++++++++------ 3 files changed, 24 insertions(+), 13 deletions(-) diff --git a/tests/test_post_render_validator.py b/tests/test_post_render_validator.py index f2772edf..a18ddd22 100644 --- a/tests/test_post_render_validator.py +++ b/tests/test_post_render_validator.py @@ -141,11 +141,13 @@ def test_F7_2_the_malformed_fixture_is_refused_without_strict_too(tmp_path) -> N @pytest.mark.parametrize("fixture,expected", [(PLAIN, 0), (MALFORMED, 1)]) def test_generate_and_open_gives_the_same_verdicts(tmp_path, fixture, expected) -> None: - """`generate-and-open --no-open --no-serve --strict`: the good fixture - builds its server and prints its URL, and the malformed one stops before - a server is built, naming the rule.""" + """`generate-and-open --local --no-open --no-serve --strict`: the good + fixture builds its server and prints its URL, and the malformed one stops + before a server is built, naming the rule. `--local` because this is the + single-user install: since plan 034 T070, an unflagged run is HOSTED and + refuses without its broker's issuer, before the validator is reached.""" repo = fresh_repository(fixture, tmp_path) - child, status = run_module(tmp_path, "opendox.cli", "generate-and-open", + child, status = run_module(tmp_path, "opendox.cli", "generate-and-open", "--local", "--repo-root", str(repo), "--repository", "fixture", "--no-open", "--no-serve", "--strict", "--run-dir", str(tmp_path / "run")) diff --git a/tests/test_projection_seams.py b/tests/test_projection_seams.py index 1d5c5883..766a848a 100644 --- a/tests/test_projection_seams.py +++ b/tests/test_projection_seams.py @@ -1397,9 +1397,11 @@ def test_generate_and_open_refuses_an_empty_source_option_before_its_run_dir( _declaring_generator(calls) repo = _repository(tmp_path) run_dir = tmp_path / "run" - rc = cli.main(["generate-and-open", "--repo-root", str(repo), "--repository", - "garden", "--run-dir", str(run_dir), "--no-open", "--no-serve", - "--possibles", ""]) + # `--local`: the single-user install. Since plan 034 T070 an unflagged run + # is HOSTED, and its issuer refusal would come first. + rc = cli.main(["generate-and-open", "--local", "--repo-root", str(repo), + "--repository", "garden", "--run-dir", str(run_dir), + "--no-open", "--no-serve", "--possibles", ""]) assert rc == 1 assert ("generate-and-open refused: --possibles was given an empty path" in capsys.readouterr().err) diff --git a/tests/test_standalone_generate_path.py b/tests/test_standalone_generate_path.py index 0a95efca..b71f3650 100644 --- a/tests/test_standalone_generate_path.py +++ b/tests/test_standalone_generate_path.py @@ -17,7 +17,7 @@ role keys, the verb reports it, naming the document, the value and the six keys, and the snapshot it writes reads that document as a source. T054 tests the projection's half in process. -3. `python -m opendox.cli generate-and-open --no-open` STARTS a server, which +3. `python -m opendox.cli generate-and-open --local --no-open` STARTS a server, which answers `/index.html`, `/snapshot.json`, `/capabilities` and `/source/`, refuses `/source/.git/config`, and stops on an interrupt with status 0. 4. `python -m opendox.serve`, the server's own entry point, starts and answers @@ -46,8 +46,14 @@ seconds). T056 flushes it in both entry points, and cases 3 and 4 fail without that. -NOT HERE: F10.1's run through a plain install, with the console script and no -`--local`, arrives in phase 3 (T070, and T077 as batch H amends it). +`--local` (plan 034 T070; #1144 13.4, 13.5): case 3 is the single-user install, +so it says so. Since T070, `generate-and-open` with neither `--local` nor +`OPENDOX_INSTALL_MODE=local` is a HOSTED install, which refuses without its +broker's issuer. That refusal is what an unflagged run of this case would now +hit, and it is T070's own subject, held in `tests/test_install_mode_entrypoint.py`. + +NOT HERE: F10.1's run through a plain install, with the console script, arrives +in phase 3 (T077, as batch H amends it). A CREATED FILE: no carve-manifest row (RULED OQ-C). """ @@ -256,9 +262,10 @@ def test_the_unedited_fixture_declares_that_document_a_candidate(tmp_path) -> No # --------------------------------------------------------------------------- def test_generate_and_open_starts_a_server_that_answers_with_no_sibling(tmp_path) -> None: - """`python -m opendox.cli generate-and-open --no-open`, with no + """`python -m opendox.cli generate-and-open --local --no-open`, with no `--no-serve`: the server starts, says where on a buffered pipe, answers - the core routes, and stops on an interrupt with status 0. + the core routes, and stops on an interrupt with status 0. `--local` + because this is the single-user install (T070). Both lines it prints before blocking in `serve_forever()`, the URL and "serving until interrupted", are read WHILE IT RUNS, before the @@ -267,7 +274,7 @@ def test_generate_and_open_starts_a_server_that_answers_with_no_sibling(tmp_path e3574774, r4146289331).""" repo = _fresh_repository(tmp_path) run_dir = tmp_path / "run" - child = Child(tmp_path, "opendox.cli", "generate-and-open", + child = Child(tmp_path, "opendox.cli", "generate-and-open", "--local", "--repo-root", str(repo), "--repository", "fixture", "--no-open", "--port", "0", "--run-dir", str(run_dir)) try: From 19e32f0cdb074e4ad12249f72858711403c8b0d9 Mon Sep 17 00:00:00 2001 From: Brett Heap <1513478+brettheap@users.noreply.github.com> Date: Fri, 2 Oct 2026 20:20:59 +0000 Subject: [PATCH 21/39] T072, owed at the merge round: the real entry point, and every --local child keeps its own state Phase 2 is on this stack's base, so four things T072 owed at its merge round are done. - The stand-ins go. tests_runtime/local_entrypoint_driver.py is deleted. Its stand-ins patched names that T055 has since replaced, so on the merged tree they stood in for nothing, and all three background cases failed: the real corpus-root check refused the stand-in corpus, a directory with no repository. test_bundled_postgres.py now launches `python -m opendox.cli generate-and-open --local`, with the validator on, over T050's tests/fixtures/plain-documents copied into a fresh repository, as F13.1's preamble does. - Every cheap refusal comes before the database start. main's T055 added _refuse_empty_source_options to the generate path, so the local path asks it before it builds the bundled server, beside the corpus-root and generated-at refusals. test_projection_seams.py's empty-option case now carries a tripwire bundle, so a regression neither starts a server nor passes. - No child touches the user's state directory. A `generate-and-open --local` child now starts the bundled server, and OPENDOX_STATE_DIR defaults to the user's own ~/.local/state/opendox. tests/standalone_child.py gives every child a fresh, short, private state directory under /tmp and removes it when the child is stopped. Measured before: the three --local children of T056 and T058 initialized a cluster in the (sandboxed) default state home. - T056's case 3 asserts that its bundled server's data directory is under the child's own state directory while serving, and that the directory is gone after the stop. Four mutants are killed. They drop the cheap refusal, the private state dir, its removal, and the fixture's repository. Full suite: 3195 selected, 3184 passed, 11 skipped, 0 failed. Nothing is left under ~/.local/state/opendox or /tmp/odx-child-*. Lane: openxfactory-4 (openXfactory-4-openDox_extraction) Arc: neutral-product-standalone-operability Co-Authored-By: Claude Opus 5.5 (1M context) --- src/opendox/cli.py | 5 +- tests/standalone_child.py | 12 ++++ tests/test_projection_seams.py | 20 +++++- tests/test_standalone_generate_path.py | 6 ++ tests_runtime/local_entrypoint_driver.py | 77 ------------------------ tests_runtime/test_bundled_postgres.py | 38 ++++++++---- 6 files changed, 68 insertions(+), 90 deletions(-) delete mode 100644 tests_runtime/local_entrypoint_driver.py diff --git a/src/opendox/cli.py b/src/opendox/cli.py index 67315a57..1438ac30 100644 --- a/src/opendox/cli.py +++ b/src/opendox/cli.py @@ -642,9 +642,12 @@ def cmd_generate_and_open(args: argparse.Namespace, *, opener=webbrowser.open) - args.runtime_settings = settings if settings.install_mode != runtime_config.INSTALL_MODE_LOCAL: return _generate_and_open(args, opener=opener) - # THE CHEAP REFUSALS FIRST, so a mistyped root never costs a database start. + # THE CHEAP REFUSALS FIRST, so a mistyped root never costs a database + # start: every one `_generate_and_open` asks before it mints its run + # directory, main's empty-source-option refusal (T055) included. _refuse_non_corpus_repo_root(args) _refuse_malformed_generated_at(args) + _refuse_empty_source_options(args) server = bundle_mod.BundledServer(settings) args.database_bundle = server # NO `PG*` DEFAULT REACHES THE BUNDLE'S CONNECTIONS while this process diff --git a/tests/standalone_child.py b/tests/standalone_child.py index 37d1fe5f..f8bb1c6a 100644 --- a/tests/standalone_child.py +++ b/tests/standalone_child.py @@ -20,6 +20,13 @@ * It is read on threads while it runs (`Child.wait_for_line`), so a server that never exits can still be asked where it serves, and then interrupted (`Child.interrupt`, SIGINT, as Ctrl-C sends). +* ITS STATE DIRECTORY IS ITS OWN. A `generate-and-open --local` child starts + the local install's bundled PostgreSQL server (plan 034 T072) under + `OPENDOX_STATE_DIR`, whose default is the USER's own state directory. So + every child is given a fresh, short, private one (`Child.state_dir`, under + `/tmp` because a Unix socket's whole path is bounded), and it is removed + once the child is stopped. No case ever initializes a database in the home + directory of whoever runs the suite. * CTRL-C REACHES IT AS IT WOULD AT A TERMINAL, whatever the runner's own disposition. The same `sitecustomize` sets SIGINT back to Python's KeyboardInterrupt handler. A runner started as a background job @@ -45,6 +52,7 @@ import signal import subprocess import sys +import tempfile import threading import time from pathlib import Path @@ -145,6 +153,9 @@ def __init__(self, workdir: Path, module: str, *args: str) -> None: env["PYTHONPATH"] = os.pathsep.join( [str(blocker), *filter(None, [env.get("PYTHONPATH")])]) env[REFUSED_LOG_ENV] = str(self.refused_log) + self.state_dir = Path(tempfile.mkdtemp( + prefix="odx-child-", dir="/tmp" if os.path.isdir("/tmp") else None)) + env["OPENDOX_STATE_DIR"] = str(self.state_dir) self.argv = [sys.executable, "-m", module, *args] verb = args[0] if args and not args[0].startswith("-") else "" self.label = f"python -m {module} {verb}".strip() @@ -221,6 +232,7 @@ def kill(self) -> None: if self.process.poll() is None: self.process.kill() self.process.wait(timeout=STOP_DEADLINE_SECONDS) + shutil.rmtree(self.state_dir, ignore_errors=True) def _join(self) -> None: for pump in self._pumps: diff --git a/tests/test_projection_seams.py b/tests/test_projection_seams.py index 766a848a..ac2f942d 100644 --- a/tests/test_projection_seams.py +++ b/tests/test_projection_seams.py @@ -1392,7 +1392,24 @@ def test_a_given_source_option_is_resolved_and_an_unset_one_is_not_passed( def test_generate_and_open_refuses_an_empty_source_option_before_its_run_dir( - tmp_path, capsys) -> None: + tmp_path, capsys, monkeypatch) -> None: + """And before the local install's bundled server (plan 034 T072): a + refused option costs no database start. A tripwire stands in for the + server, so a regression neither starts one nor passes.""" + started: list = [] + + class _Tripwire: + def __init__(self, settings) -> None: + started.append(settings) + + def start(self): + raise AssertionError("the bundled server was started for a " + "refused source option") + + def stop(self) -> None: + pass + + monkeypatch.setattr(cli.bundle_mod, "BundledServer", _Tripwire) calls: list = [] _declaring_generator(calls) repo = _repository(tmp_path) @@ -1406,6 +1423,7 @@ def test_generate_and_open_refuses_an_empty_source_option_before_its_run_dir( assert ("generate-and-open refused: --possibles was given an empty path" in capsys.readouterr().err) assert calls == [] and not run_dir.exists() + assert started == [], "a bundled server was built for a refused option" def test_a_root_openDoxs_predicate_refuses_is_refused_with_its_message(tmp_path, capsys) -> None: diff --git a/tests/test_standalone_generate_path.py b/tests/test_standalone_generate_path.py index b71f3650..60d50f0e 100644 --- a/tests/test_standalone_generate_path.py +++ b/tests/test_standalone_generate_path.py @@ -284,11 +284,17 @@ def test_generate_and_open_starts_a_server_that_answers_with_no_sibling(tmp_path child.wait_for_line(_SERVING) assert child.process.poll() is None, "the server exited after saying it serves" _assert_the_server_answers(base, run_dir / "snapshot.json", repo) + # THE LOCAL INSTALL'S DATABASE IS THE CHILD'S OWN (plan 034 T072): its + # bundled server was started under the private state directory the + # harness gave this child, never under the user's. + assert (child.state_dir / "postgres" / "data" / "PG_VERSION").is_file(), \ + "the bundled server was not started under the child's state dir" assert child.interrupt() == 0, child.stderr_text() finally: child.kill() _assert_the_port_is_closed(base) assert child.refused() == [], child.refused() + assert not child.state_dir.exists(), "the child's state dir outlived it" # --------------------------------------------------------------------------- diff --git a/tests_runtime/local_entrypoint_driver.py b/tests_runtime/local_entrypoint_driver.py deleted file mode 100644 index 43e6d63c..00000000 --- a/tests_runtime/local_entrypoint_driver.py +++ /dev/null @@ -1,77 +0,0 @@ -"""Run `opendox generate-and-open` for real, with ONLY its generation stood in. - -`tests_runtime/test_bundled_postgres.py` launches this file as a child process -(`python tests_runtime/local_entrypoint_driver.py generate-and-open --local …`) -so that F13.1's local probe can be run the way F13.1 runs it: in the -BACKGROUND, reached over HTTP, asked about by a SECOND process -(`runtime status`), and then stopped with a signal (plan 034 T072). - -WHAT IS STOOD IN, AND WHY IT IS ONLY THIS. At this stack's base, the generate -verbs still reach openXdox for three names (`corpus_root_refusal`, -`generate_snapshot`, `snapshot.write_snapshot`) and `serve` for two -(`_checkout_real`, `registry_mod`'s binding constants): phase 2's T055 and T056 -(openDox-code#59 and its successor) give openDox its own, and a lone checkout -cannot ask for them before then. They are replaced here exactly as -`tests/test_doxbench_entrypoint.py` replaces them, and NOTHING ELSE is: -`cli.main`, the install shape, the bundled server, `build_server` and the serve -loop all run as a user's `opendox generate-and-open --local` runs them. Once -T055 and T056 have landed on this branch's base, this driver's stand-ins go and -the probe runs the real entry point on `tests/fixtures/plain-documents`. -""" - -from __future__ import annotations - -import json -import sys -import types -from pathlib import Path - - -class _StandInSource: - """`build_server`'s snapshot source, as T011's `standalone` fixture has it.""" - - refresh_binding = None - baked_repository = None - - class registry: - active = None - - def bootstrap(self): - pass - - -def _generate_snapshot(repo_root, repository, *, source_revision=None, - **_ignored): - return {"repository": repository, - "generation": {"source_revision": source_revision}, - "documents": [{"id": "stand-in.md"}]} - - -def _write_snapshot(snapshot, output, boundary): - Path(output).write_text(json.dumps(snapshot), encoding="utf-8") - return output - - -def main(argv: list[str]) -> int: - from opendox import cli as cli_mod - from opendox import serve as serve_mod - - cli_mod.corpus_root_refusal = lambda root, shape=None: None - cli_mod.generate_snapshot = _generate_snapshot - cli_mod.snapshot_mod = types.SimpleNamespace(write_snapshot=_write_snapshot) - serve_mod._checkout_real = lambda root: False - serve_mod.registry_mod = types.SimpleNamespace( - BINDING_REGENERATE="regenerate", BINDING_REFETCH="refetch") - real_build_server = serve_mod.build_server - - def _build_server(*args, **kwargs): - kwargs.setdefault("snapshot_source", _StandInSource()) - return real_build_server(*args, **kwargs) - - serve_mod.build_server = _build_server - return cli_mod.main(argv) - - -if __name__ == "__main__": - sys.stdout.reconfigure(line_buffering=True) - sys.exit(main(sys.argv[1:])) diff --git a/tests_runtime/test_bundled_postgres.py b/tests_runtime/test_bundled_postgres.py index 62b6b3b8..bae2a7f5 100644 --- a/tests_runtime/test_bundled_postgres.py +++ b/tests_runtime/test_bundled_postgres.py @@ -3,9 +3,13 @@ T072's falsifier is F13.1's TCP-listener block, which reads the kernel's socket table at run time, and its `runtime status` block. Both are run here -against a server the REAL entry point started: `generate-and-open --local`, -launched in the background as F13.1 launches it, reached over HTTP, asked -about by a second process, and stopped with a signal. Where F13.1 reads the +against a server the REAL entry point started: `python -m opendox.cli +generate-and-open --local` over a fresh repository copied from T050's +`tests/fixtures/plain-documents`, launched in the background as F13.1 launches +it, reached over HTTP, asked about by a second process, and stopped with a +signal. Nothing is stood in: the corpus root check, the generation, the +validator and the serve loop are phase 2's own, landed on this stack's base +(T054 to T058), so the stand-in driver this module once launched is gone. Where F13.1 reads the server's pid from `caps.json`, these cases read the same pid from `runtime status`'s `database_bundle`. `/capabilities`' `install` block is T073's, and nothing here pretends it exists. @@ -54,7 +58,8 @@ ROOT = Path(__file__).resolve().parents[1] SRC = ROOT / "src" -DRIVER = Path(__file__).resolve().parent / "local_entrypoint_driver.py" +#: T050's fixture, which F13.1's preamble copies into a fresh repository. +PLAIN_DOCUMENTS = ROOT / "tests" / "fixtures" / "plain-documents" MODE = PREFIX + "INSTALL_MODE" STATE = PREFIX + "STATE_DIR" @@ -174,10 +179,9 @@ def _launch(corpus: Path, state: Path, run_dir: Path, **extra: str) -> tuple[subprocess.Popen, str]: """`generate-and-open --local` in the BACKGROUND, and the URL it serves.""" child = subprocess.Popen( - [sys.executable, str(DRIVER), "generate-and-open", config.LOCAL_FLAG, - "--repo-root", str(corpus), "--repository", "fixture", - "--run-dir", str(run_dir), "--no-open", "--no-validate", - "--port", "0"], + [sys.executable, "-m", "opendox.cli", "generate-and-open", + config.LOCAL_FLAG, "--repo-root", str(corpus), "--repository", + "fixture", "--run-dir", str(run_dir), "--no-open", "--port", "0"], env=_clean_env(**{STATE: str(state)}, **extra), cwd=ROOT, stdout=subprocess.PIPE, stderr=subprocess.PIPE) url = _first_url(child, 90) @@ -225,9 +229,21 @@ def _status(state: Path, **extra: str) -> tuple[int, dict]: @pytest.fixture() def corpus(tmp_path: Path) -> Path: - root = tmp_path / "plain-documents" - root.mkdir() - (root / "note.md").write_text("# A note\n\nPlain text.\n", encoding="utf-8") + """F13.1's preamble: T050's `plain-documents` copied into a FRESH git + repository, committed under the fixture's own identity and none of the + user's git configuration.""" + root = tmp_path / PLAIN_DOCUMENTS.name + shutil.copytree(PLAIN_DOCUMENTS, root) + env = {name: value for name, value in os.environ.items() + if not name.startswith("GIT_")} + env.update({"GIT_AUTHOR_NAME": "fixture", + "GIT_AUTHOR_EMAIL": "fixture@example.invalid", + "GIT_COMMITTER_NAME": "fixture", + "GIT_COMMITTER_EMAIL": "fixture@example.invalid", + "GIT_CONFIG_GLOBAL": os.devnull, "GIT_CONFIG_SYSTEM": os.devnull}) + for argv in (["git", "-c", "init.defaultBranch=main", "init", "-q"], + ["git", "add", "-A"], ["git", "commit", "-qm", "fixture"]): + subprocess.run(argv, cwd=root, env=env, check=True, capture_output=True) return root From c39d960e4144a934fb2d39c2361ddd2b5ed0f8d4 Mon Sep 17 00:00:00 2001 From: Brett Heap <1513478+brettheap@users.noreply.github.com> Date: Fri, 2 Oct 2026 20:28:35 +0000 Subject: [PATCH 22/39] Fix round: a PostgreSQL scheme libpq would not read as a URI is refused (Copilot review) Copilot's review at the merge-from-main head (adeb6fed) noted that `postgresql:foo` is still accepted, although the supported URI forms need `://`. Measured, it is worse than that. urlsplit reads `postgresql:` with no `//`, and any capitalized `PostgreSQL://` or `POSTGRES://`, as the PostgreSQL scheme, so the dialect gate passed all of them. libpq reads none of them as a URI. It recognizes only the exact, lower-case `postgresql://` and `postgres://`, parses the rest as keyword/value, and refuses them with a message that repeats the whole value (psycopg 3.3.6: `missing "=" after "postgresql:svc:hunter2@db/x" in connection info string`). That is the un-named failure at the driver that 13.2 exists to stop, and it carries the password. _refuse_non_postgresql_dsn now refuses a PostgreSQL scheme in any spelling other than libpq's two. The refusal names the setting and the two spellings, and does not repeat the value. Both loaders ask it. The new case covers four spellings for each of the two settings. All 8 fail against adeb6fed's config.py and pass here. The two accepted spellings and the keyword/value form still load. Five mutants are killed: the check dropped, case-insensitive matching, a check of only the `//`, every PostgreSQL DSN refused, and the value repeated. Full suite: 3069 selected, 3058 passed, 11 skipped, 0 failed. Lane: openxfactory-4 (openXfactory-4-openDox_extraction) Arc: neutral-product-standalone-operability Co-Authored-By: Claude Opus 5.5 (1M context) --- src/opendox/runtime/config.py | 20 ++++++++++++++++++++ tests_runtime/test_runtime_cli.py | 28 ++++++++++++++++++++++++++++ 2 files changed, 48 insertions(+) diff --git a/src/opendox/runtime/config.py b/src/opendox/runtime/config.py index 5c755006..7ae28936 100644 --- a/src/opendox/runtime/config.py +++ b/src/opendox/runtime/config.py @@ -1350,6 +1350,26 @@ def _refuse_non_postgresql_dsn(name: str, dsn: str | None) -> None: "runtime keeps: a second one would double every migration and " "every schema test forever, for a database that holds no " "document (RULING Q1)") + # A POSTGRESQL SCHEME IS A URI ONLY IN LIBPQ'S OWN SPELLING (Copilot + # review of openDox-code#60, at its merge-from-main round). `urlsplit` + # reads `postgresql:` with no `//`, and any capitalized `PostgreSQL://`, + # as the PostgreSQL scheme. libpq does not: it recognizes a URI only by + # the exact, lower-case `postgresql://` or `postgres://`, and parses + # anything else as keyword/value, which it then refuses with a message + # that REPEATS THE WHOLE VALUE (measured, psycopg 3.3.6: + # `missing "=" after "postgresql:svc:hunter2@db/x" in connection info + # string`). That is the un-named failure at the driver that 13.2 exists to + # stop, and it carries the password with it. So it is refused here, named, + # and the value is not repeated. + if scheme in POSTGRESQL_SCHEMES and not dsn.startswith( + tuple(f"{known}://" for known in sorted(POSTGRESQL_SCHEMES))): + raise ConfigurationError( + f"{name} reads as the PostgreSQL scheme but is not a URI libpq " + "reads: libpq recognizes only the exact, lower-case " + "`postgresql://` or `postgres://` prefix, and would refuse any " + "other spelling with a message that repeats the whole value. " + "Write the scheme as one of those two (the value is not " + "repeated here, because it can carry a password)") def _refuse_the_same_dsn_in_both_settings( diff --git a/tests_runtime/test_runtime_cli.py b/tests_runtime/test_runtime_cli.py index feadfa1a..c3e13bb6 100644 --- a/tests_runtime/test_runtime_cli.py +++ b/tests_runtime/test_runtime_cli.py @@ -2264,6 +2264,34 @@ def test_a_non_postgresql_dsn_is_refused_naming_the_dialect_kept() -> None: "host=h dbname=db user=m"}) +@pytest.mark.parametrize("dsn", ["postgresql:svc:hunter2@db.invalid/x", + "postgresql:/svc:hunter2@db.invalid/x", + "PostgreSQL://svc:hunter2@db.invalid/x", + "POSTGRES://svc:hunter2@db.invalid/x"]) +@pytest.mark.parametrize("setting", ["DATABASE_URL", "MIGRATION_DATABASE_URL"]) +def test_a_postgresql_scheme_libpq_would_not_read_as_a_uri_is_refused( + dsn: str, setting: str) -> None: + """`urlsplit` reads each of these as the PostgreSQL scheme. libpq reads + none of them as a URI: it knows only the exact, lower-case + `postgresql://` and `postgres://`, so it parses the rest as keyword/value + and refuses them with a message that repeats the whole value, password + and all. So each is refused at configuration, named, and the value is + not repeated (Copilot review of this PR, at its merge-from-main round). + The two spellings libpq does read stay accepted (the case above).""" + from opendox.runtime.config import ConfigurationError, load_settings + + good = {PREFIX + "DATABASE_URL": "postgresql://u:p@h/db", + PREFIX + "MIGRATION_DATABASE_URL": "postgresql://m:q@h/db"} + with pytest.raises(ConfigurationError) as refused: + load_settings({PREFIX + "OIDC_ISSUER": "https://broker/realms/x", + PREFIX + "OIDC_AUDIENCE": "opendox", + **good, PREFIX + setting: dsn}) + message = str(refused.value) + assert PREFIX + setting in message, message + assert "postgresql://" in message, message + assert "hunter2" not in message and dsn not in message, message + + def test_an_unparseable_dsn_is_refused_and_never_raises_a_bare_valueerror( ) -> None: """`urlsplit` itself raises for a DSN it cannot parse, and this module's From cdf7382b7e3f30884e0c93703b8254fe782bdb2b Mon Sep 17 00:00:00 2001 From: Brett Heap <1513478+brettheap@users.noreply.github.com> Date: Fri, 2 Oct 2026 20:35:03 +0000 Subject: [PATCH 23/39] Fix round: the --local callers inherit none of the runner's runtime settings (Copilot review) Copilot's review at c8fac05e opened three threads, and all three are real. The four callers this PR gave --local took the runner's environment along: tests/standalone_child.py's Child copied os.environ, and the in-process projection-seams case read it directly. An exported OPENDOX_INSTALL_MODE=hosted, or a broker issuer, made each --local run refuse before it reached what it tests. - tests/standalone_child.py: every child's environment drops each name in opendox.runtime.config.SETTING_NAMES. This covers T056's case 3 and T058's two post-render cases, and any later child. - tests/test_projection_seams.py: the in-process empty-option case scrubs SETTING_NAMES first, as the doxBench entrypoint fixture does. - tests/test_standalone_generate_path.py: a harness case exports a hosted install's four settings and asserts that a child sees none of them. Measured with OPENDOX_INSTALL_MODE=hosted and OPENDOX_OIDC_ISSUER exported: all 5 cases fail against 9ae5e72's harness and pass here. Mutant "the child keeps the runner's settings" is killed by the harness case in a plain environment. Mutant "the in-process case keeps them" is killed under the exported hosted mode. Full suite: 3126 selected, 3115 passed, 11 skipped, 0 failed. Lane: openxfactory-4 (openXfactory-4-openDox_extraction) Arc: neutral-product-standalone-operability Co-Authored-By: Claude Opus 5.5 (1M context) --- tests/standalone_child.py | 11 ++++++++- tests/test_projection_seams.py | 11 ++++++++- tests/test_standalone_generate_path.py | 31 +++++++++++++++++++++++++- 3 files changed, 50 insertions(+), 3 deletions(-) diff --git a/tests/standalone_child.py b/tests/standalone_child.py index 37d1fe5f..34bbda01 100644 --- a/tests/standalone_child.py +++ b/tests/standalone_child.py @@ -20,6 +20,12 @@ * It is read on threads while it runs (`Child.wait_for_line`), so a server that never exits can still be asked where it serves, and then interrupted (`Child.interrupt`, SIGINT, as Ctrl-C sends). +* IT INHERITS NO RUNTIME SETTING. Every `OPENDOX_*` name the runtime reads + (`opendox.runtime.config.SETTING_NAMES`) is taken out of the child's + environment, so an `OPENDOX_INSTALL_MODE=hosted` or a broker issuer the + runner happens to export cannot make a `generate-and-open --local` child + refuse before the case it exists for (plan 034 T070; Copilot review of + openDox-code#67). * CTRL-C REACHES IT AS IT WOULD AT A TERMINAL, whatever the runner's own disposition. The same `sitecustomize` sets SIGINT back to Python's KeyboardInterrupt handler. A runner started as a background job @@ -140,7 +146,10 @@ def __init__(self, workdir: Path, module: str, *args: str) -> None: blocker.mkdir(parents=True, exist_ok=True) (blocker / "sitecustomize.py").write_text(_BLOCKER, encoding="utf-8") self.refused_log = workdir / "refused-imports.log" - env = dict(os.environ) + from opendox.runtime.config import SETTING_NAMES + + env = {name: value for name, value in os.environ.items() + if name not in SETTING_NAMES} env.pop("PYTHONUNBUFFERED", None) env["PYTHONPATH"] = os.pathsep.join( [str(blocker), *filter(None, [env.get("PYTHONPATH")])]) diff --git a/tests/test_projection_seams.py b/tests/test_projection_seams.py index 766a848a..02c5ec65 100644 --- a/tests/test_projection_seams.py +++ b/tests/test_projection_seams.py @@ -1392,7 +1392,16 @@ def test_a_given_source_option_is_resolved_and_an_unset_one_is_not_passed( def test_generate_and_open_refuses_an_empty_source_option_before_its_run_dir( - tmp_path, capsys) -> None: + tmp_path, capsys, monkeypatch) -> None: + """In process, so the runtime settings the runner exports are scrubbed + first, as the doxBench entrypoint fixture does: an exported + `OPENDOX_INSTALL_MODE=hosted` or broker issuer would otherwise make + `--local` refuse before the empty option is reached (Copilot review of + openDox-code#67).""" + from opendox.runtime.config import SETTING_NAMES + + for name in SETTING_NAMES: + monkeypatch.delenv(name, raising=False) calls: list = [] _declaring_generator(calls) repo = _repository(tmp_path) diff --git a/tests/test_standalone_generate_path.py b/tests/test_standalone_generate_path.py index b71f3650..a199de51 100644 --- a/tests/test_standalone_generate_path.py +++ b/tests/test_standalone_generate_path.py @@ -22,7 +22,8 @@ refuses `/source/.git/config`, and stops on an interrupt with status 0. 4. `python -m opendox.serve`, the server's own entry point, starts and answers the same way. -5. The harness itself: a child that ignores the interrupt is killed at the +5. The harness itself: a child inherits none of the runner's runtime + settings, and a child that ignores the interrupt is killed at the deadline, and the timeout is raised, so a server that will not stop is reported rather than waited out. And a child stops on the interrupt even when the RUNNER ignores SIGINT, as a suite started as a background job @@ -321,6 +322,34 @@ def test_serve_main_starts_a_server_that_answers_with_no_sibling(tmp_path) -> No # 5 — the harness itself: an ignored interrupt is reported, not waited out # --------------------------------------------------------------------------- +def test_a_child_inherits_none_of_the_runners_runtime_settings( + tmp_path, monkeypatch) -> None: + """The runner exports a HOSTED install's settings, and the child sees + none of them (plan 034 T070; Copilot review of openDox-code#67). The + `--local` cases above would otherwise refuse before they reach what they + test, for a reason that is the runner's configuration and not theirs.""" + from opendox.runtime.config import PREFIX, SETTING_NAMES + + exported = {PREFIX + "INSTALL_MODE": "hosted", + PREFIX + "OIDC_ISSUER": "https://issuer.example.invalid/realms/x", + PREFIX + "OIDC_AUDIENCE": "fixture", + PREFIX + "DATABASE_URL": "postgresql://s@127.0.0.1:1/x"} + for name, value in exported.items(): + monkeypatch.setenv(name, value) + blocker = tmp_path / "sibling-blocker" + blocker.mkdir() + (blocker / "t070_env_probe.py").write_text(textwrap.dedent(""" + import json, os + print(json.dumps(sorted(n for n in os.environ if n.startswith("OPENDOX_"))), + flush=True) + """), encoding="utf-8") + child, status = run_module(tmp_path, "t070_env_probe") + assert status == 0, child.stderr_text() + seen = set(json.loads(child.stdout_text().strip().splitlines()[-1])) + assert not seen & set(SETTING_NAMES), sorted(seen & set(SETTING_NAMES)) + assert set(exported) <= set(SETTING_NAMES) + + def test_a_child_that_ignores_the_interrupt_is_killed_at_the_deadline( tmp_path, monkeypatch) -> None: """The regression path cases 3 and 4 guard: a server that does not stop From 6eb0bbdb962c611c11466c9e6efd406134ebf47c Mon Sep 17 00:00:00 2001 From: Brett Heap <1513478+brettheap@users.noreply.github.com> Date: Fri, 2 Oct 2026 20:41:59 +0000 Subject: [PATCH 24/39] T072: the env probe expects the child's own state directory, never the runner's #67's harness case asserted a child sees no runtime setting at all. Under T072 every child is given one: OPENDOX_STATE_DIR, its private state directory. The probe now also exports a runner state directory and asserts that the child sees exactly OPENDOX_STATE_DIR among the runtime settings, with the value of its own Child.state_dir (not the runner's), and that the directory is gone once the child has exited. Mutants, all run under an exported hosted install's settings, are all killed: - the child keeps the runner's settings; - the scrub runs after the state directory is set; - the runner's own state directory is passed through; - the in-process empty-option case does not scrub. Lane: openxfactory-4 (openXfactory-4-openDox_extraction) Co-Authored-By: Claude Opus 5.5 (1M context) --- tests/test_standalone_generate_path.py | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/tests/test_standalone_generate_path.py b/tests/test_standalone_generate_path.py index f752f311..567484b7 100644 --- a/tests/test_standalone_generate_path.py +++ b/tests/test_standalone_generate_path.py @@ -333,27 +333,35 @@ def test_a_child_inherits_none_of_the_runners_runtime_settings( """The runner exports a HOSTED install's settings, and the child sees none of them (plan 034 T070; Copilot review of openDox-code#67). The `--local` cases above would otherwise refuse before they reach what they - test, for a reason that is the runner's configuration and not theirs.""" + test, for a reason that is the runner's configuration and not theirs. + The one runtime setting a child does see is the state directory the + harness gives it (plan 034 T072), never the runner's own.""" from opendox.runtime.config import PREFIX, SETTING_NAMES + state_setting = PREFIX + "STATE_DIR" exported = {PREFIX + "INSTALL_MODE": "hosted", PREFIX + "OIDC_ISSUER": "https://issuer.example.invalid/realms/x", PREFIX + "OIDC_AUDIENCE": "fixture", - PREFIX + "DATABASE_URL": "postgresql://s@127.0.0.1:1/x"} + PREFIX + "DATABASE_URL": "postgresql://s@127.0.0.1:1/x", + state_setting: str(tmp_path / "runners-own-state")} for name, value in exported.items(): monkeypatch.setenv(name, value) blocker = tmp_path / "sibling-blocker" blocker.mkdir() - (blocker / "t070_env_probe.py").write_text(textwrap.dedent(""" + (blocker / "t070_env_probe.py").write_text(textwrap.dedent(f""" import json, os - print(json.dumps(sorted(n for n in os.environ if n.startswith("OPENDOX_"))), - flush=True) + print(json.dumps([sorted(n for n in os.environ if n.startswith("OPENDOX_")), + os.environ.get({state_setting!r})]), flush=True) """), encoding="utf-8") - child, status = run_module(tmp_path, "t070_env_probe") + child = Child(tmp_path, "t070_env_probe") + state_dir = child.state_dir + status = child.wait() assert status == 0, child.stderr_text() - seen = set(json.loads(child.stdout_text().strip().splitlines()[-1])) - assert not seen & set(SETTING_NAMES), sorted(seen & set(SETTING_NAMES)) + names, state_value = json.loads(child.stdout_text().strip().splitlines()[-1]) + assert set(names) & set(SETTING_NAMES) == {state_setting}, names + assert state_value == str(state_dir) != exported[state_setting] assert set(exported) <= set(SETTING_NAMES) + assert not state_dir.exists(), "the child's state directory outlived it" def test_a_child_that_ignores_the_interrupt_is_killed_at_the_deadline( From d1de1fd90402ab7f3fe65de4cabc67fe6a652040 Mon Sep 17 00:00:00 2001 From: Brett Heap <1513478+brettheap@users.noreply.github.com> Date: Fri, 2 Oct 2026 20:49:17 +0000 Subject: [PATCH 25/39] Fix round: the healthy-local status case sets its schema with make_conninfo (Copilot review) OPENDOX_TEST_DATABASE_URL may be libpq's keyword/value form as well as a URI (the runtime reads both, and schema_selected_by reads the schema out of both). The case appended "?options=..." to it. On the keyword/value form that suffix becomes part of dbname, so the case failed for a reason that was the fixture's spelling, while the fixtures themselves still connected. psycopg.conninfo.make_conninfo now sets "options" on either form. Measured with a keyword/value OPENDOX_TEST_DATABASE_URL: the case failed before and passes now, and it passes on the URI form both times. A mutant that drops the search_path option is killed on both forms. Lane: openxfactory-4 (openXfactory-4-openDox_extraction) Co-Authored-By: Claude Opus 5.5 (1M context) --- tests_runtime/test_install_mode.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/tests_runtime/test_install_mode.py b/tests_runtime/test_install_mode.py index 24d44eb6..0c5477a4 100644 --- a/tests_runtime/test_install_mode.py +++ b/tests_runtime/test_install_mode.py @@ -377,10 +377,14 @@ def _no_broker(_settings): "install, which has no broker") monkeypatch.setattr(oidc, "build_verifier", _no_broker) - joiner = "&" if "?" in postgres_dsn else "?" - scrubbed.setenv(PREFIX + "DATABASE_URL", - f"{postgres_dsn}{joiner}options=-c%20search_path%3D" - f"{database.schema}%2Cpublic") + # `make_conninfo`, NOT a `?options=` suffix: `OPENDOX_TEST_DATABASE_URL` + # may be libpq's keyword/value form as well as a URI, and a suffix on + # `… dbname=opendox` names the database `opendox?options=…` instead of + # selecting the schema (Copilot review of openDox-code#67). + from psycopg.conninfo import make_conninfo + + scrubbed.setenv(PREFIX + "DATABASE_URL", make_conninfo( + postgres_dsn, options=f"-c search_path={database.schema},public")) scrubbed.setenv(MODE, "local") code, evidence = _run(["runtime", "status", "--probe-timeout", "5"]) assert evidence["database"] == "reachable", evidence From 0488f5bd3d7fb10b1096ef91d6cee2e7bb6b2950 Mon Sep 17 00:00:00 2001 From: Brett Heap <1513478+brettheap@users.noreply.github.com> Date: Fri, 2 Oct 2026 20:58:33 +0000 Subject: [PATCH 26/39] Fix round: nothing is made through a path the tree check would refuse (Copilot review) Copilot's review at 19e32f0c: _prepare_directories made the missing postgres/run before the tree check judged the path. So a component the check refuses had already been written through: another user's link (the directories were made in its target), or a directory every user can write. In a sticky directory such as /tmp, another user could also put the state directory's name in place between the check and the mkdir. - What exists is judged first. _refuse_an_unsafe_tree(existing_only=True) asks the same rules of only what exists, the link-ownership loop first so a foreign link is named even when broken. The whole tree is still judged again after the directories are made, before the socket directory's chmod. - _make_private_directories opens the deepest existing directory and makes each missing component relative to its parent's descriptor. It opens each one O_NOFOLLOW and asks fstat that it is this user's alone before anything is made beneath it. A planted link, non-directory or foreign directory is the named refusal, never followed and never re-moded. - Each component is born 0700 under a umask of 077, which is put back after, so there is no chmod window at all. The six new cases fail against 19e32f0c's bundle.py and pass here. Seven mutants are killed, one of them only after the umask test was taught to assert the restore. Lane: openxfactory-4 (openXfactory-4-openDox_extraction) Co-Authored-By: Claude Opus 5.5 (1M context) --- src/opendox/runtime/bundle.py | 104 +++++++++++++++++------ tests_runtime/test_local_lifecycle.py | 114 +++++++++++++++++++++++--- 2 files changed, 185 insertions(+), 33 deletions(-) diff --git a/src/opendox/runtime/bundle.py b/src/opendox/runtime/bundle.py index f5ff03c0..bd7834d4 100644 --- a/src/opendox/runtime/bundle.py +++ b/src/opendox/runtime/bundle.py @@ -346,7 +346,7 @@ def _preexec() -> None: # pragma: no cover - runs in the child def _make_private_directories(leaf: Path) -> None: - """`leaf` and every missing directory above it, each created 0700. + """`leaf` and every missing directory above it, each born exactly 0700. `Path.mkdir(parents=True)` gives the directories it creates on the way the default mode less the umask, whatever mode the leaf is given. So @@ -354,20 +354,59 @@ def _make_private_directories(leaf: Path) -> None: create `.local` and `state` group-writable, and the tree check would then refuse the directories this install had just made (Copilot review of openDox-code#69). Each missing component is created here, one at a - time, and set to exactly 0700, whatever the umask is. A directory that - already exists is left as it is, and the tree check judges it. + time, under a umask of 077, so it is born 0700, whatever the user's + umask is, with no `chmod` after it. A directory that already exists is + left as it is, and the tree check judges it. + + NOTHING IS MADE THROUGH A PATH THAT WAS NOT JUDGED FIRST (Copilot review + of openDox-code#69). The caller has refused an unsafe EXISTING prefix + before this runs (`BundledServer._prepare_directories`), so the deepest + directory that exists is safe to open. Each missing component is then + made RELATIVE TO ITS PARENT'S DESCRIPTOR and opened with `O_NOFOLLOW` + before anything is made beneath it. A name that another user put there + first, in a sticky directory such as `/tmp`, is refused, never followed + or written through: a symbolic link, something that is not a directory, + or a directory that is not this user's alone. + + THE UMASK IS PROCESS-WIDE, and it is narrowed only for these few + `mkdir`s and then put back. A file another thread creates meanwhile can + only come out more private than asked, never less. """ - missing = [] - for directory in (leaf, *leaf.parents): - if os.path.lexists(directory): - break - missing.append(directory) - for directory in reversed(missing): - try: - directory.mkdir(mode=0o700) - except FileExistsError: - continue # made by a concurrent start; judged below - os.chmod(directory, 0o700) + uid = os.getuid() + missing: list[str] = [] + base = leaf + while not os.path.lexists(base): + missing.append(base.name) + base = base.parent + if not missing: + return + descriptor = os.open(base, os.O_RDONLY | os.O_DIRECTORY) + path = base + previous = os.umask(0o077) + try: + for name in reversed(missing): + path = path / name + try: + os.mkdir(name, 0o700, dir_fd=descriptor) + except FileExistsError: + pass # made first, by a concurrent start or by someone else: judged next + try: + child = os.open(name, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW, + dir_fd=descriptor) + except OSError: + info = os.stat(name, dir_fd=descriptor, follow_symlinks=False) + reason = _unsafe_because(info, uid=uid, own=True) + if reason is None: + raise + raise BundledServer._unsafe(path, reason) from None + os.close(descriptor) + descriptor = child + reason = _unsafe_because(os.fstat(descriptor), uid=uid, own=True) + if reason is not None: + raise BundledServer._unsafe(path, reason) + finally: + os.umask(previous) + os.close(descriptor) def os_user() -> str: @@ -555,13 +594,19 @@ def _prepare_directories(self) -> None: under its own state directory, and it is narrowed to 0700 whatever it was. """ + # JUDGED BEFORE ANY WRITE, AND AGAIN AFTER (Copilot review of + # openDox-code#69). What exists already is checked first, so no + # directory is made through a link, or beneath a directory, that the + # tree check would refuse, and `_make_private_directories` refuses a + # name someone else put in its way. The whole tree is then checked + # BEFORE THE CHMOD, which follows a symbolic link: a `run` placed + # there as a link would otherwise have its TARGET re-moded. + self._refuse_an_unsafe_tree(existing_only=True) _make_private_directories(self.bundle.socket_dir) - # CHECKED BEFORE THE CHMOD, which follows a symbolic link: a `run` - # placed there as a link would otherwise have its TARGET re-moded. self._refuse_an_unsafe_tree() os.chmod(self.bundle.socket_dir, 0o700) - def _refuse_an_unsafe_tree(self) -> None: + def _refuse_an_unsafe_tree(self, *, existing_only: bool = False) -> None: """The socket's whole path is this user's to change, or it is refused. The socket's directory is how this install's clients find ITS @@ -587,9 +632,26 @@ def _refuse_an_unsafe_tree(self) -> None: `OPENDOX_STATE_DIR` never holds `..` (`config.state_dir` refuses it), so the configured path's components are the ones the kernel walks. + + With `existing_only`, the same rules are asked of only what exists + yet. `_prepare_directories` asks that BEFORE it creates anything, + so the links are checked first, a broken one included (Copilot + review of openDox-code#69). """ uid = os.getuid() configured = self.bundle.state_dir + + def present(path: Path) -> bool: + return not existing_only or os.path.lexists(path) + + for component in (configured, *configured.parents): + if not present(component): + continue + info = os.lstat(component) + if stat.S_ISLNK(info.st_mode) and info.st_uid not in (uid, 0): + raise self._unsafe( + component, f"is a symbolic link owned by uid {info.st_uid}, " + "neither this user nor root, who could point it elsewhere") state = configured.resolve() tree = [state, state / BUNDLE_TREE[0], state / BUNDLE_TREE[0] / BUNDLE_TREE[1]] # AND THE DATA DIRECTORY, where one exists already, a broken link @@ -604,16 +666,12 @@ def _refuse_an_unsafe_tree(self) -> None: (path, False) for path in dict.fromkeys( [*state.parents, *configured.parents])] for directory, mine in checks: + if not present(directory): + continue info = os.lstat(directory) if mine else os.stat(directory) reason = _unsafe_because(info, uid=uid, own=mine) if reason is not None: raise self._unsafe(directory, reason) - for component in (configured, *configured.parents): - info = os.lstat(component) - if stat.S_ISLNK(info.st_mode) and info.st_uid not in (uid, 0): - raise self._unsafe( - component, f"is a symbolic link owned by uid {info.st_uid}, " - "neither this user nor root, who could point it elsewhere") @staticmethod def _unsafe(directory: Path, reason: str) -> BundleRefused: diff --git a/tests_runtime/test_local_lifecycle.py b/tests_runtime/test_local_lifecycle.py index 58696b0b..e8688393 100644 --- a/tests_runtime/test_local_lifecycle.py +++ b/tests_runtime/test_local_lifecycle.py @@ -498,26 +498,33 @@ def test_this_users_own_link_to_a_private_directory_is_accepted( assert "initdb" in str(caught.value), caught.value -def test_a_link_another_user_owns_is_refused( - monkeypatch, tmp_path: Path, short_state: Path) -> None: - """Another user's link could be pointed elsewhere after the check. A - non-root suite cannot create one, so its `lstat` is stood in, for that - one path only.""" - private = short_state / "private" - private.mkdir(mode=0o700) - link = short_state / "link" - link.symlink_to(private) +def _foreign_lstat(monkeypatch, *links: Path) -> None: + """`os.lstat` answers that each of `links` belongs to another user. A + non-root suite cannot create another user's link, so it is stood in for + those paths only.""" real_lstat = os.lstat def _lstat(path, *args, **kwargs): info = real_lstat(path, *args, **kwargs) - if Path(path) == link: + if Path(path) in links: fields = list(info) fields[4] = os.getuid() + 4242 # st_uid return os.stat_result(fields) return info monkeypatch.setattr(bundle_mod.os, "lstat", _lstat) + + +def test_a_link_another_user_owns_is_refused( + monkeypatch, tmp_path: Path, short_state: Path) -> None: + """Another user's link could be pointed elsewhere after the check. A + non-root suite cannot create one, so its `lstat` is stood in, for that + one path only.""" + private = short_state / "private" + private.mkdir(mode=0o700) + link = short_state / "link" + link.symlink_to(private) + _foreign_lstat(monkeypatch, link) server = _prepared(monkeypatch, tmp_path, link / "state") with pytest.raises(bundle_mod.BundleRefused) as caught: server.start() @@ -539,6 +546,8 @@ def test_missing_directories_are_created_0700_whatever_the_umask( try: with pytest.raises(bundle_mod.BundleRefused) as caught: server.start() + # The narrowed umask is the start's alone: the user's is put back. + assert os.umask(umask) == umask, "the start left its own umask in place" finally: os.umask(previous) assert "initdb" in str(caught.value), caught.value @@ -547,6 +556,91 @@ def test_missing_directories_are_created_0700_whatever_the_umask( assert stat.S_IMODE(directory.stat().st_mode) == 0o700, directory +@pytest.mark.parametrize("shape", ["foreign-link", "foreign-broken-link", + "open-ancestor", "link-in-open-dir"]) +def test_nothing_is_made_through_a_path_the_tree_check_refuses( + monkeypatch, tmp_path: Path, short_state: Path, shape: str) -> None: + """What exists is judged BEFORE anything is created (Copilot review of + #69). Before, the missing `postgres/run` was made through the path first + and refused only after, so it was created in a foreign link's target, or + beneath a directory every user can write.""" + private = short_state / "private" + private.mkdir(mode=0o700) + open_dir = short_state / "open" + open_dir.mkdir() + open_dir.chmod(0o777) + if shape in {"foreign-link", "foreign-broken-link"}: + target = private if shape == "foreign-link" else private / "gone" + state = short_state / "link" + state.symlink_to(target) + _foreign_lstat(monkeypatch, state) + expected, made = "symbolic link owned by", target / "postgres" + elif shape == "open-ancestor": + state = open_dir / "state" + expected, made = "not sticky", state + else: + (open_dir / "link").symlink_to(private) + state = open_dir / "link" / "state" + expected, made = "not sticky", private / "state" + server = _prepared(monkeypatch, tmp_path, state) + with pytest.raises(bundle_mod.BundleRefused) as caught: + server.start() + assert expected in str(caught.value), caught.value + assert not os.path.lexists(made), f"{made} was made before the refusal" + + +@pytest.mark.parametrize("shape", ["link", "foreign-directory"]) +def test_a_name_put_in_the_way_first_is_refused_never_followed( + monkeypatch, tmp_path: Path, short_state: Path, shape: str) -> None: + """The race in a sticky directory, `/tmp`'s shape (Copilot review of + #69): every user can create a name there, so another user can put the + state directory's name in place between the check and the `mkdir`. The + stand-in `mkdir` plays that user, once. The name is refused, and nothing + is made through it or beneath it, nor is it re-moded.""" + sticky = short_state / "sticky" + sticky.mkdir() + sticky.chmod(0o1777) + state = sticky / "state" + target = tmp_path / "somewhere-else" + target.mkdir(mode=0o755) + target.chmod(0o755) + real_mkdir, real_fstat = os.mkdir, os.fstat + planted: dict = {} + + def _mkdir(path, mode=0o777, *, dir_fd=None): + if dir_fd is not None and path == state.name and not planted: + if shape == "link": + os.symlink(target, path, dir_fd=dir_fd) + else: + real_mkdir(path, 0o755, dir_fd=dir_fd) + os.chmod(state, 0o755) + planted["inode"] = os.lstat(state).st_ino + raise FileExistsError(path) + return real_mkdir(path, mode, dir_fd=dir_fd) + + def _fstat(descriptor): + info = real_fstat(descriptor) + if info.st_ino == planted.get("inode"): + fields = list(info) + fields[4] = os.getuid() + 4242 # st_uid + return os.stat_result(fields) + return info + + monkeypatch.setattr(bundle_mod.os, "mkdir", _mkdir) + monkeypatch.setattr(bundle_mod.os, "fstat", _fstat) + server = _prepared(monkeypatch, tmp_path, state) + with pytest.raises(bundle_mod.BundleRefused) as caught: + server.start() + assert planted, "the stand-in never ran: the state directory was not made by descriptor" + message = str(caught.value) + assert str(state) in message, message + assert ("is a symbolic link" if shape == "link" + else "is owned by uid") in message, message + beneath = target if shape == "link" else state + assert not (beneath / "postgres").exists(), "made beneath the planted name" + assert stat.S_IMODE(os.stat(beneath).st_mode) == 0o755, "the planted name was re-moded" + + @pytest.mark.parametrize("shape", ["link", "broken-link", "open"]) def test_a_data_directory_that_is_not_this_installs_own_is_refused( monkeypatch, tmp_path: Path, short_state: Path, shape: str) -> None: From 105f2f1222fbdd19d245110d443c3fc6475ebeb9 Mon Sep 17 00:00:00 2001 From: Brett Heap <1513478+brettheap@users.noreply.github.com> Date: Fri, 2 Oct 2026 22:22:13 +0000 Subject: [PATCH 27/39] Fix round: no runtime setting the runner exports reaches a tests_runtime case (Copilot review) Since T070 the runtime reads a selector, OPENDOX_INSTALL_MODE. A runner that exported OPENDOX_INSTALL_MODE=local, as a local install's own shell would, turned every hosted case that leaves it unset into a configuration refusal. Measured: 20 cases failed, 18 in test_runtime_cli.py and 2 in test_migrations_apply.py (status then has no "settings" key, so the case raises KeyError). The scrubbed fixture covered only test_install_mode.py. tests_runtime/conftest.py gains an autouse fixture that clears every name in opendox.runtime.config.SETTING_NAMES before each case, and a case that wants one sets it. OPENDOX_TEST_DATABASE_URL is not a runtime setting, so the harness's own DSN is kept. The production refusal is unchanged. Measured full-suite results: - with OPENDOX_INSTALL_MODE=local and a broker issuer and audience exported: 3126 selected, 3115 passed, 11 skipped, 0 failed; - in a clean environment: the same. A mutant that clears nothing is killed: the 20 cases fail. Lane: openxfactory-4 (openXfactory-4-openDox_extraction) Co-Authored-By: Claude Opus 5.5 (1M context) --- tests_runtime/conftest.py | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/tests_runtime/conftest.py b/tests_runtime/conftest.py index 2c6e0b7b..55553539 100644 --- a/tests_runtime/conftest.py +++ b/tests_runtime/conftest.py @@ -57,6 +57,31 @@ ) +@pytest.fixture(autouse=True) +def _no_inherited_runtime_setting(monkeypatch: pytest.MonkeyPatch) -> None: + """No `OPENDOX_*` runtime setting the runner exports reaches a case. + + Since plan 034 T070 the runtime reads a selector, `OPENDOX_INSTALL_MODE`, + and a runner that exports `local` (as a local install's own shell would) + turned every hosted case that leaves it unset into a configuration + refusal: 20 cases in `test_runtime_cli.py` and `test_migrations_apply.py` + (measured; Copilot review of openDox-code#67). So every name the runtime + reads (`opendox.runtime.config.SETTING_NAMES`) is cleared before each + case, and a case that wants one sets it. `OPENDOX_TEST_DATABASE_URL` is + not one of them (see `TEST_DSN_ENV`), so the harness's own DSN is kept. + The production refusal is unchanged. + + An `opendox` that cannot be imported leaves nothing to clear: the case + then fails on its own import, which is the failure worth seeing. + """ + try: + from opendox.runtime.config import SETTING_NAMES + except ImportError: + return + for name in SETTING_NAMES: + monkeypatch.delenv(name, raising=False) + + def in_ci() -> bool: """Whether this run is CI's, by CI's own variable. From 3426c753eff5701987b808ef2296ba74161fa7c2 Mon Sep 17 00:00:00 2001 From: Brett Heap <1513478+brettheap@users.noreply.github.com> Date: Fri, 2 Oct 2026 22:29:58 +0000 Subject: [PATCH 28/39] Fix round 12: the auth files are exactly 0600, and the cluster runs on its own files (Copilot review) Two threads from Copilot's reviews at 6eb0bbdb and fedfa75d. write_authentication's 0600 was only a creation request. The umask filters it, and it changes nothing about a file that already exists. So a 0277 umask left a 0400 file, a temporary file an interrupted start left at the same name kept its 0644, and a link left there was written through. Now: - the stale temporary is unlinked first (a link itself, never its target); - the new one is opened O_CREAT|O_EXCL|O_NOFOLLOW; - its descriptor is fchmod-ed to 0600 before anything is written. A link raced in after the unlink is a FileExistsError, never followed. An existing cluster's postgresql.conf could point data_directory, hba_file and ident_file elsewhere: at an outside trust file the rewritten files would never replace, or at a cluster outside the state directory. The launch now pins all three on the command line, which outranks every configuration file. New cases: - tests_runtime/test_local_lifecycle.py: the umask, a stale 0644 temporary, a stale link, and a raced link; - tests_runtime/test_bundled_postgres.py: a real cluster whose postgresql.conf points all three elsewhere still shows its own three and serves peer. The first four fail against fedfa75d's bundle.py, and so does the real-cluster case. Six mutants are all killed (one only after the raced-link case was added). Lane: openxfactory-4 (openXfactory-4-openDox_extraction) Co-Authored-By: Claude Opus 5.5 (1M context) --- src/opendox/runtime/bundle.py | 42 ++++++++++++++--- tests_runtime/test_bundled_postgres.py | 33 +++++++++++++ tests_runtime/test_local_lifecycle.py | 65 ++++++++++++++++++++++++++ 3 files changed, 134 insertions(+), 6 deletions(-) diff --git a/src/opendox/runtime/bundle.py b/src/opendox/runtime/bundle.py index bd7834d4..2da965d3 100644 --- a/src/opendox/runtime/bundle.py +++ b/src/opendox/runtime/bundle.py @@ -470,15 +470,34 @@ def write_authentication(data_dir: Path, user: str) -> None: Before EVERY launch, not only after `initdb`: a data directory an earlier build initialized, or a file edited by hand, is brought back to the one configuration this install runs with. + + THE 0600 IS SET, NOT ASKED FOR (Copilot review of openDox-code#69). An + `open` mode is only a creation request: the umask filters it, and it + changes nothing about a file that exists already. So a temporary file an + interrupted start left at the same name (a 0644 one, or a symbolic link) + is removed first. The new one is created EXCLUSIVELY and without following + a link, and its descriptor is set to exactly 0600 before anything is + written into it. """ for name, content in authentication_files(user).items(): target = data_dir / name temporary = data_dir / f".{name}.opendox-{os.getpid()}" - descriptor = os.open(temporary, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, - 0o600) - with os.fdopen(descriptor, "w", encoding="utf-8") as handle: - handle.write(content) - os.replace(temporary, target) + with contextlib.suppress(FileNotFoundError): + os.unlink(temporary) # an interrupted start's; a link itself, never its target + descriptor = os.open( + temporary, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, 0o600) + try: + os.fchmod(descriptor, 0o600) + with os.fdopen(descriptor, "w", encoding="utf-8") as handle: + descriptor = -1 # the handle closes it now + handle.write(content) + os.replace(temporary, target) + except BaseException: + if descriptor >= 0: + os.close(descriptor) + with contextlib.suppress(FileNotFoundError): + os.unlink(temporary) + raise def _unsafe_because(info: os.stat_result, *, uid: int, own: bool) -> str | None: @@ -765,7 +784,18 @@ def _launch(self, binaries: Path) -> None: # port can stand in for this server, and nothing off this # machine can reach it. "-c", "listen_addresses=", - "-c", "unix_socket_permissions=0700"], + "-c", "unix_socket_permissions=0700", + # THE CLUSTER'S OWN FILES, PINNED (Copilot review of + # openDox-code#69). An existing cluster's + # `postgresql.conf` can point `data_directory`, `hba_file` + # and `ident_file` elsewhere: at an outside `trust` file + # that the files just rewritten would never replace, or at + # a cluster outside the state directory. The command line + # outranks every configuration file, so these three are + # the data directory and the two files written above. + "-c", f"data_directory={self.bundle.data_dir}", + "-c", f"hba_file={self.bundle.data_dir / 'pg_hba.conf'}", + "-c", f"ident_file={self.bundle.data_dir / 'pg_ident.conf'}"], stdin=subprocess.DEVNULL, stdout=log, stderr=subprocess.STDOUT, env=_child_environment(), # ITS OWN SESSION, so a terminal's Ctrl-C reaches this process diff --git a/tests_runtime/test_bundled_postgres.py b/tests_runtime/test_bundled_postgres.py index bae2a7f5..e650275f 100644 --- a/tests_runtime/test_bundled_postgres.py +++ b/tests_runtime/test_bundled_postgres.py @@ -561,6 +561,39 @@ def test_an_older_trust_cluster_is_brought_back_to_peer_on_start( assert method == f"peer:{bundle_mod.os_user()}", method +def test_a_cluster_whose_configuration_points_elsewhere_runs_on_its_own_files( + state_dir: Path, tmp_path: Path) -> None: + """An existing cluster's `postgresql.conf` can point `hba_file` and + `ident_file` at outside files, a `trust` one say, and `data_directory` + at a cluster that is not this install's (Copilot review of #69). The + launch pins all three on the command line, which outranks the file. So + the server reads the two files this install wrote, from its own data + directory, and every connection is still peer.""" + settings = config.load_settings({MODE: "local", STATE: str(state_dir)}) + bundle_mod.BundledServer(settings).start().stop() + data = config.DatabaseBundle(state_dir).data_dir + outside = tmp_path / "outside" + outside.mkdir() + (outside / "trust_hba.conf").write_text("local all all trust\n", encoding="utf-8") + (outside / "ident.conf").write_text("", encoding="utf-8") + with (data / "postgresql.conf").open("a", encoding="utf-8") as conf: + conf.write(f"\nhba_file = '{outside / 'trust_hba.conf'}'\n" + f"ident_file = '{outside / 'ident.conf'}'\n" + f"data_directory = '{outside / 'no-such-cluster'}'\n") + with bundle_mod.BundledServer(settings) as server: + import psycopg + + with _owner(server) as conn: + shown = {name: conn.execute(f"show {name}").fetchone()[0] + for name in ("data_directory", "hba_file", "ident_file")} + with psycopg.connect(server.bundle.served_dsn) as conn: + method = conn.execute("select system_user").fetchone()[0] + assert shown == {"data_directory": str(data), + "hba_file": str(data / "pg_hba.conf"), + "ident_file": str(data / "pg_ident.conf")}, shown + assert method == f"peer:{bundle_mod.os_user()}", method + + def test_migrate_under_the_local_mode_uses_the_bundle_and_refuses_a_dsn( state_dir: Path) -> None: """`runtime migrate` is part of the same install: it reads the bundle's diff --git a/tests_runtime/test_local_lifecycle.py b/tests_runtime/test_local_lifecycle.py index e8688393..7bc74535 100644 --- a/tests_runtime/test_local_lifecycle.py +++ b/tests_runtime/test_local_lifecycle.py @@ -759,6 +759,71 @@ def test_the_files_are_written_0600_and_replace_what_was_there( assert sorted(p.name for p in tmp_path.iterdir()) == ["pg_hba.conf", "pg_ident.conf"] +@pytest.mark.parametrize("shape", ["umask", "stale-0644", "stale-link"]) +def test_the_files_are_exactly_0600_whatever_the_umask_or_a_stale_temporary( + tmp_path: Path, shape: str) -> None: + """The 0600 is SET, not asked for (Copilot review of #69). A umask that + takes the owner's write bit would leave a 0400 file. A temporary file an + interrupted start left at the same name would keep its own mode, 0644, + and a link left there would be written through to its target.""" + data = tmp_path / "data" + data.mkdir(mode=0o700) + outside = tmp_path / "outside.conf" + outside.write_text("untouched\n", encoding="utf-8") + outside.chmod(0o644) + for name in bundle_mod.authentication_files("alice"): + stale = data / f".{name}.opendox-{os.getpid()}" + if shape == "stale-0644": + stale.write_text("local all all trust\n", encoding="utf-8") + stale.chmod(0o644) + elif shape == "stale-link": + stale.symlink_to(outside) + previous = os.umask(0o277 if shape == "umask" else 0o022) + try: + bundle_mod.write_authentication(data, "alice") + finally: + os.umask(previous) + for name, content in bundle_mod.authentication_files("alice").items(): + written = data / name + assert not written.is_symlink(), name + assert written.read_text(encoding="utf-8") == content, name + assert stat.S_IMODE(written.stat().st_mode) == 0o600, name + assert sorted(p.name for p in data.iterdir()) == ["pg_hba.conf", "pg_ident.conf"] + assert outside.read_text(encoding="utf-8") == "untouched\n" + assert stat.S_IMODE(outside.stat().st_mode) == 0o644, "a stale link's target was re-moded" + + +def test_a_link_put_at_the_temporary_name_after_its_removal_is_never_followed( + monkeypatch, tmp_path: Path) -> None: + """The temporary file is created EXCLUSIVELY and without following a + link, so a link that appears at its name between the removal and the + open is a refusal, never a file written through (Copilot review of + #69). The stand-in `unlink` plays that race once.""" + data = tmp_path / "data" + data.mkdir(mode=0o700) + outside = tmp_path / "outside.conf" + outside.write_text("untouched\n", encoding="utf-8") + outside.chmod(0o644) + real_unlink = os.unlink + raced: list = [] + + def _unlink(path, *args, **kwargs): + try: + real_unlink(path, *args, **kwargs) + finally: + if not raced and Path(path).name.startswith(".pg_hba.conf.opendox-"): + raced.append(path) + os.symlink(outside, path) + + monkeypatch.setattr(bundle_mod.os, "unlink", _unlink) + with pytest.raises(FileExistsError): + bundle_mod.write_authentication(data, "alice") + assert raced, "the stand-in never ran" + assert outside.read_text(encoding="utf-8") == "untouched\n" + assert stat.S_IMODE(outside.stat().st_mode) == 0o644, "the link's target was re-moded" + assert not (data / "pg_hba.conf").exists() + + @pytest.mark.parametrize("name", ["/regex", 'quo"te', "has space", "hash#tag", ""]) def test_an_os_user_name_the_map_cannot_hold_plainly_is_refused( monkeypatch: pytest.MonkeyPatch, name: str) -> None: From a9854078c0891d58e5483cae6f515077a9829987 Mon Sep 17 00:00:00 2001 From: Brett Heap <1513478+brettheap@users.noreply.github.com> Date: Fri, 2 Oct 2026 22:46:08 +0000 Subject: [PATCH 29/39] T072: the in-process local cases give themselves a short state directory The doxBench entrypoint fixture and test_projection_seams.py's empty-option case run `generate-and-open --local` in process, with the bundled server stood in. They scrubbed the runtime settings, and so they fell back to the runner's default state directory, under XDG_STATE_HOME or the home directory. When that default is too long for a Unix socket, configuration refuses it (13.1) before the case reaches what it tests. Measured with a 90-character XDG_STATE_HOME: 4 errors and 1 failure. Each now sets OPENDOX_STATE_DIR to a fresh, short directory under /tmp and removes it after the case. Nothing is made in it, because the database is stood in. - Under that long XDG_STATE_HOME, with OPENDOX_INSTALL_MODE=local exported, both modules pass whole, and both mutants without the private state directory are killed. - In a clean environment both modules pass whole. Lane: openxfactory-4 (openXfactory-4-openDox_extraction) Co-Authored-By: Claude Opus 5.5 (1M context) --- tests/test_doxbench_entrypoint.py | 22 +++++++++++++++++----- tests/test_projection_seams.py | 19 +++++++++++++++---- 2 files changed, 32 insertions(+), 9 deletions(-) diff --git a/tests/test_doxbench_entrypoint.py b/tests/test_doxbench_entrypoint.py index ade38784..e41659f2 100644 --- a/tests/test_doxbench_entrypoint.py +++ b/tests/test_doxbench_entrypoint.py @@ -51,6 +51,7 @@ import os import shutil import subprocess +import tempfile from pathlib import Path import pytest @@ -146,6 +147,14 @@ def _capture(*args, **kwargs): # from the shell would be refused beside the local mode, by design. for name in runtime_config.SETTING_NAMES: monkeypatch.delenv(name, raising=False) + # ITS OWN SHORT STATE DIRECTORY, so the configuration never reads the + # runner's: a local install's default is under `XDG_STATE_HOME` or the + # home directory, and one too long for a Unix socket is refused (13.1) + # before the entrypoint is reached. Nothing is made in it, because the + # database is stood in below, and it is removed after the case. + state = Path(tempfile.mkdtemp(prefix="odx-e-", + dir="/tmp" if os.path.isdir("/tmp") else None)) + monkeypatch.setenv(runtime_config.PREFIX + "STATE_DIR", str(state)) # AND THE LOCAL INSTALL'S DATABASE IS STOOD IN, with a tripwire of its own # (plan 034 T072). A local `generate-and-open` starts its bundled # PostgreSQL server before it serves, and these cases are about the model @@ -182,11 +191,14 @@ def report(self): "--model-session-root", str(session_root), "--no-validate", "--no-open", "--no-serve", ]) - rc = cli_mod.cmd_generate_and_open(args, opener=lambda url: None) - assert rc == 0, "the entrypoint did not complete" - assert len(bundles) == 1, "a LOCAL entrypoint run must own one database" - assert built, "the entrypoint never reached build_server" - yield _handler_class(built[-1]), spawned, session_root + try: + rc = cli_mod.cmd_generate_and_open(args, opener=lambda url: None) + assert rc == 0, "the entrypoint did not complete" + assert len(bundles) == 1, "a LOCAL entrypoint run must own one database" + assert built, "the entrypoint never reached build_server" + yield _handler_class(built[-1]), spawned, session_root + finally: + shutil.rmtree(state, ignore_errors=True) # -------------------------------------------------------------------------- diff --git a/tests/test_projection_seams.py b/tests/test_projection_seams.py index 77536de8..01bad88b 100644 --- a/tests/test_projection_seams.py +++ b/tests/test_projection_seams.py @@ -46,6 +46,7 @@ import stat import subprocess import sys +import tempfile import textwrap import threading import types @@ -1401,10 +1402,17 @@ def test_generate_and_open_refuses_an_empty_source_option_before_its_run_dir( bundled server (plan 034 T072): a refused option costs no database start. A tripwire stands in for the server, so a regression neither starts one nor passes.""" - from opendox.runtime.config import SETTING_NAMES + from opendox.runtime.config import PREFIX, SETTING_NAMES for name in SETTING_NAMES: monkeypatch.delenv(name, raising=False) + # Its own short state directory, so the configuration never reads the + # runner's state home, whose default may be too long for a Unix socket + # (13.1) and refused before the option is reached. The tripwire below + # means nothing is made in it. + state = Path(tempfile.mkdtemp(prefix="odx-e-", + dir="/tmp" if os.path.isdir("/tmp") else None)) + monkeypatch.setenv(PREFIX + "STATE_DIR", str(state)) started: list = [] class _Tripwire: @@ -1425,9 +1433,12 @@ def stop(self) -> None: run_dir = tmp_path / "run" # `--local`: the single-user install. Since plan 034 T070 an unflagged run # is HOSTED, and its issuer refusal would come first. - rc = cli.main(["generate-and-open", "--local", "--repo-root", str(repo), - "--repository", "garden", "--run-dir", str(run_dir), - "--no-open", "--no-serve", "--possibles", ""]) + try: + rc = cli.main(["generate-and-open", "--local", "--repo-root", str(repo), + "--repository", "garden", "--run-dir", str(run_dir), + "--no-open", "--no-serve", "--possibles", ""]) + finally: + shutil.rmtree(state, ignore_errors=True) assert rc == 1 assert ("generate-and-open refused: --possibles was given an empty path" in capsys.readouterr().err) From e214477dc58727ddaafcda745b65d330790011ca Mon Sep 17 00:00:00 2001 From: Brett Heap <1513478+brettheap@users.noreply.github.com> Date: Fri, 2 Oct 2026 23:02:02 +0000 Subject: [PATCH 30/39] Adversarial review M1: a comma in the state directory is refused MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit PostgreSQL splits -k (unix_socket_directories) on commas. libpq splits a host on them once the DSN's percent-encoding is decoded. So a state directory such as "…/c2,…/ev" made the server put its sockets in two directories nothing had checked, one of them 0777, while the checked 0700 socket directory stayed empty (reproduced by the review). config.database_bundle, the one place every bundle is derived (from OPENDOX_STATE_DIR, XDG_STATE_HOME or HOME), now refuses a comma at configuration. The refusal names where the directory came from and does not repeat the value. New case: the refusal from each of the three sources, through load_settings and load_migration_settings, and the message never holds the value. A mutant without the refusal is killed. The review's own state directory is now refused before anything starts. Lane: openxfactory-4 (openXfactory-4-openDox_extraction) Co-Authored-By: Claude Opus 5.5 (1M context) --- src/opendox/runtime/config.py | 21 ++++++++++++++++++++- tests_runtime/test_bundled_postgres.py | 22 ++++++++++++++++++++++ 2 files changed, 42 insertions(+), 1 deletion(-) diff --git a/src/opendox/runtime/config.py b/src/opendox/runtime/config.py index e160b5e3..c1d9ea56 100644 --- a/src/opendox/runtime/config.py +++ b/src/opendox/runtime/config.py @@ -1778,8 +1778,27 @@ def state_dir(env: Mapping[str, str] | None = None) -> Path: def database_bundle(state: Path) -> DatabaseBundle: - """The bundle under `state`, refusing a socket path the kernel cannot bind.""" + """The bundle under `state`, refusing a socket path the kernel cannot bind. + + A COMMA IS REFUSED, wherever the state directory came from + (`OPENDOX_STATE_DIR`, `XDG_STATE_HOME` or the home directory), because + both ends read the socket directory as a LIST. PostgreSQL splits `-k` + (`unix_socket_directories`) on commas, and libpq splits a `host` on them + once the DSN's percent-encoding is decoded. So `…/a,…/b` made the server + put its sockets in two directories nothing here had checked, one of + them anyone could write, while the checked 0700 one stayed empty + (adversarial review of openDox-code#69). The value is not repeated: + the refusal names where it came from, and that is enough to find it. + """ bundle = DatabaseBundle(state_dir=state) + if "," in str(state): + raise ConfigurationError( + "the state directory's path holds a `,`. PostgreSQL reads its " + "socket directories, and libpq its hosts, as comma-separated " + "lists, so a comma would split this install's one socket " + "directory into two it never checked. Choose a state directory " + f"without one: {PREFIX}STATE_DIR or, where that is unset, " + "XDG_STATE_HOME or HOME") length = len(os.fsencode(str(bundle.socket_path))) if length > UNIX_SOCKET_PATH_MAX: raise ConfigurationError( diff --git a/tests_runtime/test_bundled_postgres.py b/tests_runtime/test_bundled_postgres.py index e650275f..2ec65b4d 100644 --- a/tests_runtime/test_bundled_postgres.py +++ b/tests_runtime/test_bundled_postgres.py @@ -301,6 +301,28 @@ def test_a_state_dir_too_long_for_a_unix_socket_is_refused_naming_it() -> None: assert STATE in str(caught.value) and "socket" in str(caught.value) +@pytest.mark.parametrize("variable", [STATE, "XDG_STATE_HOME", "HOME"]) +def test_a_comma_in_the_state_dir_is_refused_without_repeating_it( + monkeypatch, variable: str) -> None: + """PostgreSQL splits its socket directories, and libpq its hosts, on a + comma, so one would split the socket's one checked directory into two + unchecked ones (adversarial review of #69). Refused at configuration, + from whichever setting it came, and the value is not repeated.""" + value = "/tmp/odx-c1,/tmp/odx-c2-secretish" + env = {MODE: "local"} + if variable == "HOME": + monkeypatch.setenv("HOME", value) # `Path.home()` reads the process's + else: + env[variable] = value + with pytest.raises(config.ConfigurationError) as caught: + config.load_settings(env) + message = str(caught.value) + assert "`,`" in message and STATE in message, message + assert "secretish" not in message and value not in message, message + with pytest.raises(config.ConfigurationError): + config.load_migration_settings(env) + + def test_a_relative_state_dir_is_refused_naming_it() -> None: with pytest.raises(config.ConfigurationError) as caught: config.load_settings({MODE: "local", STATE: "var/state"}) From c4f5df3c0fd956b6373b513787834a7693d26142 Mon Sep 17 00:00:00 2001 From: Brett Heap <1513478+brettheap@users.noreply.github.com> Date: Fri, 2 Oct 2026 23:05:10 +0000 Subject: [PATCH 31/39] Adversarial review L1: the carrier is pinned below 0.7, and another major is refused by name pixeltable-pgserver>=0.6.0 had no ceiling. bundle.py runs the PostgreSQL 16 under pginstall/, upstream's default is already 18, and a cluster is opened only by the major that made it. Brett has ruled that release 1 publishes to PyPI, which makes an unbounded server dependency a property of every user's data. - pyproject.toml's local extra is now pixeltable-pgserver>=0.6.0,<0.7. The lock's ==0.6.0 sits inside that range. The local-extra case asserts both. - Before an existing cluster is used, BundledServer asks the server's own `postgres --version` against the cluster's PG_VERSION. Another major, or a server that does not say, is the named refusal, before anything is written into the data directory. New case: an existing PostgreSQL 16 cluster, with a stand-in server that says 17.2 (refused, both majors named), 16.14 (it goes on to the launch), or nothing (refused). All four mutants are killed: the check skipped, another major accepted, a silent server accepted, the ceiling dropped. Lane: openxfactory-4 (openXfactory-4-openDox_extraction) Co-Authored-By: Claude Opus 5.5 (1M context) --- pyproject.toml | 15 ++++++++--- src/opendox/runtime/bundle.py | 35 ++++++++++++++++++++++++++ tests_runtime/test_bundled_postgres.py | 8 ++++-- tests_runtime/test_local_lifecycle.py | 28 +++++++++++++++++++++ 4 files changed, 80 insertions(+), 6 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 22c71a72..c8794fb1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -136,12 +136,19 @@ test = [ # * about 24.7 MB per wheel, because it carries the two server majors; # * Apache-2.0 for the package; the PostgreSQL License for the server. # `>=0.6.0` is the release this package has been exercised against, the same -# rule every floor in this file is set by. Its own requirements (fasteners, -# platformdirs, psutil, typing-extensions) arrive with it and are imported by -# nothing here. +# rule every floor in this file is set by. `<0.7` IS A CEILING, which no other +# requirement here carries, because this one is a DATABASE SERVER whose major +# is a property of the user's data: `bundle.py` runs the PostgreSQL 16 under +# `pginstall/`, and a cluster is opened only by the major that made it. +# Upstream's own default is already 18, so a later release may move what +# `pginstall/` holds; the ceiling keeps that a deliberate act of this package +# (adversarial review of openDox-code#69), and `bundle.py` still refuses, by +# name, a server whose major disagrees with the cluster's `PG_VERSION`. Its +# own requirements (fasteners, platformdirs, psutil, typing-extensions) arrive +# with it and are imported by nothing here. local = [ "opendox[runtime]", - "pixeltable-pgserver>=0.6.0", + "pixeltable-pgserver>=0.6.0,<0.7", ] # THE RUNTIME EXTRA — `split-opendox-two-layer-product` § 3.5, RULED Q2 diff --git a/src/opendox/runtime/bundle.py b/src/opendox/runtime/bundle.py index 2da965d3..e872c00f 100644 --- a/src/opendox/runtime/bundle.py +++ b/src/opendox/runtime/bundle.py @@ -70,6 +70,7 @@ import ctypes import importlib.util import os +import re import shutil import signal import stat @@ -722,6 +723,7 @@ def _initialize(self, binaries: Path) -> None: """ data = self.bundle.data_dir if (data / "PG_VERSION").is_file(): + self._refuse_another_major(binaries) return if data.exists(): try: @@ -748,6 +750,39 @@ def _initialize(self, binaries: Path) -> None: shutil.rmtree(attempt, ignore_errors=True) raise + def _refuse_another_major(self, binaries: Path) -> None: + """An existing cluster is opened only by the major that made it. + + PostgreSQL refuses another major's data directory itself, but only + from inside a launch, where the reason reaches nobody but the log. + The carrier's `pginstall/` is PostgreSQL 16 today, and upstream's + default is already 18 (adversarial review of openDox-code#69). So + the server's own `postgres --version` is asked first, against the + cluster's `PG_VERSION`, and a disagreement is the named refusal, + before anything is written into the data directory. + """ + data = self.bundle.data_dir + cluster = (data / "PG_VERSION").read_text(encoding="utf-8").strip() + done = subprocess.run( + [str(binaries / "postgres"), "--version"], env=_child_environment(), + capture_output=True, text=True, timeout=START_TIMEOUT_SECONDS) + found = re.search(r"\(PostgreSQL\) (\d+)", done.stdout or "") + if found is None: + raise BundleRefused( + f"the bundled `postgres` under {binaries} does not say which " + f"PostgreSQL it is (`postgres --version` exited " + f"{done.returncode}), so it is not given {data}, a " + f"PostgreSQL {cluster} cluster; reinstall the `local` extra") + if found.group(1) != cluster: + raise BundleRefused( + f"{data} holds a PostgreSQL {cluster} cluster and the bundled " + f"server is PostgreSQL {found.group(1)}: a cluster is opened " + "only by the major version that made it. Reinstall the " + f"`local` extra this install was made with (`{SERVER_DISTRIBUTION}` " + "is pinned below 0.7 for this reason), or move the data " + "directory aside, and lose its coordination state, to start " + "a new one") + def _remove_abandoned_attempts(self) -> None: """Every initialization attempt whose process no longer exists.""" for candidate in self.bundle.data_dir.parent.glob( diff --git a/tests_runtime/test_bundled_postgres.py b/tests_runtime/test_bundled_postgres.py index 2ec65b4d..ce87ed02 100644 --- a/tests_runtime/test_bundled_postgres.py +++ b/tests_runtime/test_bundled_postgres.py @@ -258,12 +258,16 @@ def test_the_local_extra_carries_the_runtime_and_the_server_and_test_joins_it( # the carrier RULED on openxFactory#656 `5916000030` item 2 assert any(req.startswith("pixeltable-pgserver") for req in extras["local"]) assert not any(req.startswith("pgserver") for req in extras["local"]) + # WITH A CEILING (adversarial review of #69): the carrier's `pginstall/` + # major is a property of the user's data, so no later release moves it + # without this package's say. The lock's pin sits inside the range. + assert "pixeltable-pgserver>=0.6.0,<0.7" in extras["local"], extras["local"] assert "opendox[local]" in extras["test"], ( "F9.1 installs `.[test]` alone; without the local extra there, this " "suite could not start the server it tests") lock = (ROOT / "constraints-cpython312-linux.txt").read_text() - assert re.search(r"(?m)^pixeltable-pgserver==", lock), \ - "the lock does not pin pixeltable-pgserver" + assert re.search(r"(?m)^pixeltable-pgserver==0\.6\.\d+$", lock), \ + "the lock does not pin pixeltable-pgserver inside >=0.6.0,<0.7" assert not re.search(r"(?m)^pgserver==", lock), "the lock still pins pgserver" files = project["tool"]["setuptools"]["data-files"] assert files == {"share/opendox/migrations": ["migrations/*.sql"]} diff --git a/tests_runtime/test_local_lifecycle.py b/tests_runtime/test_local_lifecycle.py index 7bc74535..ea1bdfa0 100644 --- a/tests_runtime/test_local_lifecycle.py +++ b/tests_runtime/test_local_lifecycle.py @@ -890,6 +890,34 @@ def _target_of_initdb() -> str: return 'while [ "$1" != "-D" ]; do shift; done; T="$2"' +@pytest.mark.parametrize("says", ["17.2", "16.14", ""]) +def test_an_existing_cluster_is_opened_only_by_its_own_major( + monkeypatch, tmp_path: Path, short_state: Path, says: str) -> None: + """The server's `postgres --version` against the cluster's `PG_VERSION` + (adversarial review of #69). Another major, or a server that does not + say, is the named refusal, before anything is written into the data + directory. The same major goes on to the launch, which the stand-in + fails on purpose.""" + data = short_state / "postgres" / "data" + data.mkdir(parents=True, mode=0o700) + (short_state / "postgres").chmod(0o700) + (data / "PG_VERSION").write_text("16\n", encoding="utf-8") + answer = f'echo "postgres (PostgreSQL) {says}"' if says else "true" + server = _server(monkeypatch, tmp_path, short_state, initdb="exit 1", + postgres=f'if [ "$1" = --version ]; then {answer}; exit 0; fi\nexit 3') + with pytest.raises(bundle_mod.BundleRefused) as caught: + server.start() + message = str(caught.value) + if says == "17.2": + assert "PostgreSQL 16 cluster" in message and "PostgreSQL 17:" in message, message + elif says == "": + assert "does not say which PostgreSQL it is" in message, message + else: + assert "exited during start (exit 3)" in message, message + if says != "16.14": + assert not (data / "pg_hba.conf").exists(), "wrote into another major's cluster" + + def test_an_initdb_that_dies_midway_leaves_no_data_directory( monkeypatch, tmp_path: Path, short_state: Path) -> None: """The half-built cluster: `PG_VERSION` written, then the run fails. The From b2d80e94fd4fcc4af7afc3caba11ca095d42c3f4 Mon Sep 17 00:00:00 2001 From: Brett Heap <1513478+brettheap@users.noreply.github.com> Date: Fri, 2 Oct 2026 23:06:12 +0000 Subject: [PATCH 32/39] Adversarial review L3: the directory creation starts from is judged by its descriptor _make_private_directories opened its base with O_DIRECTORY, which follows a link (allowed on the configured path), and never judged the descriptor. So its own guard did not cover the very directory it writes into, which is round 11's rule ("nothing is made through a path that was not judged first"), leaving that to the path-wise check in front of it. The opened base is now fstat-judged with the tree check's own question, _unsafe_because, before the first mkdir. It must be this user's alone where it is the state directory or under it. Otherwise it must be this user's or root's, with any write by others only behind the sticky bit. A new keyword, `state`, tells the function which is which. New case: the function is asked directly, with no path-wise check in front, for a 0777 base reached through a link, a 0777 base, a link that is itself the state directory to a 0777 one, and a sticky state directory. Each is refused, and nothing is made. Two mutants are killed: the base not judged, and the state directory judged as an ancestor (killed by the sticky-state shape). Lane: openxfactory-4 (openXfactory-4-openDox_extraction) Co-Authored-By: Claude Opus 5.5 (1M context) --- src/opendox/runtime/bundle.py | 20 ++++++++++++++++-- tests_runtime/test_local_lifecycle.py | 30 +++++++++++++++++++++++++++ 2 files changed, 48 insertions(+), 2 deletions(-) diff --git a/src/opendox/runtime/bundle.py b/src/opendox/runtime/bundle.py index e872c00f..4ee1827d 100644 --- a/src/opendox/runtime/bundle.py +++ b/src/opendox/runtime/bundle.py @@ -346,7 +346,7 @@ def _preexec() -> None: # pragma: no cover - runs in the child BUNDLE_TREE = BUNDLE_SOCKET_DIR.parts -def _make_private_directories(leaf: Path) -> None: +def _make_private_directories(leaf: Path, *, state: Path) -> None: """`leaf` and every missing directory above it, each born exactly 0700. `Path.mkdir(parents=True)` gives the directories it creates on the way @@ -369,6 +369,16 @@ def _make_private_directories(leaf: Path) -> None: or written through: a symbolic link, something that is not a directory, or a directory that is not this user's alone. + AND THE DIRECTORY IT STARTS FROM IS JUDGED BY ITS DESCRIPTOR, before + the first `mkdir` (adversarial review of openDox-code#69). Opening it + follows a link, which this install allows on the configured path, so + what the descriptor names is asked the tree check's own question + (`_unsafe_because`): this user's alone where it is the state directory + or under it, and otherwise this user's or root's, with any write by + others only behind the sticky bit. The path-wise check before it asks + the same question; this one asks it of the very directory that is + written into. + THE UMASK IS PROCESS-WIDE, and it is narrowed only for these few `mkdir`s and then put back. A file another thread creates meanwhile can only come out more private than asked, never less. @@ -382,6 +392,11 @@ def _make_private_directories(leaf: Path) -> None: if not missing: return descriptor = os.open(base, os.O_RDONLY | os.O_DIRECTORY) + own = base == state or state in base.parents + reason = _unsafe_because(os.fstat(descriptor), uid=uid, own=own) + if reason is not None: + os.close(descriptor) + raise BundledServer._unsafe(base, reason) path = base previous = os.umask(0o077) try: @@ -622,7 +637,8 @@ def _prepare_directories(self) -> None: # BEFORE THE CHMOD, which follows a symbolic link: a `run` placed # there as a link would otherwise have its TARGET re-moded. self._refuse_an_unsafe_tree(existing_only=True) - _make_private_directories(self.bundle.socket_dir) + _make_private_directories(self.bundle.socket_dir, + state=self.bundle.state_dir) self._refuse_an_unsafe_tree() os.chmod(self.bundle.socket_dir, 0o700) diff --git a/tests_runtime/test_local_lifecycle.py b/tests_runtime/test_local_lifecycle.py index ea1bdfa0..68bbdba8 100644 --- a/tests_runtime/test_local_lifecycle.py +++ b/tests_runtime/test_local_lifecycle.py @@ -589,6 +589,36 @@ def test_nothing_is_made_through_a_path_the_tree_check_refuses( assert not os.path.lexists(made), f"{made} was made before the refusal" +@pytest.mark.parametrize("shape", ["link-to-open", "open", "link-to-open-state", + "sticky-state"]) +def test_the_directory_creation_starts_from_judges_its_own_descriptor( + tmp_path: Path, short_state: Path, shape: str) -> None: + """`_make_private_directories` judges the directory it opens, by its + descriptor, before its first `mkdir` (adversarial review of #69). It is + asked here directly, with no path-wise check in front of it, so its own + guard is what is measured: a base that every user can write, reached + through a link or not, makes nothing beneath it. The state directory is + judged as the install's OWN, so even a sticky one is refused, where a + sticky ANCESTOR (`/tmp`'s shape) is accepted.""" + watched = short_state / "open" + watched.mkdir() + watched.chmod(0o1777 if shape == "sticky-state" else 0o777) + if shape == "link-to-open": + (short_state / "link").symlink_to(watched) + state = short_state / "link" / "state" + elif shape == "open": + state = watched / "state" + elif shape == "link-to-open-state": + (short_state / "link").symlink_to(watched) + state = short_state / "link" + else: + state = watched + with pytest.raises(bundle_mod.BundleRefused) as caught: + bundle_mod._make_private_directories(state / "postgres" / "run", state=state) + assert "writable by every user" in str(caught.value), caught.value + assert list(watched.iterdir()) == [], "made beneath an unsafe base" + + @pytest.mark.parametrize("shape", ["link", "foreign-directory"]) def test_a_name_put_in_the_way_first_is_refused_never_followed( monkeypatch, tmp_path: Path, short_state: Path, shape: str) -> None: From 9e2f303029dd9a874c2ec12c308a3bd001c335bd Mon Sep 17 00:00:00 2001 From: Brett Heap <1513478+brettheap@users.noreply.github.com> Date: Fri, 2 Oct 2026 23:13:57 +0000 Subject: [PATCH 33/39] Adversarial review L2: the local verbs judge their socket before connecting to it `runtime status`, `migrate` and `reset` of a local install connected, as the served role or as the owner, to whatever answered at the bundle's socket path. Nothing had checked that path. The review reproduced it with a 0777 state directory whose postgres/run linked to another bundle's socket directory: status and migrate ran as the owner role against the other server, while a start refused the same tree. Reproduced here too: status on such a tree reported the other bundle's applied migrations and exited 0. - bundle.refuse_an_unsafe_tree is the start's tree check lifted out of BundledServer (which now delegates to it), so a verb can ask it without a server. - bundle.refusal_before_connecting asks two things and writes nothing: - the tree check, of what exists; - a live server of THIS data directory behind the socket. Its own postmaster.pid must name a live pid, which (where /proc can say) is a postgres whose cwd is this data directory, and which listens at this socket directory (the lock file's fifth line). - runtime/cli.py asks it in all three local verbs before any connection. status reports `database: "not probed: "` (its database block is re-indented under the new branch; `git diff -w` shows only the branch). migrate and reset refuse as "local-bundle-unverified". A hosted install is not judged here. test_install_mode's local status case with no server now expects "not probed: no bundled server is running" where it read "unreachable": the same answer, reached without a connection. New cases: - a real one in test_bundled_postgres.py: a linked run directory, an open state directory, and a valid tree with no server, each against a running bundle. All three verbs are refused by name, and the running bundle's status stays reachable with its schema intact. - a hermetic one in test_local_lifecycle.py: no lock, a dead pid, a live pid that is not a postgres, a socket line that differs and one that matches (both without /proc), and an open state directory. The real case fails against the previous runtime/cli.py. Seven mutants are all killed. Lane: openxfactory-4 (openXfactory-4-openDox_extraction) Co-Authored-By: Claude Opus 5.5 (1M context) --- src/opendox/runtime/bundle.py | 192 ++++++++++++++++--------- src/opendox/runtime/cli.py | 171 +++++++++++++--------- tests_runtime/test_bundled_postgres.py | 57 ++++++++ tests_runtime/test_install_mode.py | 6 +- tests_runtime/test_local_lifecycle.py | 46 ++++++ 5 files changed, 337 insertions(+), 135 deletions(-) diff --git a/src/opendox/runtime/bundle.py b/src/opendox/runtime/bundle.py index 4ee1827d..c6aceee1 100644 --- a/src/opendox/runtime/bundle.py +++ b/src/opendox/runtime/bundle.py @@ -266,6 +266,62 @@ def _remove_a_proven_stale_lock(bundle: DatabaseBundle) -> None: (bundle.data_dir / "postmaster.pid").unlink(missing_ok=True) +def refusal_before_connecting(bundle: DatabaseBundle) -> str | None: + """Why a local verb must not connect to `bundle`'s socket, or `None`. + + `runtime status`, `migrate` and `reset` of a LOCAL install connect, as + the served role or as the OWNER, to whatever answers at the bundle's + socket path. A start judges that path, and these verbs did not: a state + directory every user could write, whose `postgres/run` was a link to + another bundle's socket directory, had `status` and `migrate` run as the + owner role against that other server while a start refused the same + tree (adversarial review of openDox-code#69). So before any client + connection, two things are asked, and neither writes anything: + + * THE TREE CHECK a start asks, of what exists (`refuse_an_unsafe_tree` + with `existing_only`): a socket directory that is a link, or that + another user could replace, is refused here as it is there; + * A LIVE SERVER OF THIS DATA DIRECTORY behind the socket. The lock + file in this data directory, `postmaster.pid`, names a live process + that, where the platform can say (`_serves`), is a `postgres` whose + working directory is this data directory, and it names THIS socket + directory as the one the server listens on (its fifth line). Where + `/proc` cannot say what the pid is, the other answers still bind + the socket to this tree. + """ + try: + refuse_an_unsafe_tree(bundle, existing_only=True) + except BundleRefused as exc: + return str(exc) + except OSError as exc: + return (f"the bundled server's path under {bundle.state_dir} could " + f"not be judged ({type(exc).__name__})") + lock = bundle.data_dir / "postmaster.pid" + try: + lines = lock.read_text(encoding="utf-8").splitlines() + pid = int(lines[0].strip()) + except (OSError, IndexError, ValueError): + return (f"no bundled server is running on {bundle.data_dir}: it has no " + "readable postmaster.pid. A local install's server is started " + f"by `opendox generate-and-open {LOCAL_FLAG}`, which owns it") + try: + os.kill(pid, 0) + except (ProcessLookupError, PermissionError): + return (f"no bundled server is running on {bundle.data_dir}: its " + f"postmaster.pid names pid {pid}, which is not a live process " + "of this user") + if _serves(pid, bundle) is False: + return (f"pid {pid}, named by {lock}, is not the postgres serving " + f"{bundle.data_dir}, so what answers at {bundle.socket_dir} " + "is not this install's server") + listens = lines[4].strip() if len(lines) > 4 else "" + if listens != str(bundle.socket_dir): + return (f"the server on {bundle.data_dir} listens at " + f"{listens or 'no Unix socket'}, not at {bundle.socket_dir}, so " + "what answers there is not this install's server") + return None + + def report(bundle: DatabaseBundle) -> dict[str, Any]: """The `database_bundle` block `runtime status` prints (#1144 13.1).""" return {"data_dir": str(bundle.data_dir), @@ -540,6 +596,75 @@ def _unsafe_because(info: os.stat_result, *, uid: int, own: bool) -> str | None: return None +def refuse_an_unsafe_tree(bundle: DatabaseBundle, *, + existing_only: bool = False) -> None: + """The socket's whole path is this user's to change, or it is refused. + + The socket's directory is how this install's clients find ITS + server, so the 0700 on it is worth only what the path above it is + worth. Peer authentication keeps other users out of the server, but + not a substitute socket out of the path: whoever could replace `run` + could stand up a server of their own for this install's clients to + talk to. A directory entry is controlled by its PARENT: a parent that + another user can write lets them rename `run` away, or put a symbolic + link in its place, after the mode is set. A symbolic link on the way there + can be pointed elsewhere by whoever owns it, or by whoever can write + the directory it sits in (Copilot review of openDox-code#69). So: + + * THE INSTALL'S OWN TREE, as the configured path resolves: the + state directory, `postgres/` and `run/` must be real directories, + owned by this user and writable by no one else; + * EVERY DIRECTORY ABOVE IT, on the configured path and on the path + it resolves to, must be owned by this user or by root. One that + anyone else can write, a group included, must be sticky, as + `/tmp` is, so nobody can rename what is not theirs; + * EVERY SYMBOLIC LINK on the configured path must be this user's or + root's. + + `OPENDOX_STATE_DIR` never holds `..` (`config.state_dir` refuses it), + so the configured path's components are the ones the kernel walks. + + With `existing_only`, the same rules are asked of only what exists + yet. `_prepare_directories` asks that BEFORE it creates anything, + so the links are checked first, a broken one included (Copilot + review of openDox-code#69). + """ + uid = os.getuid() + configured = bundle.state_dir + + def present(path: Path) -> bool: + return not existing_only or os.path.lexists(path) + + for component in (configured, *configured.parents): + if not present(component): + continue + info = os.lstat(component) + if stat.S_ISLNK(info.st_mode) and info.st_uid not in (uid, 0): + raise BundledServer._unsafe( + component, f"is a symbolic link owned by uid {info.st_uid}, " + "neither this user nor root, who could point it elsewhere") + state = configured.resolve() + tree = [state, state / BUNDLE_TREE[0], state / BUNDLE_TREE[0] / BUNDLE_TREE[1]] + # AND THE DATA DIRECTORY, where one exists already, a broken link + # included (Copilot review of openDox-code#69). A `data` placed there + # as a link to a cluster elsewhere would otherwise be launched, and + # given this install's authentication files, outside the state tree. + # A fresh one needs no check: `_initialize` renames it into place. + data = state / BUNDLE_DATA_DIR + if os.path.lexists(data): + tree.append(data) + checks = [(path, True) for path in tree] + [ + (path, False) for path in dict.fromkeys( + [*state.parents, *configured.parents])] + for directory, mine in checks: + if not present(directory): + continue + info = os.lstat(directory) if mine else os.stat(directory) + reason = _unsafe_because(info, uid=uid, own=mine) + if reason is not None: + raise BundledServer._unsafe(directory, reason) + + class BundledServer: """One local install's PostgreSQL server: started as this process's child. @@ -643,71 +768,8 @@ def _prepare_directories(self) -> None: os.chmod(self.bundle.socket_dir, 0o700) def _refuse_an_unsafe_tree(self, *, existing_only: bool = False) -> None: - """The socket's whole path is this user's to change, or it is refused. - - The socket's directory is how this install's clients find ITS - server, so the 0700 on it is worth only what the path above it is - worth. Peer authentication keeps other users out of the server, but - not a substitute socket out of the path: whoever could replace `run` - could stand up a server of their own for this install's clients to - talk to. A directory entry is controlled by its PARENT: a parent that - another user can write lets them rename `run` away, or put a symbolic - link in its place, after the mode is set. A symbolic link on the way there - can be pointed elsewhere by whoever owns it, or by whoever can write - the directory it sits in (Copilot review of openDox-code#69). So: - - * THE INSTALL'S OWN TREE, as the configured path resolves: the - state directory, `postgres/` and `run/` must be real directories, - owned by this user and writable by no one else; - * EVERY DIRECTORY ABOVE IT, on the configured path and on the path - it resolves to, must be owned by this user or by root. One that - anyone else can write, a group included, must be sticky, as - `/tmp` is, so nobody can rename what is not theirs; - * EVERY SYMBOLIC LINK on the configured path must be this user's or - root's. - - `OPENDOX_STATE_DIR` never holds `..` (`config.state_dir` refuses it), - so the configured path's components are the ones the kernel walks. - - With `existing_only`, the same rules are asked of only what exists - yet. `_prepare_directories` asks that BEFORE it creates anything, - so the links are checked first, a broken one included (Copilot - review of openDox-code#69). - """ - uid = os.getuid() - configured = self.bundle.state_dir - - def present(path: Path) -> bool: - return not existing_only or os.path.lexists(path) - - for component in (configured, *configured.parents): - if not present(component): - continue - info = os.lstat(component) - if stat.S_ISLNK(info.st_mode) and info.st_uid not in (uid, 0): - raise self._unsafe( - component, f"is a symbolic link owned by uid {info.st_uid}, " - "neither this user nor root, who could point it elsewhere") - state = configured.resolve() - tree = [state, state / BUNDLE_TREE[0], state / BUNDLE_TREE[0] / BUNDLE_TREE[1]] - # AND THE DATA DIRECTORY, where one exists already, a broken link - # included (Copilot review of openDox-code#69). A `data` placed there - # as a link to a cluster elsewhere would otherwise be launched, and - # given this install's authentication files, outside the state tree. - # A fresh one needs no check: `_initialize` renames it into place. - data = state / BUNDLE_DATA_DIR - if os.path.lexists(data): - tree.append(data) - checks = [(path, True) for path in tree] + [ - (path, False) for path in dict.fromkeys( - [*state.parents, *configured.parents])] - for directory, mine in checks: - if not present(directory): - continue - info = os.lstat(directory) if mine else os.stat(directory) - reason = _unsafe_because(info, uid=uid, own=mine) - if reason is not None: - raise self._unsafe(directory, reason) + """`refuse_an_unsafe_tree`, for this server's own bundle.""" + refuse_an_unsafe_tree(self.bundle, existing_only=existing_only) @staticmethod def _unsafe(directory: Path, reason: str) -> BundleRefused: diff --git a/src/opendox/runtime/cli.py b/src/opendox/runtime/cli.py index b5d22207..605f65ad 100644 --- a/src/opendox/runtime/cli.py +++ b/src/opendox/runtime/cli.py @@ -491,6 +491,24 @@ def cmd_init(args: argparse.Namespace) -> int: "next": "opendox-runtime runtime migrate"}, ok=True) +def _local_bundle_refusal(settings) -> str | None: + """For a LOCAL install, why its socket must not be connected to, or `None`. + + `bundle.refusal_before_connecting`: the tree check a start asks, and a + live server of THIS data directory behind the socket, both before any + client connection (adversarial review of openDox-code#69). A hosted + install's DSN is the operator's, and is not judged here. + """ + if settings.install_mode != INSTALL_MODE_LOCAL: + return None + from opendox.runtime import bundle as bundle_mod + from opendox.runtime.config import database_bundle + + reason = bundle_mod.refusal_before_connecting( + database_bundle(settings.state_dir)) + return None if reason is None else _safe_message(reason) + + def cmd_migrate(args: argparse.Namespace) -> int: """Apply the ordered SQL, or with `--plan` report what would be applied. @@ -526,6 +544,10 @@ def cmd_migrate(args: argparse.Namespace) -> int: "package with the `runtime` extra: " "pip install '.[runtime]'"}, ok=False) + refusal = _local_bundle_refusal(settings) + if refusal is not None: + return _emit({"verb": "migrate", "refusal": "local-bundle-unverified", + "message": refusal}, ok=False) runner_db = Database(dsn, application_name="opendox-runtime-migrate", checkout_timeout=args.connect_timeout) # THE OUTCOME IS COMPUTED INSIDE THE CONTEXT AND EMITTED OUTSIDE IT. A @@ -742,76 +764,85 @@ def cmd_status(args: argparse.Namespace) -> int: report["runtime_extra"] = "present" connected = False - try: - # INSIDE the context, all of it. `runner.applied()` and `runner.plan()` - # each check a connection out of the pool, so calling them after the - # `with` had closed it raised `PoolClosed` and this verb reported a - # perfectly reachable database as unreachable (Copilot review of - # openDox-code#25, and it is the kind of defect only a live database - # shows — every unreachable-database test passed). - with Database(settings.database_url, - checkout_timeout=args.probe_timeout) as db: - with db.connection() as conn: - conn.execute("select 1") - # THE CONNECTIVITY ANSWER IS RECORDED THE MOMENT IT IS TRUE, so a - # failure in the queries BELOW cannot rewrite it — see the generic - # handler at the end of this block. - connected = True - # THE SAME CANONICAL GATE `apply()` AND `/readyz` RUN. Without - # it an EMPTY migrations directory reports `pending: []` on a - # fresh database — nothing pending, nothing drifted, everything - # fine — for an install with no coordination schema at all - # (Copilot review of openDox-code#25, round 7). `status` is the - # verb an operator believes. - migrations.verify_canonical_digest(settings.migrations_dir) - runner = migrations.MigrationRunner( - db, migrations_dir=settings.migrations_dir) - applied = [row.version for row in runner.applied()] - pending = [m.version for m in runner.plan()] - drifted = runner.drift() - report["database"] = "reachable" - report["applied_migrations"] = applied - report["pending_migrations"] = pending - # NOTHING PENDING IS NOT THE SAME AS MATCHING THIS TREE: a migration - # whose file changed, or vanished, is invisible to `plan()` and is - # refused by `apply()`. See `MigrationRunner.drift`. - report["migration_drift"] = drifted - # PENDING IS UNHEALTHY, exactly as `/readyz` treats it. This reported - # the versions and left `ok` true, so a reachable but UNMIGRATED - # database exited 0 while the readiness probe on the same install - # refuses traffic — two answers to one question, and the CLI's was the - # comforting one (Copilot review of openDox-code#25, round 7). - if drifted or pending: - ok = False - # a status verb reports, never raises - except migrations.MigrationError as exc: - # THE DATABASE ANSWERED; THE TREE DID NOT. `select 1` has already - # succeeded by the time the runner is asked anything, so reporting - # `database: unreachable` for a missing or malformed migrations - # directory pointed the operator at the wrong dependency entirely - # (Copilot review of openDox-code#25, round 7). - report.setdefault("database", "reachable") - report["migrations"] = ( - f"unreadable: {type(exc).__name__}: {_safe_message(exc)}") + # A LOCAL INSTALL'S SOCKET IS JUDGED BEFORE IT IS CONNECTED TO + # (adversarial review of openDox-code#69): the tree a start checks, and a + # live server of this data directory behind it. Otherwise `status` asks + # whatever answers at that path, as the served role. + refusal = _local_bundle_refusal(settings) + if refusal is not None: + report["database"] = f"not probed: {refusal}" ok = False - except Exception as exc: # noqa: BLE001 - # THE SAME DISTINCTION THE BRANCH ABOVE MAKES, for the failures that - # are not the runner's own. Once `select 1` has answered, the database - # IS reachable, and a later failure — the served role without `select` - # on the ledger, a schema the search path does not reach, a query that - # errors — is a privilege or schema problem reported as one. Reporting - # `database: unreachable` for it pointed the operator at the network - # and hid the real fault, which is the defect round 7 fixed for - # `MigrationError` and left in place one handler down (Copilot review - # of openDox-code#25, round 10, suppressed). - if connected: + else: + try: + # INSIDE the context, all of it. `runner.applied()` and `runner.plan()` + # each check a connection out of the pool, so calling them after the + # `with` had closed it raised `PoolClosed` and this verb reported a + # perfectly reachable database as unreachable (Copilot review of + # openDox-code#25, and it is the kind of defect only a live database + # shows — every unreachable-database test passed). + with Database(settings.database_url, + checkout_timeout=args.probe_timeout) as db: + with db.connection() as conn: + conn.execute("select 1") + # THE CONNECTIVITY ANSWER IS RECORDED THE MOMENT IT IS TRUE, so a + # failure in the queries BELOW cannot rewrite it — see the generic + # handler at the end of this block. + connected = True + # THE SAME CANONICAL GATE `apply()` AND `/readyz` RUN. Without + # it an EMPTY migrations directory reports `pending: []` on a + # fresh database — nothing pending, nothing drifted, everything + # fine — for an install with no coordination schema at all + # (Copilot review of openDox-code#25, round 7). `status` is the + # verb an operator believes. + migrations.verify_canonical_digest(settings.migrations_dir) + runner = migrations.MigrationRunner( + db, migrations_dir=settings.migrations_dir) + applied = [row.version for row in runner.applied()] + pending = [m.version for m in runner.plan()] + drifted = runner.drift() report["database"] = "reachable" - report["schema_queries"] = ( - f"failed: {type(exc).__name__}: {_safe_message(exc)}") - else: - report["database"] = ( - f"unreachable: {type(exc).__name__}: {_safe_message(exc)}") - ok = False + report["applied_migrations"] = applied + report["pending_migrations"] = pending + # NOTHING PENDING IS NOT THE SAME AS MATCHING THIS TREE: a migration + # whose file changed, or vanished, is invisible to `plan()` and is + # refused by `apply()`. See `MigrationRunner.drift`. + report["migration_drift"] = drifted + # PENDING IS UNHEALTHY, exactly as `/readyz` treats it. This reported + # the versions and left `ok` true, so a reachable but UNMIGRATED + # database exited 0 while the readiness probe on the same install + # refuses traffic — two answers to one question, and the CLI's was the + # comforting one (Copilot review of openDox-code#25, round 7). + if drifted or pending: + ok = False + # a status verb reports, never raises + except migrations.MigrationError as exc: + # THE DATABASE ANSWERED; THE TREE DID NOT. `select 1` has already + # succeeded by the time the runner is asked anything, so reporting + # `database: unreachable` for a missing or malformed migrations + # directory pointed the operator at the wrong dependency entirely + # (Copilot review of openDox-code#25, round 7). + report.setdefault("database", "reachable") + report["migrations"] = ( + f"unreadable: {type(exc).__name__}: {_safe_message(exc)}") + ok = False + except Exception as exc: # noqa: BLE001 + # THE SAME DISTINCTION THE BRANCH ABOVE MAKES, for the failures that + # are not the runner's own. Once `select 1` has answered, the database + # IS reachable, and a later failure — the served role without `select` + # on the ledger, a schema the search path does not reach, a query that + # errors — is a privilege or schema problem reported as one. Reporting + # `database: unreachable` for it pointed the operator at the network + # and hid the real fault, which is the defect round 7 fixed for + # `MigrationError` and left in place one handler down (Copilot review + # of openDox-code#25, round 10, suppressed). + if connected: + report["database"] = "reachable" + report["schema_queries"] = ( + f"failed: {type(exc).__name__}: {_safe_message(exc)}") + else: + report["database"] = ( + f"unreachable: {type(exc).__name__}: {_safe_message(exc)}") + ok = False # A LOCAL INSTALL HAS NO BROKER TO PROBE (plan 034 T070; #1144 13.4), # and that is its configuration rather than a fault: reported by name, and @@ -865,6 +896,10 @@ def cmd_reset(args: argparse.Namespace) -> int: except ImportError as exc: # pragma: no cover - the extra is absent return _emit({"verb": "reset", "refusal": "runtime-extra-missing", "message": _safe_message(exc)}, ok=False) + refusal = _local_bundle_refusal(settings) + if refusal is not None: + return _emit({"verb": "reset", "refusal": "local-bundle-unverified", + "message": refusal}, ok=False) try: with Database(dsn, application_name="opendox-runtime-reset", checkout_timeout=args.connect_timeout) as db: diff --git a/tests_runtime/test_bundled_postgres.py b/tests_runtime/test_bundled_postgres.py index ce87ed02..462251f9 100644 --- a/tests_runtime/test_bundled_postgres.py +++ b/tests_runtime/test_bundled_postgres.py @@ -53,6 +53,7 @@ import pytest from opendox.runtime import bundle as bundle_mod +from opendox.runtime import cli from opendox.runtime import config from opendox.runtime.config import PREFIX @@ -620,6 +621,62 @@ def test_a_cluster_whose_configuration_points_elsewhere_runs_on_its_own_files( assert method == f"peer:{bundle_mod.os_user()}", method +def _verb(state: Path, *verb: str) -> tuple[int, dict]: + """`OPENDOX_INSTALL_MODE=local opendox-runtime runtime `, a SECOND process.""" + done = subprocess.run( + [sys.executable, "-m", "opendox.runtime.cli", "runtime", *verb], + env=_clean_env(**{MODE: "local", STATE: str(state)}), cwd=ROOT, + capture_output=True, text=True, timeout=60) + return done.returncode, json.loads(done.stdout) + + +@pytest.mark.parametrize("shape", ["linked-run", "open-state", "no-server"]) +def test_the_local_verbs_connect_only_to_their_own_verified_server( + state_dir: Path, shape: str) -> None: + """The adversarial review of #69 (L2): a second state tree whose + `postgres/run` is a link to a RUNNING bundle's socket directory, in a + state directory every user could write. `status` and `migrate` ran on + it as the owner role, against the other bundle's server, while a start + refused the same tree. Each verb now asks the tree check and a live + server of its OWN data directory before any connection. A valid tree + with no server is answered the same way, by name and unconnected.""" + settings = config.load_settings({MODE: "local", STATE: str(state_dir)}) + other = Path(tempfile.mkdtemp(prefix="odx-o-", dir="/tmp" if os.path.isdir("/tmp") else None)) + try: + (other / "postgres" / "data").mkdir(parents=True, mode=0o700) + (other / "postgres").chmod(0o700) + if shape == "no-server": + (other / "postgres" / "run").mkdir(mode=0o700) + else: + (other / "postgres" / "run").symlink_to( + config.DatabaseBundle(state_dir).socket_dir) + if shape == "open-state": + other.chmod(0o777) + with bundle_mod.BundledServer(settings): + status_code, status = _status(other) + migrate_code, migrate = _verb(other, "migrate") + reset_code, reset = _verb(other, "reset", "--confirm", + cli.RESET_CONFIRMATION) + # the positive control: the running bundle's own verbs connect, + # and its schema was not dropped through the other tree's link + own_code, own = _status(state_dir) + finally: + other.chmod(0o700) + shutil.rmtree(other, ignore_errors=True) + expected = {"linked-run": "is a symbolic link", + "open-state": "writable by every user", + "no-server": "no bundled server is running"}[shape] + assert status_code == 1 and status["database"].startswith("not probed: "), status + assert expected in status["database"], status + assert migrate_code == 1, migrate + assert migrate["refusal"] == "local-bundle-unverified", migrate + assert expected in migrate["message"], migrate + assert reset_code == 1 and reset["refusal"] == "local-bundle-unverified", reset + assert expected in reset["message"], reset + assert own["database"] == "reachable" and own_code == 0, own + assert own["applied_migrations"] and not own["pending_migrations"], own + + def test_migrate_under_the_local_mode_uses_the_bundle_and_refuses_a_dsn( state_dir: Path) -> None: """`runtime migrate` is part of the same install: it reads the bundle's diff --git a/tests_runtime/test_install_mode.py b/tests_runtime/test_install_mode.py index 52047299..3cbbf4bb 100644 --- a/tests_runtime/test_install_mode.py +++ b/tests_runtime/test_install_mode.py @@ -334,8 +334,10 @@ def _no_broker(_settings): assert evidence["settings"][PREFIX + "OIDC_JWKS_URL"] == "" # the database half is the ONLY reason `ok` is false: no bundled server # is running on this (nonexistent) state directory, and `status` reports - # that rather than starting one - assert evidence["database"].startswith("unreachable"), evidence + # that rather than starting one. It says so WITHOUT connecting: a local + # socket is judged before it is asked (adversarial review of #69). + assert evidence["database"].startswith( + "not probed: no bundled server is running"), evidence assert evidence["database_bundle"]["pid"] is None, evidence assert code == 1 diff --git a/tests_runtime/test_local_lifecycle.py b/tests_runtime/test_local_lifecycle.py index 68bbdba8..96c8a479 100644 --- a/tests_runtime/test_local_lifecycle.py +++ b/tests_runtime/test_local_lifecycle.py @@ -948,6 +948,52 @@ def test_an_existing_cluster_is_opened_only_by_its_own_major( assert not (data / "pg_hba.conf").exists(), "wrote into another major's cluster" +def _a_tree(state: Path) -> config.DatabaseBundle: + """A valid bundle tree under `state`, every directory 0700, no server.""" + bundle = config.DatabaseBundle(state) + for directory in (state / "postgres", bundle.data_dir, bundle.socket_dir): + directory.mkdir(mode=0o700, exist_ok=True) + directory.chmod(0o700) + return bundle + + +@pytest.mark.parametrize("shape", ["no-lock", "dead-pid", "not-postgres", + "no-proc-other-socket", "no-proc-this-socket", + "open-state"]) +def test_a_local_verb_connects_only_behind_a_verified_server( + monkeypatch, short_state: Path, shape: str) -> None: + """`bundle.refusal_before_connecting` (adversarial review of #69, L2): + the tree a start checks, then a live server of THIS data directory, + named by its own lock file, listening at THIS socket directory. Where + `/proc` cannot say what the pid is, the lock file's socket line still + binds the socket to the tree.""" + bundle = _a_tree(short_state) + live = os.getpid() # alive, and not a postgres + dead = 2 ** 22 + 17 # above the default pid_max + lock = bundle.data_dir / "postmaster.pid" + socket_line = (str(bundle.socket_dir) if shape != "no-proc-other-socket" + else "/tmp/somewhere-else") + if shape != "no-lock": + pid = dead if shape == "dead-pid" else live + lock.write_text(f"{pid}\n{bundle.data_dir}\n1\n5432\n{socket_line}\n\n", + encoding="utf-8") + if shape.startswith("no-proc"): + monkeypatch.setattr(bundle_mod, "PROC", short_state / "no-proc") + if shape == "open-state": + short_state.chmod(0o777) + reason = bundle_mod.refusal_before_connecting(bundle) + expected = {"no-lock": "no readable postmaster.pid", + "dead-pid": "not a live process", + "not-postgres": "is not the postgres serving", + "no-proc-other-socket": "listens at /tmp/somewhere-else", + "no-proc-this-socket": None, + "open-state": "writable by every user"}[shape] + if expected is None: + assert reason is None, reason + else: + assert reason is not None and expected in reason, reason + + def test_an_initdb_that_dies_midway_leaves_no_data_directory( monkeypatch, tmp_path: Path, short_state: Path) -> None: """The half-built cluster: `PG_VERSION` written, then the run fails. The From 21bde1af699d1129a84d472fc538b60cd2c348e9 Mon Sep 17 00:00:00 2001 From: Brett Heap <1513478+brettheap@users.noreply.github.com> Date: Fri, 2 Oct 2026 23:15:18 +0000 Subject: [PATCH 34/39] Adversarial review note: no replication connection, logical or physical authentication_files' docstring said "a replication connection is refused". That holds for a PHYSICAL one, which matches no pg_hba.conf rule. A LOGICAL one (replication=database) names a database, so the one local rule read it as an ordinary connection. It was accepted as peer:, and IDENTIFY_SYSTEM answered (measured here, as the review found). The docstring was not just reworded, because no pg_hba.conf rule can tell the two apart. The launch now sets max_wal_senders=0, so the server starts no WAL sender at all, and the docstring says so. New real-server case: replication=database and replication=true are both refused as the owner role, while an ordinary connection still is not and reads max_wal_senders as 0. The mutant that leaves WAL senders on is killed. Lane: openxfactory-4 (openXfactory-4-openDox_extraction) Co-Authored-By: Claude Opus 5.5 (1M context) --- src/opendox/runtime/bundle.py | 15 +++++++++++++-- tests_runtime/test_bundled_postgres.py | 25 +++++++++++++++++++++++++ 2 files changed, 38 insertions(+), 2 deletions(-) diff --git a/src/opendox/runtime/bundle.py b/src/opendox/runtime/bundle.py index c6aceee1..18853154 100644 --- a/src/opendox/runtime/bundle.py +++ b/src/opendox/runtime/bundle.py @@ -520,7 +520,14 @@ def authentication_files(user: str) -> dict[str, str]: * Every HOST connection is rejected. The server also listens on no TCP address at all (`listen_addresses` is empty), so these lines never match. They are written so that the file says what the install is. - * No replication line, so a replication connection is refused. + * No replication line, so a PHYSICAL replication connection matches + no rule and is refused. A LOGICAL one (`replication=database`) + names a database, so `pg_hba.conf` reads it as the ordinary local + connection it resembles, and no rule here can tell the two apart: + it was accepted as `peer:` (adversarial review of + openDox-code#69). So the launch sets `max_wal_senders=0`, and the + server starts no WAL sender for either kind. A replication + connection is refused by the server, whatever the files say. """ header = ("# Written by opendox.runtime.bundle before every start of this local\n" "# install's bundled server (plan 034 T072). Changes here are replaced.\n") @@ -908,7 +915,11 @@ def _launch(self, binaries: Path) -> None: # the data directory and the two files written above. "-c", f"data_directory={self.bundle.data_dir}", "-c", f"hba_file={self.bundle.data_dir / 'pg_hba.conf'}", - "-c", f"ident_file={self.bundle.data_dir / 'pg_ident.conf'}"], + "-c", f"ident_file={self.bundle.data_dir / 'pg_ident.conf'}", + # NO REPLICATION, physical or logical: see + # `authentication_files`, whose rules cannot refuse a + # logical one (adversarial review of openDox-code#69). + "-c", "max_wal_senders=0"], stdin=subprocess.DEVNULL, stdout=log, stderr=subprocess.STDOUT, env=_child_environment(), # ITS OWN SESSION, so a terminal's Ctrl-C reaches this process diff --git a/tests_runtime/test_bundled_postgres.py b/tests_runtime/test_bundled_postgres.py index 462251f9..45432e45 100644 --- a/tests_runtime/test_bundled_postgres.py +++ b/tests_runtime/test_bundled_postgres.py @@ -566,6 +566,31 @@ def test_a_role_outside_the_map_is_refused_even_for_this_os_user( assert "peer authentication failed" in str(caught.value).lower(), caught.value +@pytest.mark.parametrize("mode", ["database", "true"]) +def test_a_replication_connection_is_refused_logical_or_physical( + state_dir: Path, mode: str) -> None: + """A PHYSICAL replication connection (`replication=true`) matches no + rule in `pg_hba.conf`. A LOGICAL one (`replication=database`) names a + database, and the one local rule admitted it as `peer:`, where + `IDENTIFY_SYSTEM` answered (adversarial review of #69). The server + starts no WAL sender (`max_wal_senders=0`), so both are refused, as the + owner role and over the same socket.""" + import psycopg + from psycopg.conninfo import make_conninfo + + settings = config.load_settings({MODE: "local", STATE: str(state_dir)}) + with bundle_mod.BundledServer(settings) as server: + with pytest.raises(psycopg.OperationalError) as caught: + psycopg.connect(make_conninfo(server.bundle.migration_dsn, + replication=mode)).close() + with _owner(server) as conn: # an ordinary one still is + senders = conn.execute("show max_wal_senders").fetchone()[0] + message = str(caught.value).lower() + assert ("max_wal_senders" in message if mode == "database" + else ("max_wal_senders" in message or "no pg_hba.conf entry" in message)), message + assert senders == "0", senders + + def test_an_older_trust_cluster_is_brought_back_to_peer_on_start( state_dir: Path) -> None: """A data directory an earlier build initialized with `trust` (or a file From 28e195b98d6751d22b7fc33013b112cb8a5bd501 Mon Sep 17 00:00:00 2001 From: Brett Heap <1513478+brettheap@users.noreply.github.com> Date: Fri, 2 Oct 2026 23:43:56 +0000 Subject: [PATCH 35/39] Fix round 13: the server is found as the installed distribution's own files, never by import precedence (Copilot review) server_binaries found pixeltable_pgserver with importlib.util.find_spec, which follows sys.path. Under `python -m opendox.cli`, sys.path starts with the working directory, and a corpus checkout is where that runs. So a checkout holding an executable pixeltable_pgserver/pginstall/bin/postgres was run as this install's database server. The carrier is now looked up by distribution name (importlib.metadata) on sys.path without the working directory, whether it appears as '' or as its own absolute path. Both binaries must be: - files the distribution's RECORD lists; - located inside the distribution after symlinks are resolved; - executable. Anything else is the named refusal. Still nothing imports pixeltable_pgserver. Cases in tests_runtime/test_local_lifecycle.py: - the reworked lookup case: no distribution, a RECORD naming no binaries, binaries linked outside it file by file or through a linked bin directory, and binaries that are not executable; - a new working-directory case: an importable pixeltable_pgserver with executable binaries AND a forged .dist-info, both in the working directory, which is also on sys.path as '' and as its path. The server is not taken from there. Six of the eight fail against the find_spec lookup. Four mutants are killed. One is equivalent ('' resolves to the working directory, which the next check drops). A redundant parent-directory check was removed. Lane: openxfactory-4 (openXfactory-4-openDox_extraction) Co-Authored-By: Claude Opus 5.5 (1M context) --- src/opendox/runtime/bundle.py | 72 ++++++++++++++++----- tests_runtime/test_local_lifecycle.py | 93 +++++++++++++++++++++++---- 2 files changed, 136 insertions(+), 29 deletions(-) diff --git a/src/opendox/runtime/bundle.py b/src/opendox/runtime/bundle.py index 18853154..9bc6c025 100644 --- a/src/opendox/runtime/bundle.py +++ b/src/opendox/runtime/bundle.py @@ -19,8 +19,9 @@ THE SERVER'S OWN PACKAGE IS `pixeltable-pgserver` (pyproject.toml's `local` extra; RULED openxFactory#656 `5916000030` item 2), and only its BINARIES are used: PostgreSQL 16's `initdb` and `postgres` from the wheel's `pginstall/bin`, -found by `importlib.util.find_spec` without importing `pixeltable_pgserver` at -all. Its Python manager is deliberately not used. It daemonizes the server +found through the INSTALLED distribution's own file list +(`importlib.metadata`), never by import precedence, and without importing +`pixeltable_pgserver` at all (see `server_binaries`). Its Python manager is deliberately not used. It daemonizes the server through `pg_ctl`, which re-parents it away from this process (against (i)). It shares one server between processes by reference count and stops it from `atexit`, which a SIGTERM never runs (against (iv)). And it may put the socket @@ -68,7 +69,7 @@ import contextlib import ctypes -import importlib.util +from importlib import metadata import os import re import shutil @@ -125,32 +126,73 @@ class BundleRefused(Exception): """ +def _distribution_search_path() -> list[str]: + """`sys.path` without the working directory, which no install is. + + `python -m opendox.cli` and `python -c` put the directory they were + started in at the front of `sys.path` (as `''`, or as its absolute + path), and a corpus repository is exactly where a user runs them from. + """ + try: + here = Path.cwd().resolve() + except OSError: # a working directory since removed + here = None + kept = [] + for entry in sys.path: + if not entry: + continue + try: + if here is not None and Path(entry).resolve() == here: + continue + except (OSError, RuntimeError): + continue + kept.append(entry) + return kept + + def server_binaries() -> Path: """The directory holding the bundled `initdb` and `postgres`, or a refusal. Found WITHOUT importing `pixeltable_pgserver`: its package initializer imports its manager, which this module does not use and which registers an `atexit` handler and reaches for the user's runtime directory. + + AND FOUND AS THE INSTALLED DISTRIBUTION'S OWN FILES, never by import + precedence (Copilot review of openDox-code#69). `importlib.util.find_spec` + follows `sys.path`, whose first entry under `python -m opendox.cli` is + the working directory. So a checkout holding an executable + `pixeltable_pgserver/pginstall/bin/postgres` was run as this install's + database server. The distribution is now looked up by its name + (`importlib.metadata`), on `sys.path` WITHOUT the working directory, and + both binaries must be files its RECORD lists, located inside it. """ - spec = importlib.util.find_spec(SERVER_PACKAGE) - # THE FIRST LOCATION, OR NONE, read without an index, so no path reaches - # a subscript that could raise (SonarCloud S6466 on openDox-code#69). - location = next(iter(spec.submodule_search_locations or ()), None) \ - if spec else None - if location is None: + candidates = list(metadata.distributions( + name=SERVER_DISTRIBUTION, path=_distribution_search_path())) + if not candidates: raise BundleRefused( "the local install's PostgreSQL server is not installed: it " "arrives with the `local` extra, `pip install \"opendox[local]\"` " "(R1Q16 (iii)). A local install brings its own database and never " "borrows one") - binaries = Path(location) / "pginstall" / "bin" - missing = [name for name in ("initdb", "postgres") - if not os.access(binaries / name, os.X_OK)] + distribution = candidates[0] + suffix = ".exe" if os.name == "nt" else "" + listed = {str(entry).replace("\\", "/"): entry + for entry in (distribution.files or ())} + root = Path(distribution.locate_file("")).resolve() + binaries = Path(distribution.locate_file(f"{SERVER_PACKAGE}/pginstall/bin")) + missing = [] + for name in ("initdb", "postgres"): + entry = listed.get(f"{SERVER_PACKAGE}/pginstall/bin/{name}{suffix}") + located = (Path(distribution.locate_file(entry)).resolve() + if entry is not None else None) + if (located is None or not located.is_relative_to(root) + or not os.access(located, os.X_OK)): + missing.append(name) if missing: raise BundleRefused( - f"the `{SERVER_DISTRIBUTION}` package is installed but carries no " - f"executable {' or '.join(missing)} under {binaries}; reinstall " - "the `local` extra") + f"the `{SERVER_DISTRIBUTION}` package is installed but its own " + f"file list carries no executable {' or '.join(missing)} under " + f"{binaries}; reinstall the `local` extra") return binaries diff --git a/tests_runtime/test_local_lifecycle.py b/tests_runtime/test_local_lifecycle.py index 96c8a479..89db724a 100644 --- a/tests_runtime/test_local_lifecycle.py +++ b/tests_runtime/test_local_lifecycle.py @@ -742,23 +742,88 @@ def test_both_classes_together_name_both_reasons() -> None: assert "hunter2" not in message -@pytest.mark.parametrize("found", ["absent", "no-locations"]) +@pytest.mark.parametrize("found", ["absent", "no-binaries", "outside-it", + "dir-outside-it", "not-executable"]) def test_a_missing_server_package_is_the_named_refusal( - monkeypatch: pytest.MonkeyPatch, found: str) -> None: - """No `pixeltable_pgserver` at all, or a spec with no location: both are - the one refusal naming the `local` extra, never an `IndexError`.""" - import importlib.machinery - - spec = None - if found == "no-locations": - spec = importlib.machinery.ModuleSpec(bundle_mod.SERVER_PACKAGE, None, - is_package=True) - spec.submodule_search_locations = [] - monkeypatch.setattr(bundle_mod.importlib.util, "find_spec", - lambda name: spec) + monkeypatch: pytest.MonkeyPatch, tmp_path: Path, found: str) -> None: + """No `pixeltable-pgserver` distribution at all; one whose file list + names no binaries; or one whose listed binaries resolve outside it, + file by file or through a linked `bin` directory; or binaries that are + listed and inside it but cannot be executed. + Each is the one refusal naming the `local` extra, never a traceback, + and nothing is taken from anywhere but the distribution's own files + (Copilot review of #69).""" + root = tmp_path / "site" + bin_dir = root / bundle_mod.SERVER_PACKAGE / "pginstall" / "bin" + bin_dir.mkdir(parents=True) + elsewhere = tmp_path / "elsewhere" + elsewhere.mkdir() + for name in ("initdb", "postgres"): + for directory in (bin_dir, elsewhere): + (directory / name).write_text("#!/bin/sh\nexit 0\n", encoding="utf-8") + (directory / name).chmod(0o755) + if found == "outside-it": + for name in ("initdb", "postgres"): + (bin_dir / name).unlink() + (bin_dir / name).symlink_to(elsewhere / name) + elif found == "dir-outside-it": + shutil.rmtree(bin_dir) + bin_dir.symlink_to(elsewhere) + elif found == "not-executable": + for name in ("initdb", "postgres"): + (bin_dir / name).chmod(0o644) + + class _Distribution: + files = ([] if found == "no-binaries" else + [f"{bundle_mod.SERVER_PACKAGE}/pginstall/bin/{name}" + for name in ("initdb", "postgres")]) + + def locate_file(self, entry): + return root / str(entry) + + monkeypatch.setattr(bundle_mod.metadata, "distributions", + lambda **kwargs: [] if found == "absent" else [_Distribution()]) with pytest.raises(bundle_mod.BundleRefused) as caught: bundle_mod.server_binaries() - assert 'opendox[local]' in str(caught.value), caught.value + message = str(caught.value) + assert ('opendox[local]' if found == "absent" else "reinstall the `local` extra") \ + in message, message + + +def test_the_server_is_never_taken_from_the_working_directory( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + """`python -m opendox.cli` puts the working directory first on + `sys.path`, and a corpus checkout is where it runs (Copilot review of + #69). A checkout holding an importable `pixeltable_pgserver` with + executable binaries, AND a forged `.dist-info` naming them, is still not + where the server comes from: the search path leaves the working + directory out, as `''` and as its own path.""" + shadow = tmp_path / "checkout" + package = shadow / bundle_mod.SERVER_PACKAGE + bin_dir = package / "pginstall" / "bin" + bin_dir.mkdir(parents=True) + (package / "__init__.py").write_text("", encoding="utf-8") + for name in ("initdb", "postgres"): + (bin_dir / name).write_text("#!/bin/sh\necho planted\n", encoding="utf-8") + (bin_dir / name).chmod(0o755) + info = shadow / "pixeltable_pgserver-99.0.dist-info" + info.mkdir() + (info / "METADATA").write_text( + "Metadata-Version: 2.1\nName: pixeltable-pgserver\nVersion: 99.0\n", + encoding="utf-8") + (info / "RECORD").write_text("".join( + f"{bundle_mod.SERVER_PACKAGE}/pginstall/bin/{name},,\n" + for name in ("initdb", "postgres")), encoding="utf-8") + monkeypatch.chdir(shadow) + monkeypatch.syspath_prepend(str(shadow)) + monkeypatch.setattr(bundle_mod.sys, "path", ["", *bundle_mod.sys.path]) + assert str(shadow) not in bundle_mod._distribution_search_path() + assert "" not in bundle_mod._distribution_search_path() + try: + found = bundle_mod.server_binaries() + except bundle_mod.BundleRefused: + return # no carrier installed: nothing taken either + assert not found.resolve().is_relative_to(shadow.resolve()), found # -- peer authentication, hermetic ---------------------------------------------- From bf9bcb08ba7951b2b2fbaef04333e01e5542ec7b Mon Sep 17 00:00:00 2001 From: Brett Heap <1513478+brettheap@users.noreply.github.com> Date: Sat, 3 Oct 2026 00:11:08 +0000 Subject: [PATCH 36/39] Fix round 14: a named platform gate, resolution failures as reasons, no pid behind a refused tree (Copilot review) Three threads from Copilot's review at 21bde1af. - The bundle is a POSIX design, but the carrier ships Windows wheels. There a start failed as a generic AttributeError phase refusal, and status raised one. bundle.unsupported_platform names the missing primitives: os.getuid, O_DIRECTORY, O_NOFOLLOW, os.fchmod, mkdir with dir_fd (read once at import as MKDIR_TAKES_DIR_FD), and socket.AF_UNIX. BundledServer.start and refusal_before_connecting ask it first. It follows the pattern of local_git_adapter's refuse_without_the_no_follow_walk. - In refusal_before_connecting, a symlink loop makes Path.resolve() raise RuntimeError (Python 3.12), and an embedded NUL makes the os calls raise ValueError. Both escaped as the CLI's generic failure. They are now a named reason, beside OSError. - bundle.report printed running_pid(bundle) before the tree was verified. So a postgres/data linked to another live bundle reported that server's pid, while the connection guard refused the same tree. The pid is now reported only where refusal_before_connecting has nothing to say. New cases: - tests_runtime/test_local_lifecycle.py: four missing primitives, each refused by start and by the socket check, with nothing made; a symlink loop answered as a reason; and no pid behind a refused tree. - tests_runtime/test_bundled_postgres.py: the real verbs case gains a linked-data shape, every refused shape reports a null pid, and the running bundle's own status reports its pid. All eight mutants are killed. The unverified-pid mutant is killed by the real linked-data case as well. Lane: openxfactory-4 (openXfactory-4-openDox_extraction) Co-Authored-By: Claude Opus 5.5 (1M context) --- src/opendox/runtime/bundle.py | 68 ++++++++++++++++++++++++-- tests_runtime/test_bundled_postgres.py | 20 ++++++-- tests_runtime/test_local_lifecycle.py | 53 ++++++++++++++++++++ 3 files changed, 131 insertions(+), 10 deletions(-) diff --git a/src/opendox/runtime/bundle.py b/src/opendox/runtime/bundle.py index 9bc6c025..0dbb2e6b 100644 --- a/src/opendox/runtime/bundle.py +++ b/src/opendox/runtime/bundle.py @@ -74,6 +74,7 @@ import re import shutil import signal +import socket import stat import subprocess import sys @@ -126,6 +127,42 @@ class BundleRefused(Exception): """ +#: Whether `os.mkdir` takes `dir_fd` here, read ONCE at import: a case that +#: stands a wrapper in for `os.mkdir` must not read as another platform. +MKDIR_TAKES_DIR_FD = os.mkdir in os.supports_dir_fd + + +def unsupported_platform() -> str | None: + """Why this platform cannot run the local install's bundled server, or `None`. + + The bundle is a POSIX design, and every one of its guarantees rests on a + POSIX primitive. The socket is a Unix socket, and authentication is peer, + by the kernel's uid. The directories are judged by uid and made without + following a link (`os.getuid`, `O_DIRECTORY`, `O_NOFOLLOW`, a `dir_fd` + `mkdir`, `fchmod`). The carrier ships wheels for Windows too, and there + a start failed as a generic `AttributeError` and `status` raised one + (Copilot review of openDox-code#69). So the gap is named first, as + `runtime/local_git_adapter.py`'s `refuse_without_the_no_follow_walk` + names its own. + """ + missing = [name for name, present in ( + ("os.getuid", hasattr(os, "getuid")), + ("os.O_DIRECTORY", hasattr(os, "O_DIRECTORY")), + ("os.O_NOFOLLOW", hasattr(os, "O_NOFOLLOW")), + ("os.fchmod", hasattr(os, "fchmod")), + ("mkdir with dir_fd", MKDIR_TAKES_DIR_FD), + ("socket.AF_UNIX", hasattr(socket, "AF_UNIX")), + ) if not present] + if not missing: + return None + return (f"the local install's bundled PostgreSQL server needs a POSIX " + f"platform, and this one ({sys.platform}) lacks " + f"{', '.join(missing)}: its socket is a Unix socket authenticated " + "by peer, and its directories are judged by owner and made " + "without following a link. Use a hosted install here " + f"({PREFIX}INSTALL_MODE=hosted, with an operator's database)") + + def _distribution_search_path() -> list[str]: """`sys.path` without the working directory, which no install is. @@ -331,13 +368,21 @@ def refusal_before_connecting(bundle: DatabaseBundle) -> str | None: `/proc` cannot say what the pid is, the other answers still bind the socket to this tree. """ + gap = unsupported_platform() + if gap is not None: + return gap try: refuse_an_unsafe_tree(bundle, existing_only=True) except BundleRefused as exc: return str(exc) - except OSError as exc: + except (OSError, RuntimeError, ValueError) as exc: + # A SYMBOLIC-LINK LOOP OR A NUL IS A REASON TOO (Copilot review of + # openDox-code#69): `Path.resolve()` raises `RuntimeError` for a loop + # (Python 3.12) and the `os` calls `ValueError` for an embedded NUL, + # and either would otherwise escape as the CLI's generic failure. return (f"the bundled server's path under {bundle.state_dir} could " - f"not be judged ({type(exc).__name__})") + f"not be judged ({type(exc).__name__}): it is not a tree " + "this install can verify") lock = bundle.data_dir / "postmaster.pid" try: lines = lock.read_text(encoding="utf-8").splitlines() @@ -365,10 +410,19 @@ def refusal_before_connecting(bundle: DatabaseBundle) -> str | None: def report(bundle: DatabaseBundle) -> dict[str, Any]: - """The `database_bundle` block `runtime status` prints (#1144 13.1).""" + """The `database_bundle` block `runtime status` prints (#1144 13.1). + + THE PID ONLY BEHIND A VERIFIED TREE (Copilot review of openDox-code#69). + `running_pid` reads the lock file through whatever `postgres/data` is, + so a `data` linked to another live bundle answered with THAT server's + pid, while the connection guard refused the same tree. A pid is + reported only where `refusal_before_connecting` has nothing to say, so + `status` never claims a server it would not connect to. + """ + verified = refusal_before_connecting(bundle) is None return {"data_dir": str(bundle.data_dir), "socket_dir": str(bundle.socket_dir), - "pid": running_pid(bundle)} + "pid": running_pid(bundle) if verified else None} def _child_environment() -> dict[str, str]: @@ -742,6 +796,9 @@ def report(self) -> dict[str, Any]: # -- start ------------------------------------------------------------- def start(self) -> BundledServer: + gap = unsupported_platform() + if gap is not None: + raise BundleRefused(gap) if hasattr(os, "geteuid") and os.geteuid() == 0: raise BundleRefused( "the bundled PostgreSQL server refuses to run as root, and so " @@ -1115,5 +1172,6 @@ def __exit__(self, *exc: object) -> None: __all__ = ["BUNDLE_PORT", "BundleRefused", "BundledServer", "authentication_files", "isolated_from_libpq_environment", - "os_user", "report", "running_pid", "server_binaries", + "os_user", "refusal_before_connecting", "refuse_an_unsafe_tree", + "report", "running_pid", "server_binaries", "unsupported_platform", "write_authentication"] diff --git a/tests_runtime/test_bundled_postgres.py b/tests_runtime/test_bundled_postgres.py index 45432e45..1b70aa81 100644 --- a/tests_runtime/test_bundled_postgres.py +++ b/tests_runtime/test_bundled_postgres.py @@ -655,7 +655,8 @@ def _verb(state: Path, *verb: str) -> tuple[int, dict]: return done.returncode, json.loads(done.stdout) -@pytest.mark.parametrize("shape", ["linked-run", "open-state", "no-server"]) +@pytest.mark.parametrize("shape", ["linked-run", "linked-data", "open-state", + "no-server"]) def test_the_local_verbs_connect_only_to_their_own_verified_server( state_dir: Path, shape: str) -> None: """The adversarial review of #69 (L2): a second state tree whose @@ -664,13 +665,19 @@ def test_the_local_verbs_connect_only_to_their_own_verified_server( it as the owner role, against the other bundle's server, while a start refused the same tree. Each verb now asks the tree check and a live server of its OWN data directory before any connection. A valid tree - with no server is answered the same way, by name and unconnected.""" + with no server is answered the same way, by name and unconnected. And + `status` reports no pid for a tree it refuses, a `data` linked to the + running bundle's included (Copilot review of #69).""" settings = config.load_settings({MODE: "local", STATE: str(state_dir)}) other = Path(tempfile.mkdtemp(prefix="odx-o-", dir="/tmp" if os.path.isdir("/tmp") else None)) try: - (other / "postgres" / "data").mkdir(parents=True, mode=0o700) - (other / "postgres").chmod(0o700) - if shape == "no-server": + (other / "postgres").mkdir(mode=0o700) + if shape == "linked-data": + (other / "postgres" / "data").symlink_to( + config.DatabaseBundle(state_dir).data_dir) + else: + (other / "postgres" / "data").mkdir(mode=0o700) + if shape in {"no-server", "linked-data"}: (other / "postgres" / "run").mkdir(mode=0o700) else: (other / "postgres" / "run").symlink_to( @@ -689,9 +696,11 @@ def test_the_local_verbs_connect_only_to_their_own_verified_server( other.chmod(0o700) shutil.rmtree(other, ignore_errors=True) expected = {"linked-run": "is a symbolic link", + "linked-data": "is a symbolic link", "open-state": "writable by every user", "no-server": "no bundled server is running"}[shape] assert status_code == 1 and status["database"].startswith("not probed: "), status + assert status["database_bundle"]["pid"] is None, status["database_bundle"] assert expected in status["database"], status assert migrate_code == 1, migrate assert migrate["refusal"] == "local-bundle-unverified", migrate @@ -699,6 +708,7 @@ def test_the_local_verbs_connect_only_to_their_own_verified_server( assert reset_code == 1 and reset["refusal"] == "local-bundle-unverified", reset assert expected in reset["message"], reset assert own["database"] == "reachable" and own_code == 0, own + assert own["database_bundle"]["pid"] is not None, own["database_bundle"] assert own["applied_migrations"] and not own["pending_migrations"], own diff --git a/tests_runtime/test_local_lifecycle.py b/tests_runtime/test_local_lifecycle.py index 89db724a..134381a3 100644 --- a/tests_runtime/test_local_lifecycle.py +++ b/tests_runtime/test_local_lifecycle.py @@ -1059,6 +1059,59 @@ def test_a_local_verb_connects_only_behind_a_verified_server( assert reason is not None and expected in reason, reason +@pytest.mark.parametrize("lacking", ["O_NOFOLLOW", "getuid", "AF_UNIX", "dir_fd"]) +def test_a_platform_without_the_posix_primitives_is_the_named_refusal( + monkeypatch, tmp_path: Path, short_state: Path, lacking: str) -> None: + """The bundle rests on POSIX primitives, and the carrier ships Windows + wheels too (Copilot review of #69). Lacking any one, a start and the + local verbs' socket check both name the gap, never an `AttributeError`.""" + import socket as socket_mod + + if lacking == "AF_UNIX": + monkeypatch.delattr(socket_mod, "AF_UNIX") + elif lacking == "dir_fd": + monkeypatch.setattr(bundle_mod, "MKDIR_TAKES_DIR_FD", False) + else: + monkeypatch.delattr(bundle_mod.os, lacking) + expected = {"dir_fd": "mkdir with dir_fd", "AF_UNIX": "socket.AF_UNIX"}.get( + lacking, f"os.{lacking}") + server = _prepared(monkeypatch, tmp_path, short_state) + with pytest.raises(bundle_mod.BundleRefused) as caught: + server.start() + assert "needs a POSIX platform" in str(caught.value), caught.value + assert expected in str(caught.value), caught.value + assert not (short_state / "postgres").exists(), "made directories first" + reason = bundle_mod.refusal_before_connecting(config.DatabaseBundle(short_state)) + assert reason is not None and expected in reason, reason + + +def test_a_symlink_loop_in_the_state_path_is_a_reason_not_a_traceback( + tmp_path: Path, short_state: Path) -> None: + """`Path.resolve()` raises for a symbolic-link loop (`RuntimeError` on + Python 3.12), and the local verbs' socket check must still answer with + a reason (Copilot review of #69).""" + loop = short_state / "loop" + loop.symlink_to(loop) + reason = bundle_mod.refusal_before_connecting( + config.DatabaseBundle(loop / "state")) + assert reason is not None, reason + assert "could not be judged" in reason or "symbolic link" in reason, reason + + +def test_status_reports_no_pid_behind_a_tree_it_refuses( + monkeypatch, short_state: Path) -> None: + """A pid is reported only behind a verified tree (Copilot review of + #69): `running_pid` alone would answer for whatever `postgres/data` is, + another live bundle's included.""" + bundle = _a_tree(short_state) + monkeypatch.setattr(bundle_mod, "running_pid", lambda b: 4242) + monkeypatch.setattr(bundle_mod, "refusal_before_connecting", + lambda b: "refused, for the case") + assert bundle_mod.report(bundle)["pid"] is None + monkeypatch.setattr(bundle_mod, "refusal_before_connecting", lambda b: None) + assert bundle_mod.report(bundle)["pid"] == 4242 + + def test_an_initdb_that_dies_midway_leaves_no_data_directory( monkeypatch, tmp_path: Path, short_state: Path) -> None: """The half-built cluster: `PG_VERSION` written, then the run fails. The From 058d96efb8739ae3d3900a88959514d1ada33413 Mon Sep 17 00:00:00 2001 From: Brett Heap <1513478+brettheap@users.noreply.github.com> Date: Sat, 3 Oct 2026 00:29:50 +0000 Subject: [PATCH 37/39] Fix round 15: on Linux the parent-death signal is armed, or the server is not started (Copilot review) ctypes reports a failed prctl() by returning -1, never by raising (a seccomp filter that denies it, say). _die_with_parent's child ignored that result, so the launch succeeded without PDEATHSIG, and a killed entry point could orphan the server despite R1Q16 (iv). - The child now raises when prctl(PR_SET_PDEATHSIG) returns nonzero. subprocess re-raises that in the parent as SubprocessError, which _launch turns into a named refusal ("could not be given its parent-death signal"). No server process exists after it. - A Linux C library with no prctl at all is the same refusal, raised before the launch, where it used to fall back silently to no signal. - Other platforms are unchanged. There the ordinary stop() is the whole of (iv), as before. New case (Linux): a stand-in C library whose prctl returns -1. The start is the named refusal, server.process is None, and the stand-in postgres never ran. Both mutants are killed: the result ignored, and the launch failure left unnamed. Lane: openxfactory-4 (openXfactory-4-openDox_extraction) Co-Authored-By: Claude Opus 5.5 (1M context) --- src/opendox/runtime/bundle.py | 23 +++++++++++++++++++++-- tests_runtime/test_local_lifecycle.py | 27 +++++++++++++++++++++++++++ 2 files changed, 48 insertions(+), 2 deletions(-) diff --git a/src/opendox/runtime/bundle.py b/src/opendox/runtime/bundle.py index 0dbb2e6b..b4d3d50b 100644 --- a/src/opendox/runtime/bundle.py +++ b/src/opendox/runtime/bundle.py @@ -476,23 +476,39 @@ def _die_with_parent(): `prctl` is RESOLVED HERE, in the parent, so the forked child only calls it; and the child re-checks its parent afterwards, because a parent that died between the fork and the `prctl` would never deliver the signal. + + ON LINUX THE SIGNAL IS ARMED OR THE SERVER IS NOT STARTED (Copilot review + of openDox-code#69). `ctypes` reports a failed `prctl` by its `-1` + return, never by raising (a seccomp filter that denies it, say), and + ignoring that left a server that outlives an entry point killed + outright. So a nonzero return raises in the child, which `subprocess` + raises in this process as `SubprocessError` (`_launch` names it), and a + Linux C library with no `prctl` at all is the same refusal here. """ if not sys.platform.startswith("linux"): return None try: prctl = ctypes.CDLL(None, use_errno=True).prctl except (OSError, AttributeError): # pragma: no cover - a libc without it - return None + raise BundleRefused(_UNARMED) from None parent = os.getpid() def _preexec() -> None: # pragma: no cover - runs in the child - prctl(_PR_SET_PDEATHSIG, int(signal.SIGINT)) + if prctl(_PR_SET_PDEATHSIG, int(signal.SIGINT)) != 0: + raise OSError(ctypes.get_errno(), "prctl(PR_SET_PDEATHSIG) failed") if os.getppid() != parent: os._exit(1) return _preexec +#: The refusal when the parent-death signal cannot be armed on Linux. +_UNARMED = ("the bundled PostgreSQL server could not be given its " + "parent-death signal (prctl PR_SET_PDEATHSIG failed), so it would " + "outlive an entry point killed outright (R1Q16 (iv)); it is not " + "started") + + #: The install's own two directories under its state directory, the socket's #: parent and the socket directory (`config.BUNDLE_SOCKET_DIR`). BUNDLE_TREE = BUNDLE_SOCKET_DIR.parts @@ -1026,6 +1042,9 @@ def _launch(self, binaries: Path) -> None: # in order, after the document server has closed. start_new_session=True, preexec_fn=_die_with_parent()) + except subprocess.SubprocessError: + # `_die_with_parent`'s child refused to run unarmed: nothing started + raise BundleRefused(_UNARMED) from None finally: log.close() diff --git a/tests_runtime/test_local_lifecycle.py b/tests_runtime/test_local_lifecycle.py index 134381a3..5841bd08 100644 --- a/tests_runtime/test_local_lifecycle.py +++ b/tests_runtime/test_local_lifecycle.py @@ -1180,6 +1180,33 @@ def test_a_launch_that_cannot_exec_is_the_named_refusal( assert server.process is None +@pytest.mark.skipif(not sys.platform.startswith("linux"), + reason="PR_SET_PDEATHSIG is Linux's") +def test_a_parent_death_signal_that_cannot_be_armed_starts_no_server( + monkeypatch, tmp_path: Path, short_state: Path) -> None: + """`ctypes` reports a failed `prctl` by returning `-1` (a seccomp denial, + say), never by raising, and a server started anyway would outlive an + entry point killed outright (Copilot review of #69). The stand-in C + library's `prctl` fails; the launch is the named refusal, and no server + process exists.""" + launched = tmp_path / "launched" + + class _Libc: + @staticmethod + def prctl(*args): + return -1 + + monkeypatch.setattr(bundle_mod.ctypes, "CDLL", lambda *a, **k: _Libc()) + server = _server(monkeypatch, tmp_path, short_state, + initdb=f'{_target_of_initdb()}\necho 16 > "$T/PG_VERSION"', + postgres=f'touch "{launched}"\nsleep 30') + with pytest.raises(bundle_mod.BundleRefused) as caught: + server.start() + assert "parent-death signal" in str(caught.value), caught.value + assert server.process is None + assert not launched.exists(), "the server ran without its parent-death signal" + + def test_directories_it_cannot_make_are_the_named_refusal( monkeypatch, tmp_path: Path, short_state: Path) -> None: if hasattr(os, "geteuid") and os.geteuid() == 0: # pragma: no cover From 3ebccb3c4872b180491c88db83afd3612694860c Mon Sep 17 00:00:00 2001 From: Brett Heap <1513478+brettheap@users.noreply.github.com> Date: Sat, 3 Oct 2026 01:09:29 +0000 Subject: [PATCH 38/39] Fix round 16: a local install's served role and database are the bundle's own (Copilot review) Under local, OPENDOX_RUNTIME_PG_ROLE could override the bundle's served role in RuntimeSettings. The bundle bootstraps opendox_runtime with the default DML, and its served DSN connects as opendox_runtime. But the migration run narrows the ledger privileges of exactly the configured role. So OPENDOX_RUNTIME_PG_ROLE=pg_read_all_data let a start succeed while opendox_runtime kept INSERT, UPDATE and DELETE on opendox_schema_migrations, and the ledger's protection was broken. refuse_what_a_local_install_cannot_be, which load_settings, load_migration_settings and generate-and-open --local all ask, now refuses two settings unless they are unset or name the bundle's own: - OPENDOX_RUNTIME_PG_ROLE must be opendox_runtime; - OPENDOX_SERVED_DATABASE must be opendox (the same kind of declaration about the same identity). Anything else is refused by name. Hosted is unchanged. New case: each setting set to another name is refused, and its own name is accepted, through both loaders. Both mutants are killed: the check skipped, and the database not asked. Lane: openxfactory-4 (openXfactory-4-openDox_extraction) Co-Authored-By: Claude Opus 5.5 (1M context) --- src/opendox/runtime/config.py | 28 +++++++++++++++++++++++++++ tests_runtime/test_local_lifecycle.py | 24 +++++++++++++++++++++++ 2 files changed, 52 insertions(+) diff --git a/src/opendox/runtime/config.py b/src/opendox/runtime/config.py index c1d9ea56..b220a6d5 100644 --- a/src/opendox/runtime/config.py +++ b/src/opendox/runtime/config.py @@ -2012,11 +2012,39 @@ def refuse_what_a_local_install_cannot_be(env: Mapping[str, str]) -> None: two must not come to disagree about what a local install is. """ _refuse_hosted_only_settings(env) + _refuse_another_identity_than_the_bundles(env) refuse_a_non_loopback_local_bind( PREFIX + "BIND_HOST", _optional(env, _by_name(PREFIX + "BIND_HOST")) or "127.0.0.1") +def _refuse_another_identity_than_the_bundles(env: Mapping[str, str]) -> None: + """A LOCAL install's served role and database are the bundle's own. + + The bundled server is bootstrapped with `BUNDLE_SERVED_ROLE` as the served + identity, granted the default DML, and its served DSN connects as that + role to `BUNDLE_DATABASE`. `OPENDOX_RUNTIME_PG_ROLE` names the role the + migration run NARROWS on the ledger. So a different one, the existing + `pg_read_all_data` say, let a start succeed while `opendox_runtime` kept + INSERT, UPDATE and DELETE on `opendox_schema_migrations`, and the + ledger's protection was broken (Copilot review of openDox-code#69). + `OPENDOX_SERVED_DATABASE` is the same kind of declaration about the same + identity. Each may be set only to the bundle's own name, and anything + else is refused by name. + """ + for name, own in ((PREFIX + "RUNTIME_PG_ROLE", BUNDLE_SERVED_ROLE), + (PREFIX + "SERVED_DATABASE", BUNDLE_DATABASE)): + given = env.get(name, "").strip() + if given and given != own: + raise ConfigurationError( + f"{name} is {given!r}, and a LOCAL install's is {own!r}: the " + "bundled server is bootstrapped with that served identity, its " + "served DSN connects as it, and the migration run narrows " + "exactly the one this names on the migration ledger. Another " + "would leave the bundle's own served role able to rewrite the " + f"ledger. Unset {name}, or set it to {own!r}") + + def _named(given: list[str]) -> str: return f"{' and '.join(given)} {'are' if len(given) > 1 else 'is'} set" diff --git a/tests_runtime/test_local_lifecycle.py b/tests_runtime/test_local_lifecycle.py index 5841bd08..1d103148 100644 --- a/tests_runtime/test_local_lifecycle.py +++ b/tests_runtime/test_local_lifecycle.py @@ -711,6 +711,30 @@ def test_parent_traversal_in_the_state_path_is_refused(variable: str) -> None: # -- the two refusal classes, each with its own reason -------------------------- +@pytest.mark.parametrize("setting, own, other", [ + ("RUNTIME_PG_ROLE", config.BUNDLE_SERVED_ROLE, "pg_read_all_data"), + ("SERVED_DATABASE", config.BUNDLE_DATABASE, "postgres"), +]) +@pytest.mark.parametrize("loader", ["load_settings", "load_migration_settings"]) +def test_a_served_identity_other_than_the_bundles_is_refused( + setting: str, own: str, other: str, loader: str) -> None: + """`OPENDOX_RUNTIME_PG_ROLE` names the role the migration run narrows on + the ledger, and a local install's served DSN connects as the bundle's own + (Copilot review of #69). Another role, `pg_read_all_data` say, left + `opendox_runtime` able to rewrite `opendox_schema_migrations`. So under + `local` each of the two identity settings may name only the bundle's + own, and the bundle's own name is accepted, as before.""" + name = config.PREFIX + setting + load = getattr(config, loader) + base = {MODE: "local", STATE: "/tmp/odx-identity"} + with pytest.raises(config.ConfigurationError) as caught: + load({**base, name: other}) + assert name in str(caught.value) and repr(own) in str(caught.value), caught.value + settings = load({**base, name: own}) + assert (settings.runtime_pg_role if setting == "RUNTIME_PG_ROLE" + else settings.served_database) == own + + def test_a_dsn_beside_local_is_refused_for_the_database_not_a_broker() -> None: with pytest.raises(config.ConfigurationError) as caught: config.load_settings({MODE: "local", From 4a9dbe951955b97ab4996df1c2edd120eaae7032 Mon Sep 17 00:00:00 2001 From: Brett Heap <1513478+brettheap@users.noreply.github.com> Date: Sat, 3 Oct 2026 01:38:21 +0000 Subject: [PATCH 39/39] Fix round 17: the /proc falsifier is gated, and a backslash in the map is pinned literal (Copilot review) Two threads from Copilot's review at 52a2b8dc. - test_the_entry_point_owns_a_migrated_server_with_no_tcp_listener reads Linux's /proc for the server's parent, its TCP listeners and status's pid. The bundle runs on any POSIX platform, so it is now skipped where /proc is absent, as the other /proc and PDEATHSIG cases already are. On Linux, and in CI, nothing changes. - The second thread suggested escaping backslashes in pg_ident.conf's quoted user field. Measured against the bundled PostgreSQL 16.14, that would be wrong. pg_ident_file_mappings reads `"DOMAIN\alice"`, `"alice\"` and `"a\\b"` back as exactly those names, with no error. PostgreSQL's tokenizer treats a backslash specially only at the end of a line, as a continuation, never inside quotes. So escaping would map a different name. The code is unchanged, and the behaviour is now pinned: - test_bundled_postgres.py asks the server's own reading of the map for the three shapes; - the hermetic authentication-files case asserts that DOMAIN\alice is written verbatim. The mutant that escapes backslashes is killed by all four. Lane: openxfactory-4 (openXfactory-4-openDox_extraction) Co-Authored-By: Claude Opus 5.5 (1M context) --- tests_runtime/test_bundled_postgres.py | 33 ++++++++++++++++++++++++++ tests_runtime/test_local_lifecycle.py | 6 +++++ 2 files changed, 39 insertions(+) diff --git a/tests_runtime/test_bundled_postgres.py b/tests_runtime/test_bundled_postgres.py index 1b70aa81..3157c37d 100644 --- a/tests_runtime/test_bundled_postgres.py +++ b/tests_runtime/test_bundled_postgres.py @@ -347,6 +347,12 @@ def test_the_default_state_dir_is_the_users_own(monkeypatch) -> None: # -- 13.1 and R1Q16 (i), (ii), (iv): F13.1's two blocks, on the real entry point +# LINUX'S `/proc`, FOR THE KERNEL'S OWN ANSWERS (Copilot review of #69): the +# server's parent, its TCP listeners and `status`'s pid are read from it, so +# a POSIX platform without it skips this case rather than failing it. +# F13.1 names Linux's socket table; the lifecycle it shares with every POSIX +# platform is held by the cases around it. +@pytest.mark.skipif(not Path("/proc/self").exists(), reason="asks Linux's /proc") def test_the_entry_point_owns_a_migrated_server_with_no_tcp_listener( corpus: Path, state_dir: Path, tmp_path: Path) -> None: """F13.1's `runtime status` block and its TCP-listener block, against the @@ -546,6 +552,33 @@ def test_the_bundle_authenticates_by_peer_through_the_one_map( (bundle_mod.IDENT_MAP, user, config.BUNDLE_SERVED_ROLE, None)], mappings +@pytest.mark.parametrize("user", ["DOMAIN\\alice", "alice\\", "a\\\\b"]) +def test_the_server_reads_a_backslash_in_the_map_literally( + state_dir: Path, user: str) -> None: + """A user name holding a backslash, the shape an NSS or AD account takes + (`DOMAIN\\alice`), is written into `pg_ident.conf` as it is. PostgreSQL + 16 reads a quoted field's backslash LITERALLY: its tokenizer treats a + backslash specially only at the end of a line, as a continuation, and + never inside quotes. So the name is NOT escaped, and escaping it would + map a different name (Copilot review of #69, which suggested escaping, + answered by measurement). The server's own reading of the file is + asked, for each shape.""" + settings = config.load_settings({MODE: "local", STATE: str(state_dir)}) + with bundle_mod.BundledServer(settings) as server: + data = server.bundle.data_dir + (data / "pg_ident.conf").write_text( + bundle_mod.authentication_files(user)["pg_ident.conf"], encoding="utf-8") + try: + with _owner(server) as conn: + mappings = conn.execute( + "select sys_name, pg_username, error from pg_ident_file_mappings " + "order by map_number").fetchall() + finally: + bundle_mod.write_authentication(data, bundle_mod.os_user()) + assert mappings == [(user, config.BUNDLE_OWNER_ROLE, None), + (user, config.BUNDLE_SERVED_ROLE, None)], mappings + + def test_a_role_outside_the_map_is_refused_even_for_this_os_user( state_dir: Path) -> None: """The MAP decides, not the socket. The same OS user, over the same diff --git a/tests_runtime/test_local_lifecycle.py b/tests_runtime/test_local_lifecycle.py index 1d103148..b4692375 100644 --- a/tests_runtime/test_local_lifecycle.py +++ b/tests_runtime/test_local_lifecycle.py @@ -866,6 +866,12 @@ def test_the_authentication_files_admit_one_os_user_as_the_two_roles() -> None: [bundle_mod.IDENT_MAP, '"alice"', config.BUNDLE_OWNER_ROLE], [bundle_mod.IDENT_MAP, '"alice"', config.BUNDLE_SERVED_ROLE]] assert "trust" not in " ".join(" ".join(r) for rows in active.values() for r in rows) + # A BACKSLASH IS WRITTEN AS IT IS, never escaped: PostgreSQL 16 reads a + # quoted field's backslash literally (`test_bundled_postgres.py` asks + # the server's own reading; Copilot review of #69). + domain = bundle_mod.authentication_files("DOMAIN\\alice")["pg_ident.conf"] + assert f'{bundle_mod.IDENT_MAP} "DOMAIN\\alice" {config.BUNDLE_OWNER_ROLE}' \ + in domain.splitlines(), domain def test_the_files_are_written_0600_and_replace_what_was_there(