diff --git a/constraints-cpython312-linux.txt b/constraints-cpython312-linux.txt index 84ed0972..36437c08 100644 --- a/constraints-cpython312-linux.txt +++ b/constraints-cpython312-linux.txt @@ -30,6 +30,15 @@ # two installs, no second list of names to drift. # # Resolved 2026-09-18 on cpython 3.12.3 / linux x86_64. +# Extended 2026-09-30 on cpython 3.12.3 / linux x86_64, under the pins above +# (`-c` this file), for plan 034 T072's `local` extra and the `test` extra's +# `setuptools`: pgserver, its psutil, platformdirs and fasteners, and +# setuptools are new; no earlier pin moved. +# Re-resolved 2026-09-30 on cpython 3.12.3 / linux x86_64, in a clean +# environment under the pins above less `pgserver`, when the carrier became +# `pixeltable-pgserver` (RULED, openxFactory#656 `5916000030` item 2): the one +# line that moved is `pgserver==0.1.4` -> `pixeltable-pgserver==0.6.0`, whose +# own requirements were all pinned already. PyJWT==2.14.0 PyYAML==6.0.3 Pygments==2.21.0 @@ -41,6 +50,7 @@ cffi==2.1.1 click==8.5.0 cryptography==50.0.1 fastapi==0.141.1 +fasteners==0.20 h11==0.16.0 httpcore==1.0.9 httptools==0.8.0 @@ -48,7 +58,10 @@ httpx==0.28.1 idna==3.20 iniconfig==2.3.0 packaging==26.3 +pixeltable-pgserver==0.6.0 +platformdirs==4.12.2 pluggy==1.6.0 +psutil==7.2.2 psycopg-binary==3.3.6 psycopg-pool==3.3.2 psycopg==3.3.6 @@ -57,6 +70,7 @@ pydantic==2.13.5 pydantic_core==2.46.5 pytest==8.4.2 python-dotenv==1.2.3 +setuptools==84.0.0 starlette==1.6.0 typing-inspection==0.4.4 typing_extensions==4.16.0 diff --git a/deploy/compose/.env.example b/deploy/compose/.env.example index f8783755..5916932d 100644 --- a/deploy/compose/.env.example +++ b/deploy/compose/.env.example @@ -57,6 +57,13 @@ OPENDOX_MIGRATION_DATABASE_URL=postgresql://opendox:change-me-local-only@postgre # broker settings below, so it is never selected here. OPENDOX_INSTALL_MODE=hosted +# Where a LOCAL install keeps its bundled PostgreSQL server's data directory and +# Unix socket (plan 034 T072; #1144 13.1). This package is hosted and never +# reads it; it is here because every setting the runtime reads is named here. +# Empty means the per-user default, `$XDG_STATE_HOME/opendox` or +# `~/.local/state/opendox`. +OPENDOX_STATE_DIR= + # REQUIRED. The Keycloak broker's issuer, pinned (RULING Q2): a token from any # other issuer is refused rather than trusted. OPENDOX_OIDC_ISSUER=https://keycloak.example/realms/opendox diff --git a/pyproject.toml b/pyproject.toml index 880ab584..c8794fb1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -93,9 +93,62 @@ dependencies = [ # (`tests_runtime/conftest.py`), and a skipped case does not make a whole # suite. `opendox[runtime]` names the extra below instead of copying its five # lines, so the runtime's dependency list is still declared once. +# +# AND THE LOCAL EXTRA, the same way and for the same reason (plan 034 T072; +# R1Q16 (iii), `5850003126`): the local install's bundled PostgreSQL server is +# what `tests_runtime/test_bundled_postgres.py` starts, so F9.1's `.[test]` +# install still runs every case. `opendox[local]` carries `opendox[runtime]`; +# both are named so the T036 reason above stays readable on its own. +# +# AND `setuptools`, the backend `[build-system]` names: the same test BUILDS +# this package's wheel and installs it outside the checkout, to prove a wheel +# carries its migrations (plan 034 T072). Declared here so that build runs +# offline, with `--no-build-isolation`, in the environment the suite runs in. +# `>=70.1` and not `[build-system]`'s `>=68`: 70.1 is the first release that +# builds a wheel with no separate `wheel` package installed. test = [ "pytest>=8,<9", "opendox[runtime]", + "opendox[local]", + "setuptools>=70.1", +] + +# THE LOCAL EXTRA — the standalone install (plan 034 T072; #1144 13.1, as T007 +# batch H's addendum reads; RULED R1Q16 (iii), openxFactory#656 comment +# `5850003126`): `pip install "opendox[local]"`, then +# `opendox generate-and-open --local …`. It carries the runtime's packages and +# the bundled server's own, and nothing else. +# +# `pixeltable-pgserver` IS THE SERVER'S CARRIER (RULED, openxFactory#656 +# comment `5916000030` item 2, Brett Heap 2026-09-30: "pixeltable-pgserver +# (Recommended)"), and only its binaries are used (see `opendox/runtime/ +# bundle.py` for why its own manager is not). It is the maintained fork of +# `pgserver`, and it ships PostgreSQL 16 as `initdb` and `postgres` inside the +# wheel, built to link only libc and libz, so a host needs no system +# PostgreSQL and no ICU. Measured 2026-09-30 on its 0.6.0 cp312 manylinux +# wheel: +# * PostgreSQL 16.14 (`pginstall/bin/postgres --version`); the wheel also +# carries an 18.4 under `pginstall18/`, which this package does not use; +# * wheels for cp310 to cp314, so every interpreter `requires-python` +# admits today has one; `pgserver` 0.1.4 stopped at cp312 and 16.2; +# * linux wheels tagged manylinux_2_27/2_28 (glibc 2.27 and later), on +# x86_64 and aarch64, plus macOS and Windows; +# * about 24.7 MB per wheel, because it carries the two server majors; +# * Apache-2.0 for the package; the PostgreSQL License for the server. +# `>=0.6.0` is the release this package has been exercised against, the same +# rule every floor in this file is set by. `<0.7` IS A CEILING, which no other +# requirement here carries, because this one is a DATABASE SERVER whose major +# is a property of the user's data: `bundle.py` runs the PostgreSQL 16 under +# `pginstall/`, and a cluster is opened only by the major that made it. +# Upstream's own default is already 18, so a later release may move what +# `pginstall/` holds; the ceiling keeps that a deliberate act of this package +# (adversarial review of openDox-code#69), and `bundle.py` still refuses, by +# name, a server whose major disagrees with the cluster's `PG_VERSION`. Its +# own requirements (fasteners, platformdirs, psutil, typing-extensions) arrive +# with it and are imported by nothing here. +local = [ + "opendox[runtime]", + "pixeltable-pgserver>=0.6.0,<0.7", ] # THE RUNTIME EXTRA — `split-opendox-two-layer-product` § 3.5, RULED Q2 @@ -206,6 +259,20 @@ py-modules = ["route_extension", "subcommand_extension"] [tool.setuptools.packages.find] where = ["src"] +# THE MIGRATIONS, SHIPPED IN THE WHEEL (plan 034 T072). `migrations/` stays at +# the repository root, where the image copies it (`deploy/compose/Dockerfile`: +# `COPY migrations ./migrations`, run from `/app`) and every checkout run finds +# it; this maps the same files into the wheel's data directory, so an install +# run OUTSIDE a checkout still has the migrations its bundled server applies. +# `opendox.runtime.config.installation_migrations_dir` finds them through the +# own record of the installed distribution that holds the running module +# (after the source tree, for a checkout), and the canonical digest gate is +# what proves the copy found is the pinned one. A data directory and not package +# data because package data must live inside the package, and the one tree is +# not moved. +[tool.setuptools.data-files] +"share/opendox/migrations" = ["migrations/*.sql"] + # THE SERVED BUNDLE, PACKAGED — § 3.4 slice S5, and it is a gap this slice had # to close to make RULED Q5 true rather than true-in-a-source-checkout. # diff --git a/src/opendox/cli.py b/src/opendox/cli.py index 40a0677d..980cc784 100644 --- a/src/opendox/cli.py +++ b/src/opendox/cli.py @@ -19,6 +19,7 @@ import argparse import json import os +import signal import sys import tempfile import webbrowser @@ -88,6 +89,12 @@ # or serves anything. Stdlib-only, like `local_git_adapter` above, which # already imports it, so this adds no reach and no import weight. from opendox.runtime import config as runtime_config # noqa: E402 +# THE LOCAL INSTALL'S BUNDLED POSTGRESQL SERVER (plan 034 T072; #1144 13.1, +# R1Q16 (i)-(iv)): started as THIS process's child by `generate-and-open +# --local`, and stopped with it. Stdlib-only at import, like `runtime_config`; +# the driver is imported when the server is started, never here. +from opendox.runtime import bundle as bundle_mod # noqa: E402 +from opendox.runtime import migrations as migrations_mod # noqa: E402 from opendox.boundary import ( # noqa: E402 BoundaryViolation, HumanGate, OutputBoundary, ) @@ -558,8 +565,8 @@ def _validate(written: Path, args: argparse.Namespace, *, def _resolve_install_shape(args: argparse.Namespace, - env=None) -> str: - """The install shape this run serves as, or `ConfigurationError` naming why. + env=None) -> runtime_config.RuntimeSettings: + """The settings this run serves with, or `ConfigurationError` naming why. `--local` and `OPENDOX_INSTALL_MODE` are resolved by `runtime_config.install_mode`, the one reading of the selector, which @@ -569,27 +576,45 @@ def _resolve_install_shape(args: argparse.Namespace, * LOCAL binds loopback only, with no opt-in: a non-loopback `--host` is refused naming the rule (13.4), and so is anything a local install cannot be (`refuse_what_a_local_install_cannot_be`: a broker setting - beside it, or a non-loopback `OPENDOX_BIND_HOST`). It needs no broker. - Its datastore is 13.1's, and arrives with T072. + beside it, a DSN beside it, or a non-loopback `OPENDOX_BIND_HOST`). It + needs no broker, and it supplies BOTH DSNs itself, from the server it + bundles under `OPENDOX_STATE_DIR` (13.1; plan 034 T072). * HOSTED, set or by default, refuses with no issuer, NAMING THE ISSUER - (13.5), and then loads the runtime's whole configuration, because the - serving process is the one whose settings are the install's (13.4a; - R1Q16 (i)). Otherwise unchanged (13.6). + (13.5), and then loads the runtime's whole configuration. + Otherwise unchanged (13.6). + + Either way the result is the runtime's own `load_settings`, because the + serving process is the one whose settings are the install's (13.4a; + R1Q16 (i)). Asked before anything is scanned, minted or bound, so a refused run leaves nothing behind and exits at once rather than starting a server that a bound would have to kill (F13.1's `test "$rc" -ne 124`). """ env = os.environ if env is None else env - mode = runtime_config.install_mode( - env, local_flag=bool(getattr(args, "local", False))) + local_flag = bool(getattr(args, "local", False)) + mode = runtime_config.install_mode(env, local_flag=local_flag) if mode == runtime_config.INSTALL_MODE_LOCAL: runtime_config.refuse_a_non_loopback_local_bind("--host", args.host) runtime_config.refuse_what_a_local_install_cannot_be(env) else: runtime_config.require_the_hosted_issuer(env) - runtime_config.load_settings(env) - return mode + return runtime_config.load_settings(env, local_flag=local_flag) + + +class _Terminated(KeyboardInterrupt): + """SIGTERM, raised as the interrupt the serve loop already stops cleanly on. + + A subclass, so the serve loop's own `except KeyboardInterrupt` still ends + a served run with 0 (F13.1's `kill "$SERVER"; wait "$SERVER"`). An + interrupt that arrives BEFORE the serve loop can still say which signal + it was. + """ + + +def _terminate_as_interrupt(signum, frame): + """SIGTERM, read as the Ctrl-C the serve loop already stops cleanly on.""" + raise _Terminated def cmd_generate_and_open(args: argparse.Namespace, *, opener=webbrowser.open) -> int: @@ -602,12 +627,85 @@ def cmd_generate_and_open(args: argparse.Namespace, *, opener=webbrowser.open) - `OPENDOX_INSTALL_MODE=local`, selects the local single-user install, and with neither the install is hosted — see `_resolve_install_shape`. A refusal there is printed on stderr and the command exits 1, before any - other work.""" + other work. + + A LOCAL RUN OWNS ITS DATABASE (plan 034 T072; #1144 13.1, R1Q16 (i)-(iv)). + Once the corpus root and the anchors are known good, the bundled + PostgreSQL server is started as THIS process's child, bootstrapped and + migrated — starting and migrating it is all release 1 asks of it — and it + is stopped when this command returns, however it returns: a served run + ended by Ctrl-C or by SIGTERM (read here as the same interrupt), a + `--no-serve` run, or a failure. A refused start exits 1 on stderr, like + every other refusal of this verb.""" try: - args.install_mode = _resolve_install_shape(args) + settings = _resolve_install_shape(args) except runtime_config.ConfigurationError as exc: print(f"generate-and-open refused: {exc}", file=sys.stderr) return 1 + args.install_mode = settings.install_mode + args.runtime_settings = settings + if settings.install_mode != runtime_config.INSTALL_MODE_LOCAL: + return _generate_and_open(args, opener=opener) + # THE CHEAP REFUSALS FIRST, so a mistyped root never costs a database + # start: every one `_generate_and_open` asks before it mints its run + # directory, main's empty-source-option refusal (T055) included. + _refuse_non_corpus_repo_root(args) + _refuse_malformed_generated_at(args) + _refuse_empty_source_options(args) + server = bundle_mod.BundledServer(settings) + args.database_bundle = server + # NO `PG*` DEFAULT REACHES THE BUNDLE'S CONNECTIONS while this process + # runs its database (Copilot review of openDox-code#69): see + # `bundle.isolated_from_libpq_environment`. + with bundle_mod.isolated_from_libpq_environment(): + return _run_the_local_lifecycle(args, server, opener=opener) + + +def _run_the_local_lifecycle(args: argparse.Namespace, server, *, opener) -> int: + """Start the bundled server, generate and serve, and stop it, however + this ends: a served run ended by Ctrl-C or SIGTERM, a `--no-serve` run, a + refusal, a failure, or an interrupt before anything was served.""" + try: + previous = signal.signal(signal.SIGTERM, _terminate_as_interrupt) + except ValueError: # not the main thread: no handler to own + previous = None + try: + try: + server.start() + except (bundle_mod.BundleRefused, runtime_config.ConfigurationError, + migrations_mod.MigrationError) as exc: + print(f"generate-and-open refused: {exc}", file=sys.stderr) + return 1 + report = server.report() + print(f" database {report['socket_dir']} (bundled, pid {report['pid']}, " + f"migrations applied now: {server.applied or 'none pending'})") + return _generate_and_open(args, opener=opener) + except KeyboardInterrupt as interrupt: + # AN INTERRUPT ANYWHERE IN THE LOCAL LIFECYCLE IS A CLEAN STOP (Copilot + # review of openDox-code#69). SIGTERM, or Ctrl-C, can arrive while the + # server is initializing or migrating, or while the snapshot is being + # generated, all before the serve loop's own handler. It is a stop + # that was asked for, so it is not a traceback: the `finally` below + # stops the bundled server and restores the handler. Nothing was + # served, so the exit is the signal's conventional status (128 + its + # number) and not 0. + signum = (signal.SIGTERM if isinstance(interrupt, _Terminated) + else signal.SIGINT) + print(f"generate-and-open interrupted ({signum.name}) before it " + "served; its bundled PostgreSQL server stops with it", + file=sys.stderr) + return 128 + int(signum) + finally: + # THE HANDLER FIRST, so a second SIGTERM during the stop takes the + # default action at once; the parent-death signal still stops the + # server if this process goes before `stop()` has finished. + if previous is not None: + signal.signal(signal.SIGTERM, previous) + server.stop() + + +def _generate_and_open(args: argparse.Namespace, *, opener) -> int: + """`generate-and-open`'s generate-then-serve half, once the install is known.""" # Ahead of minting the run dir, so a refused root leaves not even an empty # temp directory behind. `_generate_and_write` is still the guard that MATTERS # (it is the one no caller can skip); these are the same checks, earlier. diff --git a/src/opendox/runtime/bundle.py b/src/opendox/runtime/bundle.py new file mode 100644 index 00000000..b4d3d50b --- /dev/null +++ b/src/opendox/runtime/bundle.py @@ -0,0 +1,1196 @@ +"""A LOCAL install's bundled PostgreSQL server (plan 034 T072; #1144 13.1). + +WHAT #1144 FIXES AND WHAT IT LEAVES. 13.1 fixes the server's IDENTITY: its data +directory and its Unix socket live under the install's own `OPENDOX_STATE_DIR`, +it listens on that socket and on NO TCP port, the product supplies BOTH DSNs +itself, and `runtime status` reports a `database_bundle` naming `data_dir`, +`socket_dir` and the server's `pid`. It leaves the PACKAGING to the realization, +and R1Q16's answer (openxFactory#656 `5850003126`, as T007 batch H's 13.1 +addendum records it) names it: + + (i) the document server starts the bundled server as ITS OWN CHILD and + reports it, so the process a user reaches is the one that owns it; + (ii) starting and migrating the store is all release 1 asks of it, since + the document surface reads nothing from it yet; + (iii) it ships as the `opendox[local]` extra, which carries the runtime's + packages and the server's own; + (iv) it stops with the entry point. + +THE SERVER'S OWN PACKAGE IS `pixeltable-pgserver` (pyproject.toml's `local` +extra; RULED openxFactory#656 `5916000030` item 2), and only its BINARIES are +used: PostgreSQL 16's `initdb` and `postgres` from the wheel's `pginstall/bin`, +found through the INSTALLED distribution's own file list +(`importlib.metadata`), never by import precedence, and without importing +`pixeltable_pgserver` at all (see `server_binaries`). Its Python manager is deliberately not used. It daemonizes the server +through `pg_ctl`, which re-parents it away from this process (against (i)). It +shares one server between processes by reference count and stops it from +`atexit`, which a SIGTERM never runs (against (iv)). And it may put the socket +under the user's runtime directory, opened to 0777, instead of the state +directory (against 13.1). The binaries link only the C library (libc, libm, +libpthread, librt, libdl) and libz from the system, plus the wheel's own +vendored libpq, so they need no system PostgreSQL and no ICU. + +THE LIFECYCLE, IN FULL: + + * `initdb` once per data directory, into an attempt directory that is + renamed into place only when it has succeeded, so an interrupted first + start never leaves a half-built cluster: the MIGRATION identity + (`config.BUNDLE_OWNER_ROLE`) is the bootstrap superuser, local connections + are `peer` and host connections are `reject`, UTF-8 in the `C` locale. + * PEER AUTHENTICATION, re-asserted before every launch (RULED + openxFactory#656 `5916000030` item 3). `pg_hba.conf` admits Unix-socket + connections through the `opendox` map only, and `pg_ident.conf`'s map + admits THIS install's OS user as the two roles and nobody else. The kernel + reports the connecting process's uid (`SO_PEERCRED`), so no password + exists to leak or to store, and a process of any other user is refused + even where it could reach the socket. The socket's directory is 0700 + besides, and the server opens no TCP port at all. + * `postgres` started as a DIRECT CHILD of this process (`subprocess.Popen`, + never `pg_ctl`), with `listen_addresses` empty and the socket directory + given. On Linux it also carries `PR_SET_PDEATHSIG`, so an entry point + killed without any chance to clean up (SIGKILL) still takes its server + with it; an ordinary stop is `stop()`, a fast shutdown. + * a bootstrap, idempotent: the database, the SERVED role with no password + and the grants `deploy/compose/init-runtime-role.sh` makes for the compose + stack's role of the same name, narrowed to what this install's owner + creates. + * the ordered-SQL migrations, through `migrations.MigrationRunner`, as the + owner, with the served role and the database declared, so the run narrows + the ledger and verifies the served role's access exactly as a hosted + `runtime migrate` does. + +IMPORT WEIGHT: standard library and `opendox.runtime.{config,migrations}` at +module level. `psycopg` and `opendox.runtime.db` are imported at CALL time, +inside the functions that connect, like every other module the +`tests_runtime/test_runtime_surface.py` contract names. +""" + +from __future__ import annotations + +import contextlib +import ctypes +from importlib import metadata +import os +import re +import shutil +import signal +import socket +import stat +import subprocess +import sys +import tempfile +import time +from collections.abc import Iterator +from pathlib import Path +from typing import Any + +from opendox.runtime import migrations +from opendox.runtime.config import ( + BUNDLE_DATABASE, + BUNDLE_DATA_DIR, + BUNDLE_OWNER_ROLE, + BUNDLE_PORT, + BUNDLE_SERVED_ROLE, + BUNDLE_SOCKET_DIR, + LOCAL_FLAG, + PREFIX, + DatabaseBundle, + RuntimeSettings, + database_bundle, +) + +#: The distribution the `local` extra installs for the server's binaries, and +#: the package it installs them under. +SERVER_DISTRIBUTION = "pixeltable-pgserver" +SERVER_PACKAGE = "pixeltable_pgserver" + +#: The `pg_ident.conf` map `pg_hba.conf`'s one local line authenticates through. +IDENT_MAP = "opendox" + +#: How long a start may take before it is a failure: `initdb` on a slow disk, +#: plus the server's own recovery on a data directory an earlier run did not +#: stop cleanly. +START_TIMEOUT_SECONDS = 60.0 + +#: How long a fast shutdown may take before the server is told to stop NOW. +STOP_TIMEOUT_SECONDS = 30.0 + +#: `prctl(2)`'s option number for the parent-death signal (linux/prctl.h). +_PR_SET_PDEATHSIG = 1 + + +class BundleRefused(Exception): + """The bundled server could not be started, named. Carries no credential. + + One exception, because the caller does nothing different for any of them: + the local install does not start. + """ + + +#: Whether `os.mkdir` takes `dir_fd` here, read ONCE at import: a case that +#: stands a wrapper in for `os.mkdir` must not read as another platform. +MKDIR_TAKES_DIR_FD = os.mkdir in os.supports_dir_fd + + +def unsupported_platform() -> str | None: + """Why this platform cannot run the local install's bundled server, or `None`. + + The bundle is a POSIX design, and every one of its guarantees rests on a + POSIX primitive. The socket is a Unix socket, and authentication is peer, + by the kernel's uid. The directories are judged by uid and made without + following a link (`os.getuid`, `O_DIRECTORY`, `O_NOFOLLOW`, a `dir_fd` + `mkdir`, `fchmod`). The carrier ships wheels for Windows too, and there + a start failed as a generic `AttributeError` and `status` raised one + (Copilot review of openDox-code#69). So the gap is named first, as + `runtime/local_git_adapter.py`'s `refuse_without_the_no_follow_walk` + names its own. + """ + missing = [name for name, present in ( + ("os.getuid", hasattr(os, "getuid")), + ("os.O_DIRECTORY", hasattr(os, "O_DIRECTORY")), + ("os.O_NOFOLLOW", hasattr(os, "O_NOFOLLOW")), + ("os.fchmod", hasattr(os, "fchmod")), + ("mkdir with dir_fd", MKDIR_TAKES_DIR_FD), + ("socket.AF_UNIX", hasattr(socket, "AF_UNIX")), + ) if not present] + if not missing: + return None + return (f"the local install's bundled PostgreSQL server needs a POSIX " + f"platform, and this one ({sys.platform}) lacks " + f"{', '.join(missing)}: its socket is a Unix socket authenticated " + "by peer, and its directories are judged by owner and made " + "without following a link. Use a hosted install here " + f"({PREFIX}INSTALL_MODE=hosted, with an operator's database)") + + +def _distribution_search_path() -> list[str]: + """`sys.path` without the working directory, which no install is. + + `python -m opendox.cli` and `python -c` put the directory they were + started in at the front of `sys.path` (as `''`, or as its absolute + path), and a corpus repository is exactly where a user runs them from. + """ + try: + here = Path.cwd().resolve() + except OSError: # a working directory since removed + here = None + kept = [] + for entry in sys.path: + if not entry: + continue + try: + if here is not None and Path(entry).resolve() == here: + continue + except (OSError, RuntimeError): + continue + kept.append(entry) + return kept + + +def server_binaries() -> Path: + """The directory holding the bundled `initdb` and `postgres`, or a refusal. + + Found WITHOUT importing `pixeltable_pgserver`: its package initializer + imports its manager, which this module does not use and which registers an + `atexit` handler and reaches for the user's runtime directory. + + AND FOUND AS THE INSTALLED DISTRIBUTION'S OWN FILES, never by import + precedence (Copilot review of openDox-code#69). `importlib.util.find_spec` + follows `sys.path`, whose first entry under `python -m opendox.cli` is + the working directory. So a checkout holding an executable + `pixeltable_pgserver/pginstall/bin/postgres` was run as this install's + database server. The distribution is now looked up by its name + (`importlib.metadata`), on `sys.path` WITHOUT the working directory, and + both binaries must be files its RECORD lists, located inside it. + """ + candidates = list(metadata.distributions( + name=SERVER_DISTRIBUTION, path=_distribution_search_path())) + if not candidates: + raise BundleRefused( + "the local install's PostgreSQL server is not installed: it " + "arrives with the `local` extra, `pip install \"opendox[local]\"` " + "(R1Q16 (iii)). A local install brings its own database and never " + "borrows one") + distribution = candidates[0] + suffix = ".exe" if os.name == "nt" else "" + listed = {str(entry).replace("\\", "/"): entry + for entry in (distribution.files or ())} + root = Path(distribution.locate_file("")).resolve() + binaries = Path(distribution.locate_file(f"{SERVER_PACKAGE}/pginstall/bin")) + missing = [] + for name in ("initdb", "postgres"): + entry = listed.get(f"{SERVER_PACKAGE}/pginstall/bin/{name}{suffix}") + located = (Path(distribution.locate_file(entry)).resolve() + if entry is not None else None) + if (located is None or not located.is_relative_to(root) + or not os.access(located, os.X_OK)): + missing.append(name) + if missing: + raise BundleRefused( + f"the `{SERVER_DISTRIBUTION}` package is installed but its own " + f"file list carries no executable {' or '.join(missing)} under " + f"{binaries}; reinstall the `local` extra") + return binaries + + +def _lock_file_pid(bundle: DatabaseBundle) -> int | None: + """The pid on the first line of the server's `postmaster.pid`, or `None`.""" + try: + first = (bundle.data_dir / "postmaster.pid").read_text( + encoding="utf-8").splitlines()[0] + return int(first.strip()) + except (OSError, IndexError, ValueError): + return None + + +#: Where the kernel answers what a pid is, on Linux. A module constant so a +#: case can take it away and exercise a platform without it. +PROC = Path("/proc") + + +def _identity(pid: int) -> tuple[str, str] | None: + """`(executable, working directory)` of `pid`, from the kernel's `/proc`. + + `None` where the process is gone or is not this user's to inspect. A + process that exits between the lock file's read and this one is GONE, + never proof of anything (Copilot review of openDox-code#69). Raises + `LookupError` where there is no `/proc` at all (macOS, the BSDs): the + standard library has no portable way to ask, and `running_pid` then + believes nothing it cannot prove. + """ + if not PROC.joinpath("self").exists(): + raise LookupError("no /proc to ask") + try: + return (os.readlink(PROC / str(pid) / "exe"), + os.readlink(PROC / str(pid) / "cwd")) + except OSError: # gone, or another user's: not inspectable + return None + + +def _serves(pid: int, bundle: DatabaseBundle) -> bool | None: + """Whether `pid` is the postmaster of `bundle`'s data directory. + + Asked of the platform, as the pair this module launches: an executable + named `postgres` whose working directory IS the data directory. The + postmaster changes into its data directory at startup, and neither of the + two can be rewritten by its process title. So a recycled pid given to + anything else is not it, even another `postgres` serving another directory + (Copilot review of openDox-code#69). + + `False` also for a process that is gone, or that the platform will not + describe: another user's process cannot be this bundle's server, because + the server runs as the owner of a 0700 data directory, which is the user + this runs as. `None` only where nothing can be asked at all. + """ + try: + identity = _identity(pid) + except LookupError: + return None + if identity is None: + return False + executable, cwd = identity + if Path(executable.removesuffix(" (deleted)")).name not in {"postgres", + "postgres.exe"}: + return False + try: + return Path(cwd).resolve() == bundle.data_dir.resolve() + except OSError: + return False + + +def running_pid(bundle: DatabaseBundle) -> int | None: + """The pid of a live server on `bundle`'s data directory, or `None`. + + Read from the server's own `postmaster.pid` (its first line), and only + believed while that process exists and IS this bundle's server: a file + left by a server that did not stop cleanly names a pid that is gone, or + that the kernel has since given to something else (`_serves`). + """ + pid = _lock_file_pid(bundle) + if pid is None: + return None + try: + os.kill(pid, 0) + except (ProcessLookupError, PermissionError): + # gone; or alive and another user's, which cannot be this server + return None + # BELIEVED ONLY WHEN PROVEN. Where nothing can say what the pid is (no + # `/proc`), it is not reported as this server, and this module does not + # refuse a start over it. PostgreSQL's own interlocks, the lock file's + # live-pid check and the shared-memory check, still refuse a second + # postmaster on one data directory. So an unverifiable pid never yields + # two servers, and never a refusal over a process that is not one. The + # price on such a platform is a `status` that reports no pid. + return pid if _serves(pid, bundle) is True else None + + +def _remove_a_proven_stale_lock(bundle: DatabaseBundle) -> None: + """Remove a `postmaster.pid` the kernel PROVES is not this server's. + + PostgreSQL removes a lock file whose pid is gone. It refuses to start, + however, over one whose pid the kernel has given to another live process + of the same user, and that refusal would last as long as the unrelated + process does. Where `/proc` shows that process is not this data + directory's postmaster, the lock is stale by proof and is removed. Where + nothing can be proven, it is left for PostgreSQL to judge. + """ + pid = _lock_file_pid(bundle) + if pid is None: + return + try: + os.kill(pid, 0) + except (ProcessLookupError, PermissionError): + return # PostgreSQL's own rule covers both + if _serves(pid, bundle) is False: + (bundle.data_dir / "postmaster.pid").unlink(missing_ok=True) + + +def refusal_before_connecting(bundle: DatabaseBundle) -> str | None: + """Why a local verb must not connect to `bundle`'s socket, or `None`. + + `runtime status`, `migrate` and `reset` of a LOCAL install connect, as + the served role or as the OWNER, to whatever answers at the bundle's + socket path. A start judges that path, and these verbs did not: a state + directory every user could write, whose `postgres/run` was a link to + another bundle's socket directory, had `status` and `migrate` run as the + owner role against that other server while a start refused the same + tree (adversarial review of openDox-code#69). So before any client + connection, two things are asked, and neither writes anything: + + * THE TREE CHECK a start asks, of what exists (`refuse_an_unsafe_tree` + with `existing_only`): a socket directory that is a link, or that + another user could replace, is refused here as it is there; + * A LIVE SERVER OF THIS DATA DIRECTORY behind the socket. The lock + file in this data directory, `postmaster.pid`, names a live process + that, where the platform can say (`_serves`), is a `postgres` whose + working directory is this data directory, and it names THIS socket + directory as the one the server listens on (its fifth line). Where + `/proc` cannot say what the pid is, the other answers still bind + the socket to this tree. + """ + gap = unsupported_platform() + if gap is not None: + return gap + try: + refuse_an_unsafe_tree(bundle, existing_only=True) + except BundleRefused as exc: + return str(exc) + except (OSError, RuntimeError, ValueError) as exc: + # A SYMBOLIC-LINK LOOP OR A NUL IS A REASON TOO (Copilot review of + # openDox-code#69): `Path.resolve()` raises `RuntimeError` for a loop + # (Python 3.12) and the `os` calls `ValueError` for an embedded NUL, + # and either would otherwise escape as the CLI's generic failure. + return (f"the bundled server's path under {bundle.state_dir} could " + f"not be judged ({type(exc).__name__}): it is not a tree " + "this install can verify") + lock = bundle.data_dir / "postmaster.pid" + try: + lines = lock.read_text(encoding="utf-8").splitlines() + pid = int(lines[0].strip()) + except (OSError, IndexError, ValueError): + return (f"no bundled server is running on {bundle.data_dir}: it has no " + "readable postmaster.pid. A local install's server is started " + f"by `opendox generate-and-open {LOCAL_FLAG}`, which owns it") + try: + os.kill(pid, 0) + except (ProcessLookupError, PermissionError): + return (f"no bundled server is running on {bundle.data_dir}: its " + f"postmaster.pid names pid {pid}, which is not a live process " + "of this user") + if _serves(pid, bundle) is False: + return (f"pid {pid}, named by {lock}, is not the postgres serving " + f"{bundle.data_dir}, so what answers at {bundle.socket_dir} " + "is not this install's server") + listens = lines[4].strip() if len(lines) > 4 else "" + if listens != str(bundle.socket_dir): + return (f"the server on {bundle.data_dir} listens at " + f"{listens or 'no Unix socket'}, not at {bundle.socket_dir}, so " + "what answers there is not this install's server") + return None + + +def report(bundle: DatabaseBundle) -> dict[str, Any]: + """The `database_bundle` block `runtime status` prints (#1144 13.1). + + THE PID ONLY BEHIND A VERIFIED TREE (Copilot review of openDox-code#69). + `running_pid` reads the lock file through whatever `postgres/data` is, + so a `data` linked to another live bundle answered with THAT server's + pid, while the connection guard refused the same tree. A pid is + reported only where `refusal_before_connecting` has nothing to say, so + `status` never claims a server it would not connect to. + """ + verified = refusal_before_connecting(bundle) is None + return {"data_dir": str(bundle.data_dir), + "socket_dir": str(bundle.socket_dir), + "pid": running_pid(bundle) if verified else None} + + +def _child_environment() -> dict[str, str]: + """This process's environment, less libpq's and PostgreSQL's own variables. + + `PGDATA`, `PGPORT`, `PGHOST`, `PGOPTIONS` and the rest would otherwise + steer `initdb` and the server somewhere other than the paths given on + their command lines; the command line is the configuration, stated once. + """ + return {name: value for name, value in os.environ.items() + if not name.startswith("PG")} + + +@contextlib.contextmanager +def isolated_from_libpq_environment() -> Iterator[None]: + """Run a LOCAL install's client side with libpq's `PG*` defaults out of reach. + + libpq fills every connection parameter a DSN leaves unset from the + process environment. Some of those parameters move the connection + somewhere else. `PGHOSTADDR` outranks the DSN's socket `host` and sends + it to a TCP server. `PGSERVICE` fills parameters from a service file. + `PGOPTIONS` sets session parameters, a `search_path` among them. The + bundle's DSNs name their socket, port, user and database, but they + cannot name every parameter libpq has, and an explicitly empty `service` + is itself an error. So a local install's process reads NONE of them while + it runs its database: they are lifted out of `os.environ` for the + duration and put back afterwards (Copilot review of openDox-code#69). + `_child_environment` already does the same for `initdb` and the server. + + For the process's own entry points only: `generate-and-open --local` + around its whole lifecycle, and the runtime CLI's verbs under `local`. + Nothing else in those processes speaks libpq. + """ + lifted = {name: os.environ.pop(name) for name in + [name for name in os.environ if name.startswith("PG")]} + try: + yield + finally: + for name, value in lifted.items(): + os.environ.setdefault(name, value) + + +def _die_with_parent(): + """A `preexec_fn` that signals the server when its parent goes away (iv). + + `PR_SET_PDEATHSIG` with SIGINT, PostgreSQL's FAST shutdown, so an entry + point killed outright (SIGKILL, an OOM kill) still stops its server + instead of leaving it running with the socket held. Linux only, and `None` + elsewhere, where the ordinary `stop()` is the whole of (iv). + + `prctl` is RESOLVED HERE, in the parent, so the forked child only calls + it; and the child re-checks its parent afterwards, because a parent that + died between the fork and the `prctl` would never deliver the signal. + + ON LINUX THE SIGNAL IS ARMED OR THE SERVER IS NOT STARTED (Copilot review + of openDox-code#69). `ctypes` reports a failed `prctl` by its `-1` + return, never by raising (a seccomp filter that denies it, say), and + ignoring that left a server that outlives an entry point killed + outright. So a nonzero return raises in the child, which `subprocess` + raises in this process as `SubprocessError` (`_launch` names it), and a + Linux C library with no `prctl` at all is the same refusal here. + """ + if not sys.platform.startswith("linux"): + return None + try: + prctl = ctypes.CDLL(None, use_errno=True).prctl + except (OSError, AttributeError): # pragma: no cover - a libc without it + raise BundleRefused(_UNARMED) from None + parent = os.getpid() + + def _preexec() -> None: # pragma: no cover - runs in the child + if prctl(_PR_SET_PDEATHSIG, int(signal.SIGINT)) != 0: + raise OSError(ctypes.get_errno(), "prctl(PR_SET_PDEATHSIG) failed") + if os.getppid() != parent: + os._exit(1) + + return _preexec + + +#: The refusal when the parent-death signal cannot be armed on Linux. +_UNARMED = ("the bundled PostgreSQL server could not be given its " + "parent-death signal (prctl PR_SET_PDEATHSIG failed), so it would " + "outlive an entry point killed outright (R1Q16 (iv)); it is not " + "started") + + +#: The install's own two directories under its state directory, the socket's +#: parent and the socket directory (`config.BUNDLE_SOCKET_DIR`). +BUNDLE_TREE = BUNDLE_SOCKET_DIR.parts + + +def _make_private_directories(leaf: Path, *, state: Path) -> None: + """`leaf` and every missing directory above it, each born exactly 0700. + + `Path.mkdir(parents=True)` gives the directories it creates on the way + the default mode less the umask, whatever mode the leaf is given. So + under a common umask of 0002 a fresh `~/.local/state/opendox/...` would + create `.local` and `state` group-writable, and the tree check would + then refuse the directories this install had just made (Copilot review + of openDox-code#69). Each missing component is created here, one at a + time, under a umask of 077, so it is born 0700, whatever the user's + umask is, with no `chmod` after it. A directory that already exists is + left as it is, and the tree check judges it. + + NOTHING IS MADE THROUGH A PATH THAT WAS NOT JUDGED FIRST (Copilot review + of openDox-code#69). The caller has refused an unsafe EXISTING prefix + before this runs (`BundledServer._prepare_directories`), so the deepest + directory that exists is safe to open. Each missing component is then + made RELATIVE TO ITS PARENT'S DESCRIPTOR and opened with `O_NOFOLLOW` + before anything is made beneath it. A name that another user put there + first, in a sticky directory such as `/tmp`, is refused, never followed + or written through: a symbolic link, something that is not a directory, + or a directory that is not this user's alone. + + AND THE DIRECTORY IT STARTS FROM IS JUDGED BY ITS DESCRIPTOR, before + the first `mkdir` (adversarial review of openDox-code#69). Opening it + follows a link, which this install allows on the configured path, so + what the descriptor names is asked the tree check's own question + (`_unsafe_because`): this user's alone where it is the state directory + or under it, and otherwise this user's or root's, with any write by + others only behind the sticky bit. The path-wise check before it asks + the same question; this one asks it of the very directory that is + written into. + + THE UMASK IS PROCESS-WIDE, and it is narrowed only for these few + `mkdir`s and then put back. A file another thread creates meanwhile can + only come out more private than asked, never less. + """ + uid = os.getuid() + missing: list[str] = [] + base = leaf + while not os.path.lexists(base): + missing.append(base.name) + base = base.parent + if not missing: + return + descriptor = os.open(base, os.O_RDONLY | os.O_DIRECTORY) + own = base == state or state in base.parents + reason = _unsafe_because(os.fstat(descriptor), uid=uid, own=own) + if reason is not None: + os.close(descriptor) + raise BundledServer._unsafe(base, reason) + path = base + previous = os.umask(0o077) + try: + for name in reversed(missing): + path = path / name + try: + os.mkdir(name, 0o700, dir_fd=descriptor) + except FileExistsError: + pass # made first, by a concurrent start or by someone else: judged next + try: + child = os.open(name, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW, + dir_fd=descriptor) + except OSError: + info = os.stat(name, dir_fd=descriptor, follow_symlinks=False) + reason = _unsafe_because(info, uid=uid, own=True) + if reason is None: + raise + raise BundledServer._unsafe(path, reason) from None + os.close(descriptor) + descriptor = child + reason = _unsafe_because(os.fstat(descriptor), uid=uid, own=True) + if reason is not None: + raise BundledServer._unsafe(path, reason) + finally: + os.umask(previous) + os.close(descriptor) + + +def os_user() -> str: + """The name of the OS user this runs as, which peer authentication maps. + + The SERVER resolves a connecting uid to a name through the same password + database, so a uid with no entry could not be authenticated at all, and is + refused here, by name. So is a name that `pg_ident.conf` could read as more + than a name: a regular expression (a leading `/`), a quote, a comment + mark, or white space. + """ + import pwd + + uid = os.getuid() + try: + name = pwd.getpwuid(uid).pw_name + except KeyError: + raise BundleRefused( + f"uid {uid} has no entry in the password database, and the bundled " + "server's peer authentication maps the connecting user BY NAME, so " + "it could never admit this one. Run the local install as a user " + "the system knows") from None + if not name or name.startswith("/") or any( + ch in name for ch in '"#') or any(ch.isspace() for ch in name): + raise BundleRefused( + f"the OS user name {name!r} cannot be written into the bundled " + "server's pg_ident.conf as a plain name (it holds a quote, a `#`, " + "white space, or starts with `/`)") + return name + + +def authentication_files(user: str) -> dict[str, str]: + """`pg_hba.conf` and `pg_ident.conf` for a local install run by `user`. + + RULED openxFactory#656 `5916000030` item 3 ("Peer auth + accept"): + * ONE local line, PEER through the `opendox` map. The kernel reports the + connecting process's uid, and the map admits `user` as the owner role + and as the served role, and nobody else as anybody. + * Every HOST connection is rejected. The server also listens on no TCP + address at all (`listen_addresses` is empty), so these lines never + match. They are written so that the file says what the install is. + * No replication line, so a PHYSICAL replication connection matches + no rule and is refused. A LOGICAL one (`replication=database`) + names a database, so `pg_hba.conf` reads it as the ordinary local + connection it resembles, and no rule here can tell the two apart: + it was accepted as `peer:` (adversarial review of + openDox-code#69). So the launch sets `max_wal_senders=0`, and the + server starts no WAL sender for either kind. A replication + connection is refused by the server, whatever the files say. + """ + header = ("# Written by opendox.runtime.bundle before every start of this local\n" + "# install's bundled server (plan 034 T072). Changes here are replaced.\n") + hba = (header + + "# TYPE DATABASE USER ADDRESS METHOD\n" + f"local all all peer map={IDENT_MAP}\n" + "host all all 0.0.0.0/0 reject\n" + "host all all ::/0 reject\n") + ident = (header + + "# MAPNAME SYSTEM-USERNAME PG-USERNAME\n" + f'{IDENT_MAP} "{user}" {BUNDLE_OWNER_ROLE}\n' + f'{IDENT_MAP} "{user}" {BUNDLE_SERVED_ROLE}\n') + return {"pg_hba.conf": hba, "pg_ident.conf": ident} + + +def write_authentication(data_dir: Path, user: str) -> None: + """Write both files into `data_dir`, each replaced atomically, mode 0600. + + Before EVERY launch, not only after `initdb`: a data directory an earlier + build initialized, or a file edited by hand, is brought back to the one + configuration this install runs with. + + THE 0600 IS SET, NOT ASKED FOR (Copilot review of openDox-code#69). An + `open` mode is only a creation request: the umask filters it, and it + changes nothing about a file that exists already. So a temporary file an + interrupted start left at the same name (a 0644 one, or a symbolic link) + is removed first. The new one is created EXCLUSIVELY and without following + a link, and its descriptor is set to exactly 0600 before anything is + written into it. + """ + for name, content in authentication_files(user).items(): + target = data_dir / name + temporary = data_dir / f".{name}.opendox-{os.getpid()}" + with contextlib.suppress(FileNotFoundError): + os.unlink(temporary) # an interrupted start's; a link itself, never its target + descriptor = os.open( + temporary, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, 0o600) + try: + os.fchmod(descriptor, 0o600) + with os.fdopen(descriptor, "w", encoding="utf-8") as handle: + descriptor = -1 # the handle closes it now + handle.write(content) + os.replace(temporary, target) + except BaseException: + if descriptor >= 0: + os.close(descriptor) + with contextlib.suppress(FileNotFoundError): + os.unlink(temporary) + raise + + +def _unsafe_because(info: os.stat_result, *, uid: int, own: bool) -> str | None: + """Why one directory of the socket's path is unsafe, or `None`.""" + mode = info.st_mode + if stat.S_ISLNK(mode): + return "is a symbolic link" + if not stat.S_ISDIR(mode): + return "is not a directory" + if own: + if info.st_uid != uid: + return f"is owned by uid {info.st_uid}, not by this user" + if mode & 0o022: + return (f"is writable by {'every user' if mode & 0o002 else 'its group'}" + f" (mode {stat.S_IMODE(mode):o})") + return None + if info.st_uid not in (uid, 0): + return f"is owned by uid {info.st_uid}, neither this user nor root" + # A GROUP IS OTHER USERS, the user's own primary group included: it can + # have other members (Copilot review of openDox-code#69). + if mode & 0o022 and not mode & stat.S_ISVTX: + return (f"is writable by {'every user' if mode & 0o002 else 'its group'}" + f" and is not sticky (mode {stat.S_IMODE(mode):o})") + return None + + +def refuse_an_unsafe_tree(bundle: DatabaseBundle, *, + existing_only: bool = False) -> None: + """The socket's whole path is this user's to change, or it is refused. + + The socket's directory is how this install's clients find ITS + server, so the 0700 on it is worth only what the path above it is + worth. Peer authentication keeps other users out of the server, but + not a substitute socket out of the path: whoever could replace `run` + could stand up a server of their own for this install's clients to + talk to. A directory entry is controlled by its PARENT: a parent that + another user can write lets them rename `run` away, or put a symbolic + link in its place, after the mode is set. A symbolic link on the way there + can be pointed elsewhere by whoever owns it, or by whoever can write + the directory it sits in (Copilot review of openDox-code#69). So: + + * THE INSTALL'S OWN TREE, as the configured path resolves: the + state directory, `postgres/` and `run/` must be real directories, + owned by this user and writable by no one else; + * EVERY DIRECTORY ABOVE IT, on the configured path and on the path + it resolves to, must be owned by this user or by root. One that + anyone else can write, a group included, must be sticky, as + `/tmp` is, so nobody can rename what is not theirs; + * EVERY SYMBOLIC LINK on the configured path must be this user's or + root's. + + `OPENDOX_STATE_DIR` never holds `..` (`config.state_dir` refuses it), + so the configured path's components are the ones the kernel walks. + + With `existing_only`, the same rules are asked of only what exists + yet. `_prepare_directories` asks that BEFORE it creates anything, + so the links are checked first, a broken one included (Copilot + review of openDox-code#69). + """ + uid = os.getuid() + configured = bundle.state_dir + + def present(path: Path) -> bool: + return not existing_only or os.path.lexists(path) + + for component in (configured, *configured.parents): + if not present(component): + continue + info = os.lstat(component) + if stat.S_ISLNK(info.st_mode) and info.st_uid not in (uid, 0): + raise BundledServer._unsafe( + component, f"is a symbolic link owned by uid {info.st_uid}, " + "neither this user nor root, who could point it elsewhere") + state = configured.resolve() + tree = [state, state / BUNDLE_TREE[0], state / BUNDLE_TREE[0] / BUNDLE_TREE[1]] + # AND THE DATA DIRECTORY, where one exists already, a broken link + # included (Copilot review of openDox-code#69). A `data` placed there + # as a link to a cluster elsewhere would otherwise be launched, and + # given this install's authentication files, outside the state tree. + # A fresh one needs no check: `_initialize` renames it into place. + data = state / BUNDLE_DATA_DIR + if os.path.lexists(data): + tree.append(data) + checks = [(path, True) for path in tree] + [ + (path, False) for path in dict.fromkeys( + [*state.parents, *configured.parents])] + for directory, mine in checks: + if not present(directory): + continue + info = os.lstat(directory) if mine else os.stat(directory) + reason = _unsafe_because(info, uid=uid, own=mine) + if reason is not None: + raise BundledServer._unsafe(directory, reason) + + +class BundledServer: + """One local install's PostgreSQL server: started as this process's child. + + Built from the settings a LOCAL `load_settings` returned, so the DSNs it + serves are the ones every other verb of the same install derives. `start` + initializes, launches, bootstraps and migrates; `stop` shuts it down; both + are safe to call more than once, and it is a context manager. + """ + + def __init__(self, settings: RuntimeSettings) -> None: + self.settings = settings + self.bundle = database_bundle(settings.state_dir) + self.process: subprocess.Popen[bytes] | None = None + self.applied: list[str] = [] + + @property + def log_path(self) -> Path: + return self.bundle.data_dir.parent / "postgres.log" + + def report(self) -> dict[str, Any]: + live = self.process is not None and self.process.poll() is None + return {"data_dir": str(self.bundle.data_dir), + "socket_dir": str(self.bundle.socket_dir), + "pid": self.process.pid if live else None} + + # -- start ------------------------------------------------------------- + + def start(self) -> BundledServer: + gap = unsupported_platform() + if gap is not None: + raise BundleRefused(gap) + if hasattr(os, "geteuid") and os.geteuid() == 0: + raise BundleRefused( + "the bundled PostgreSQL server refuses to run as root, and so " + "does this local install: run it as the ordinary user whose " + "documents it serves") + binaries = server_binaries() + already = running_pid(self.bundle) + if already is not None: + raise BundleRefused( + f"a server is already running on {self.bundle.data_dir} " + f"(pid {already}). One local install's database belongs to one " + "entry point at a time: stop the other `generate-and-open " + f"{LOCAL_FLAG}`, or give this one its own {PREFIX}STATE_DIR") + # THE WHOLE START IS ONE GUARDED OPERATION (Copilot review of + # openDox-code#69). The directories, `initdb` and the launch fail as + # plainly as the connection does: a timeout, a permission, a missing + # file. Each one comes out as the one named refusal the entry point + # prints, with whatever was started stopped, never as a traceback. + phase = "preparing its directories" + try: + self._prepare_directories() + phase = "initializing its data directory" + self._initialize(binaries) + phase = "configuring its authentication" + write_authentication(self.bundle.data_dir, os_user()) + phase = "launching it" + _remove_a_proven_stale_lock(self.bundle) + self._launch(binaries) + phase = "waiting for it to accept a connection" + self._wait_until_ready() + phase = "bootstrapping its database and served role" + self._bootstrap() + phase = "migrating it" + self._migrate() + except (BundleRefused, migrations.MigrationError): + self.stop() + raise + except Exception as exc: # noqa: BLE001 - named, never quoted + # THE DRIVER'S TEXT IS NOT REPEATED, for the reason `runtime/cli.py` + # gives: it quotes the connection string it was handed. The phase + # and the class name say what went wrong, and the server's own log + # says the rest. + self.stop() + raise BundleRefused( + f"the bundled PostgreSQL server could not be started " + f"({phase}: {type(exc).__name__}); its log is " + f"{self.log_path}") from None + except BaseException: + self.stop() + raise + return self + + def _prepare_directories(self) -> None: + """The state, data and socket directories, each 0700 where this makes it. + + A directory that already exists is NOT re-moded, on the rule the + runtime's `init` keeps for an operator's own path — except the socket + directory, which guards the only way in: it is this install's own, + under its own state directory, and it is narrowed to 0700 whatever it + was. + """ + # JUDGED BEFORE ANY WRITE, AND AGAIN AFTER (Copilot review of + # openDox-code#69). What exists already is checked first, so no + # directory is made through a link, or beneath a directory, that the + # tree check would refuse, and `_make_private_directories` refuses a + # name someone else put in its way. The whole tree is then checked + # BEFORE THE CHMOD, which follows a symbolic link: a `run` placed + # there as a link would otherwise have its TARGET re-moded. + self._refuse_an_unsafe_tree(existing_only=True) + _make_private_directories(self.bundle.socket_dir, + state=self.bundle.state_dir) + self._refuse_an_unsafe_tree() + os.chmod(self.bundle.socket_dir, 0o700) + + def _refuse_an_unsafe_tree(self, *, existing_only: bool = False) -> None: + """`refuse_an_unsafe_tree`, for this server's own bundle.""" + refuse_an_unsafe_tree(self.bundle, existing_only=existing_only) + + @staticmethod + def _unsafe(directory: Path, reason: str) -> BundleRefused: + return BundleRefused( + f"{directory} {reason}, so another user could replace the bundled " + "server's socket directory and put a server of their own where " + "this install's clients look for it. Use a state directory only " + f"this user can change ({PREFIX}STATE_DIR)") + + #: The prefix an initialization attempt's directory carries, beside the + #: data directory, followed by the pid of the process making it. + ATTEMPT_PREFIX = "data.initdb-" + + def _initialize(self, binaries: Path) -> None: + """A complete cluster at `data_dir`, or a refusal. Never a partial one. + + `initdb` runs into an ATTEMPT directory beside the data directory, and + the attempt is renamed into place only once `initdb` has succeeded. So + a data directory exists only as a finished cluster, and a first start + that dies midway (a kill, a full disk, a power cut) leaves an attempt + and not a half-built data directory that the next start would either + launch or fail to re-initialize for ever (Copilot review of + openDox-code#69). The next start removes an attempt whose process is + gone, and starts again. + + A data directory that exists and holds no cluster is NOT this + install's to remove, unless it is empty: it is refused, named, and + left as it is. + """ + data = self.bundle.data_dir + if (data / "PG_VERSION").is_file(): + self._refuse_another_major(binaries) + return + if data.exists(): + try: + data.rmdir() # an empty directory holds nothing + except OSError: + raise BundleRefused( + f"{data} exists and holds no PostgreSQL cluster, so it is " + "not this install's database, and it is left untouched: " + "move it aside, or give this install its own " + f"{PREFIX}STATE_DIR") from None + self._remove_abandoned_attempts() + attempt = Path(tempfile.mkdtemp( + prefix=f"{self.ATTEMPT_PREFIX}{os.getpid()}-", dir=data.parent)) + try: + self._initdb(binaries, attempt) + try: + os.rename(attempt, data) + except OSError: + if not (data / "PG_VERSION").is_file(): + raise + # another start finished first; its cluster is the one used + shutil.rmtree(attempt, ignore_errors=True) + except BaseException: + shutil.rmtree(attempt, ignore_errors=True) + raise + + def _refuse_another_major(self, binaries: Path) -> None: + """An existing cluster is opened only by the major that made it. + + PostgreSQL refuses another major's data directory itself, but only + from inside a launch, where the reason reaches nobody but the log. + The carrier's `pginstall/` is PostgreSQL 16 today, and upstream's + default is already 18 (adversarial review of openDox-code#69). So + the server's own `postgres --version` is asked first, against the + cluster's `PG_VERSION`, and a disagreement is the named refusal, + before anything is written into the data directory. + """ + data = self.bundle.data_dir + cluster = (data / "PG_VERSION").read_text(encoding="utf-8").strip() + done = subprocess.run( + [str(binaries / "postgres"), "--version"], env=_child_environment(), + capture_output=True, text=True, timeout=START_TIMEOUT_SECONDS) + found = re.search(r"\(PostgreSQL\) (\d+)", done.stdout or "") + if found is None: + raise BundleRefused( + f"the bundled `postgres` under {binaries} does not say which " + f"PostgreSQL it is (`postgres --version` exited " + f"{done.returncode}), so it is not given {data}, a " + f"PostgreSQL {cluster} cluster; reinstall the `local` extra") + if found.group(1) != cluster: + raise BundleRefused( + f"{data} holds a PostgreSQL {cluster} cluster and the bundled " + f"server is PostgreSQL {found.group(1)}: a cluster is opened " + "only by the major version that made it. Reinstall the " + f"`local` extra this install was made with (`{SERVER_DISTRIBUTION}` " + "is pinned below 0.7 for this reason), or move the data " + "directory aside, and lose its coordination state, to start " + "a new one") + + def _remove_abandoned_attempts(self) -> None: + """Every initialization attempt whose process no longer exists.""" + for candidate in self.bundle.data_dir.parent.glob( + f"{self.ATTEMPT_PREFIX}*"): + owner = candidate.name[len(self.ATTEMPT_PREFIX):].split("-", 1)[0] + try: + os.kill(int(owner), 0) + except ProcessLookupError: + shutil.rmtree(candidate, ignore_errors=True) + except (ValueError, OSError): + continue # alive, or not ours to judge + + def _initdb(self, binaries: Path, target: Path) -> None: + done = subprocess.run( + [str(binaries / "initdb"), "-D", str(target), + "-U", BUNDLE_OWNER_ROLE, "--auth-local=peer", "--auth-host=reject", + "--encoding=UTF8", "--locale=C", "--no-instructions"], + env=_child_environment(), capture_output=True, text=True, + timeout=START_TIMEOUT_SECONDS) + if done.returncode != 0: + detail = (done.stderr or done.stdout).strip().splitlines()[-5:] + raise BundleRefused( + f"initdb could not initialize {self.bundle.data_dir} " + f"(exit {done.returncode}): {' | '.join(detail)}") + + def _launch(self, binaries: Path) -> None: + log = open(self.log_path, "ab") # noqa: SIM115 - the child keeps it + try: + self.process = subprocess.Popen( + [str(binaries / "postgres"), "-D", str(self.bundle.data_dir), + "-k", str(self.bundle.socket_dir), "-p", str(BUNDLE_PORT), + # NO TCP LISTENER (13.1): an empty `listen_addresses` opens + # no TCP socket at all, so no pre-existing service on any + # port can stand in for this server, and nothing off this + # machine can reach it. + "-c", "listen_addresses=", + "-c", "unix_socket_permissions=0700", + # THE CLUSTER'S OWN FILES, PINNED (Copilot review of + # openDox-code#69). An existing cluster's + # `postgresql.conf` can point `data_directory`, `hba_file` + # and `ident_file` elsewhere: at an outside `trust` file + # that the files just rewritten would never replace, or at + # a cluster outside the state directory. The command line + # outranks every configuration file, so these three are + # the data directory and the two files written above. + "-c", f"data_directory={self.bundle.data_dir}", + "-c", f"hba_file={self.bundle.data_dir / 'pg_hba.conf'}", + "-c", f"ident_file={self.bundle.data_dir / 'pg_ident.conf'}", + # NO REPLICATION, physical or logical: see + # `authentication_files`, whose rules cannot refuse a + # logical one (adversarial review of openDox-code#69). + "-c", "max_wal_senders=0"], + stdin=subprocess.DEVNULL, stdout=log, stderr=subprocess.STDOUT, + env=_child_environment(), + # ITS OWN SESSION, so a terminal's Ctrl-C reaches this process + # and not the server behind its back: the stop is `stop()`'s, + # in order, after the document server has closed. + start_new_session=True, + preexec_fn=_die_with_parent()) + except subprocess.SubprocessError: + # `_die_with_parent`'s child refused to run unarmed: nothing started + raise BundleRefused(_UNARMED) from None + finally: + log.close() + + def _wait_until_ready(self) -> None: + import psycopg + + deadline = time.monotonic() + START_TIMEOUT_SECONDS + last = "no attempt" + while time.monotonic() < deadline: + if self.process is not None and self.process.poll() is not None: + raise BundleRefused( + f"the bundled PostgreSQL server exited during start " + f"(exit {self.process.returncode}); its log is " + f"{self.log_path}: {self._log_tail()}") + try: + with psycopg.connect(self._dsn(BUNDLE_OWNER_ROLE, "postgres"), + connect_timeout=2, autocommit=True) as conn: + conn.execute("select 1") + # READY MEANS THIS CHILD IS SERVING, not merely that the socket + # answered (Copilot review of openDox-code#69). Two entry points + # racing from an idle state both launch. The loser's `postgres` + # lives a moment before it refuses the winner's lock, and the + # winner's socket already answers. So once a connection has + # answered, the data directory's lock file must name THIS + # child. Otherwise the wait goes on until this child exits and + # is refused. One check, AFTER the connection, is enough. The + # lock admits one postmaster per data directory, the socket + # directory belongs to exactly one data directory, and a + # lock naming this child therefore means the socket that + # answered is this child's. + if self._serving_is_this_child(): + return + last = "the socket answered, but not from this child" + time.sleep(0.1) + except psycopg.OperationalError as exc: + # THE CLASS NAME ONLY: a driver's message quotes the DSN it + # could not reach, and this package never repeats one + # (`runtime/cli.py`'s redaction contract). + last = type(exc).__name__ + time.sleep(0.1) + raise BundleRefused( + f"the bundled PostgreSQL server did not accept a connection within " + f"{START_TIMEOUT_SECONDS:.0f}s ({last}); its log is " + f"{self.log_path}") + + def _serving_is_this_child(self) -> bool: + """Whether the data directory's lock file names the child just launched.""" + return (self.process is not None + and _lock_file_pid(self.bundle) == self.process.pid) + + def _dsn(self, role: str, database: str) -> str: + """`DatabaseBundle.dsn`, aimed at a database other than the served one.""" + return self.bundle.dsn(role).replace( + f"@/{BUNDLE_DATABASE}?", f"@/{database}?", 1) + + def _bootstrap(self) -> None: + """The database, the served role and its grants — idempotent. + + The grants are the compose stack's (`init-runtime-role.sh`), narrowed + the same way: CONNECT on the database, USAGE on `public`, and DML on + every table the OWNER creates from here on, by default privileges — + which on this install's own fresh database is the six coordination + tables and the ledger, and nothing else. The migration then narrows + the served role's rights on the ledger to SELECT. + """ + import psycopg + from psycopg import sql + + with psycopg.connect(self._dsn(BUNDLE_OWNER_ROLE, "postgres"), + autocommit=True) as conn: + if conn.execute("select 1 from pg_database where datname = %s", + (BUNDLE_DATABASE,)).fetchone() is None: + conn.execute(sql.SQL("create database {} owner {}").format( + sql.Identifier(BUNDLE_DATABASE), + sql.Identifier(BUNDLE_OWNER_ROLE))) + if conn.execute("select 1 from pg_roles where rolname = %s", + (BUNDLE_SERVED_ROLE,)).fetchone() is None: + conn.execute(sql.SQL("create role {} login").format( + sql.Identifier(BUNDLE_SERVED_ROLE))) + with psycopg.connect(self.bundle.migration_dsn, autocommit=True) as conn: + served, owner = (sql.Identifier(BUNDLE_SERVED_ROLE), + sql.Identifier(BUNDLE_OWNER_ROLE)) + conn.execute(sql.SQL("grant connect on database {} to {}").format( + sql.Identifier(BUNDLE_DATABASE), served)) + conn.execute(sql.SQL("grant usage on schema public to {}").format( + served)) + conn.execute(sql.SQL( + "alter default privileges for role {} in schema public grant " + "select, insert, update, delete on tables to {}").format( + owner, served)) + + def _migrate(self) -> None: + """Starting AND MIGRATING is all release 1 asks of the store (R1Q16 (ii)).""" + from opendox.runtime.db import Database + + database = Database(self.bundle.migration_dsn, + application_name="opendox-local-migrate") + with database: + self.applied = migrations.MigrationRunner( + database, migrations_dir=self.settings.migrations_dir, + runtime_role=self.settings.runtime_pg_role, + served_schema=self.settings.served_schema, + served_database=self.settings.served_database).apply() + + def _log_tail(self) -> str: + try: + lines = self.log_path.read_text(encoding="utf-8", + errors="replace").splitlines() + except OSError: + return "(no log)" + return " | ".join(lines[-5:]) + + # -- stop -------------------------------------------------------------- + + def stop(self) -> None: + """A FAST shutdown (SIGINT), then an immediate one, then a kill. + + FAST, and not PostgreSQL's default "smart" shutdown, because the + document server that is its only client has already closed: waiting + for sessions to end would wait for nothing, or for a leaked + connection. Safe to call twice, and on a server that never started. + """ + process, self.process = self.process, None + if process is None or process.poll() is not None: + return + for sig, wait in ((signal.SIGINT, STOP_TIMEOUT_SECONDS), + (signal.SIGQUIT, 5.0), (signal.SIGKILL, 5.0)): + try: + process.send_signal(sig) + except ProcessLookupError: + return + try: + process.wait(timeout=wait) + return + except subprocess.TimeoutExpired: + continue + + def __enter__(self) -> BundledServer: + return self.start() + + def __exit__(self, *exc: object) -> None: + self.stop() + + +__all__ = ["BUNDLE_PORT", "BundleRefused", "BundledServer", + "authentication_files", "isolated_from_libpq_environment", + "os_user", "refusal_before_connecting", "refuse_an_unsafe_tree", + "report", "running_pid", "server_binaries", "unsupported_platform", + "write_authentication"] diff --git a/src/opendox/runtime/cli.py b/src/opendox/runtime/cli.py index a69bfd41..605f65ad 100644 --- a/src/opendox/runtime/cli.py +++ b/src/opendox/runtime/cli.py @@ -80,6 +80,7 @@ import contextlib import json import logging +import os import re import stat import sys @@ -97,6 +98,7 @@ ConfigurationError, redacted_url, RuntimeSettings, + install_mode, load_migration_settings, load_settings, migration_database_url, @@ -348,6 +350,10 @@ def _redacted_settings(settings: RuntimeSettings) -> dict[str, Any]: # needs to know, because it decides whether the broker lines below # mean anything at all. "OPENDOX_INSTALL_MODE": settings.install_mode, + # WHERE A LOCAL INSTALL'S BUNDLED SERVER LIVES (plan 034 T072): a path + # and never a credential. Reported for a hosted install too, which + # never reads it, so the report covers the whole declared list. + "OPENDOX_STATE_DIR": str(settings.state_dir), # THE BROKER URLS ARE REDACTED HERE TOO. `load_settings` refuses # userinfo in the issuer and in an explicit JWKS URL — but this report # prints a DERIVED value, and a settings object can also be built by @@ -485,6 +491,24 @@ def cmd_init(args: argparse.Namespace) -> int: "next": "opendox-runtime runtime migrate"}, ok=True) +def _local_bundle_refusal(settings) -> str | None: + """For a LOCAL install, why its socket must not be connected to, or `None`. + + `bundle.refusal_before_connecting`: the tree check a start asks, and a + live server of THIS data directory behind the socket, both before any + client connection (adversarial review of openDox-code#69). A hosted + install's DSN is the operator's, and is not judged here. + """ + if settings.install_mode != INSTALL_MODE_LOCAL: + return None + from opendox.runtime import bundle as bundle_mod + from opendox.runtime.config import database_bundle + + reason = bundle_mod.refusal_before_connecting( + database_bundle(settings.state_dir)) + return None if reason is None else _safe_message(reason) + + def cmd_migrate(args: argparse.Namespace) -> int: """Apply the ordered SQL, or with `--plan` report what would be applied. @@ -520,6 +544,10 @@ def cmd_migrate(args: argparse.Namespace) -> int: "package with the `runtime` extra: " "pip install '.[runtime]'"}, ok=False) + refusal = _local_bundle_refusal(settings) + if refusal is not None: + return _emit({"verb": "migrate", "refusal": "local-bundle-unverified", + "message": refusal}, ok=False) runner_db = Database(dsn, application_name="opendox-runtime-migrate", checkout_timeout=args.connect_timeout) # THE OUTCOME IS COMPUTED INSIDE THE CONTEXT AND EMITTED OUTSIDE IT. A @@ -693,6 +721,20 @@ def cmd_status(args: argparse.Namespace) -> int: return _emit({"verb": "status", "refusal": "configuration", "message": _safe_message(exc)}, ok=False) report["settings"] = _redacted_settings(settings) + # THE BUNDLED SERVER THIS INSTALL OWNS (plan 034 T072; #1144 13.1): where + # its data directory and socket are, and the pid of the server running on + # them, read from the server's own `postmaster.pid`. `null` for a hosted + # install, which brings no server. Reported, never started: `status` + # changes nothing, and the process that owns the server is the document + # server that started it (R1Q16 (i)). + if settings.install_mode == INSTALL_MODE_LOCAL: + from opendox.runtime import bundle as bundle_mod + from opendox.runtime.config import database_bundle + + report["database_bundle"] = bundle_mod.report( + database_bundle(settings.state_dir)) + else: + report["database_bundle"] = None try: report["canonical_sha256"] = migrations.verify_canonical_digest( @@ -722,76 +764,85 @@ def cmd_status(args: argparse.Namespace) -> int: report["runtime_extra"] = "present" connected = False - try: - # INSIDE the context, all of it. `runner.applied()` and `runner.plan()` - # each check a connection out of the pool, so calling them after the - # `with` had closed it raised `PoolClosed` and this verb reported a - # perfectly reachable database as unreachable (Copilot review of - # openDox-code#25, and it is the kind of defect only a live database - # shows — every unreachable-database test passed). - with Database(settings.database_url, - checkout_timeout=args.probe_timeout) as db: - with db.connection() as conn: - conn.execute("select 1") - # THE CONNECTIVITY ANSWER IS RECORDED THE MOMENT IT IS TRUE, so a - # failure in the queries BELOW cannot rewrite it — see the generic - # handler at the end of this block. - connected = True - # THE SAME CANONICAL GATE `apply()` AND `/readyz` RUN. Without - # it an EMPTY migrations directory reports `pending: []` on a - # fresh database — nothing pending, nothing drifted, everything - # fine — for an install with no coordination schema at all - # (Copilot review of openDox-code#25, round 7). `status` is the - # verb an operator believes. - migrations.verify_canonical_digest(settings.migrations_dir) - runner = migrations.MigrationRunner( - db, migrations_dir=settings.migrations_dir) - applied = [row.version for row in runner.applied()] - pending = [m.version for m in runner.plan()] - drifted = runner.drift() - report["database"] = "reachable" - report["applied_migrations"] = applied - report["pending_migrations"] = pending - # NOTHING PENDING IS NOT THE SAME AS MATCHING THIS TREE: a migration - # whose file changed, or vanished, is invisible to `plan()` and is - # refused by `apply()`. See `MigrationRunner.drift`. - report["migration_drift"] = drifted - # PENDING IS UNHEALTHY, exactly as `/readyz` treats it. This reported - # the versions and left `ok` true, so a reachable but UNMIGRATED - # database exited 0 while the readiness probe on the same install - # refuses traffic — two answers to one question, and the CLI's was the - # comforting one (Copilot review of openDox-code#25, round 7). - if drifted or pending: - ok = False - # a status verb reports, never raises - except migrations.MigrationError as exc: - # THE DATABASE ANSWERED; THE TREE DID NOT. `select 1` has already - # succeeded by the time the runner is asked anything, so reporting - # `database: unreachable` for a missing or malformed migrations - # directory pointed the operator at the wrong dependency entirely - # (Copilot review of openDox-code#25, round 7). - report.setdefault("database", "reachable") - report["migrations"] = ( - f"unreadable: {type(exc).__name__}: {_safe_message(exc)}") + # A LOCAL INSTALL'S SOCKET IS JUDGED BEFORE IT IS CONNECTED TO + # (adversarial review of openDox-code#69): the tree a start checks, and a + # live server of this data directory behind it. Otherwise `status` asks + # whatever answers at that path, as the served role. + refusal = _local_bundle_refusal(settings) + if refusal is not None: + report["database"] = f"not probed: {refusal}" ok = False - except Exception as exc: # noqa: BLE001 - # THE SAME DISTINCTION THE BRANCH ABOVE MAKES, for the failures that - # are not the runner's own. Once `select 1` has answered, the database - # IS reachable, and a later failure — the served role without `select` - # on the ledger, a schema the search path does not reach, a query that - # errors — is a privilege or schema problem reported as one. Reporting - # `database: unreachable` for it pointed the operator at the network - # and hid the real fault, which is the defect round 7 fixed for - # `MigrationError` and left in place one handler down (Copilot review - # of openDox-code#25, round 10, suppressed). - if connected: + else: + try: + # INSIDE the context, all of it. `runner.applied()` and `runner.plan()` + # each check a connection out of the pool, so calling them after the + # `with` had closed it raised `PoolClosed` and this verb reported a + # perfectly reachable database as unreachable (Copilot review of + # openDox-code#25, and it is the kind of defect only a live database + # shows — every unreachable-database test passed). + with Database(settings.database_url, + checkout_timeout=args.probe_timeout) as db: + with db.connection() as conn: + conn.execute("select 1") + # THE CONNECTIVITY ANSWER IS RECORDED THE MOMENT IT IS TRUE, so a + # failure in the queries BELOW cannot rewrite it — see the generic + # handler at the end of this block. + connected = True + # THE SAME CANONICAL GATE `apply()` AND `/readyz` RUN. Without + # it an EMPTY migrations directory reports `pending: []` on a + # fresh database — nothing pending, nothing drifted, everything + # fine — for an install with no coordination schema at all + # (Copilot review of openDox-code#25, round 7). `status` is the + # verb an operator believes. + migrations.verify_canonical_digest(settings.migrations_dir) + runner = migrations.MigrationRunner( + db, migrations_dir=settings.migrations_dir) + applied = [row.version for row in runner.applied()] + pending = [m.version for m in runner.plan()] + drifted = runner.drift() report["database"] = "reachable" - report["schema_queries"] = ( - f"failed: {type(exc).__name__}: {_safe_message(exc)}") - else: - report["database"] = ( - f"unreachable: {type(exc).__name__}: {_safe_message(exc)}") - ok = False + report["applied_migrations"] = applied + report["pending_migrations"] = pending + # NOTHING PENDING IS NOT THE SAME AS MATCHING THIS TREE: a migration + # whose file changed, or vanished, is invisible to `plan()` and is + # refused by `apply()`. See `MigrationRunner.drift`. + report["migration_drift"] = drifted + # PENDING IS UNHEALTHY, exactly as `/readyz` treats it. This reported + # the versions and left `ok` true, so a reachable but UNMIGRATED + # database exited 0 while the readiness probe on the same install + # refuses traffic — two answers to one question, and the CLI's was the + # comforting one (Copilot review of openDox-code#25, round 7). + if drifted or pending: + ok = False + # a status verb reports, never raises + except migrations.MigrationError as exc: + # THE DATABASE ANSWERED; THE TREE DID NOT. `select 1` has already + # succeeded by the time the runner is asked anything, so reporting + # `database: unreachable` for a missing or malformed migrations + # directory pointed the operator at the wrong dependency entirely + # (Copilot review of openDox-code#25, round 7). + report.setdefault("database", "reachable") + report["migrations"] = ( + f"unreadable: {type(exc).__name__}: {_safe_message(exc)}") + ok = False + except Exception as exc: # noqa: BLE001 + # THE SAME DISTINCTION THE BRANCH ABOVE MAKES, for the failures that + # are not the runner's own. Once `select 1` has answered, the database + # IS reachable, and a later failure — the served role without `select` + # on the ledger, a schema the search path does not reach, a query that + # errors — is a privilege or schema problem reported as one. Reporting + # `database: unreachable` for it pointed the operator at the network + # and hid the real fault, which is the defect round 7 fixed for + # `MigrationError` and left in place one handler down (Copilot review + # of openDox-code#25, round 10, suppressed). + if connected: + report["database"] = "reachable" + report["schema_queries"] = ( + f"failed: {type(exc).__name__}: {_safe_message(exc)}") + else: + report["database"] = ( + f"unreachable: {type(exc).__name__}: {_safe_message(exc)}") + ok = False # A LOCAL INSTALL HAS NO BROKER TO PROBE (plan 034 T070; #1144 13.4), # and that is its configuration rather than a fault: reported by name, and @@ -845,6 +896,10 @@ def cmd_reset(args: argparse.Namespace) -> int: except ImportError as exc: # pragma: no cover - the extra is absent return _emit({"verb": "reset", "refusal": "runtime-extra-missing", "message": _safe_message(exc)}, ok=False) + refusal = _local_bundle_refusal(settings) + if refusal is not None: + return _emit({"verb": "reset", "refusal": "local-bundle-unverified", + "message": refusal}, ok=False) try: with Database(dsn, application_name="opendox-runtime-reset", checkout_timeout=args.connect_timeout) as db: @@ -1207,6 +1262,27 @@ def build_parser() -> argparse.ArgumentParser: return parser +def _isolated_when_local() -> contextlib.AbstractContextManager[None]: + """A LOCAL install's verbs run with libpq's `PG*` defaults out of reach. + + The bundle's DSNs name their socket, but libpq fills everything else from + the environment, and `PGHOSTADDR` alone would send `status` or `migrate` + to a TCP server instead (Copilot review of openDox-code#69; see + `bundle.isolated_from_libpq_environment`). A hosted install's operator + configures libpq as they please, as before (13.6). A selector that cannot + be read isolates nothing; the verb refuses it by name. + """ + try: + local = install_mode(os.environ) == INSTALL_MODE_LOCAL + except ConfigurationError: + local = False + if not local: + return contextlib.nullcontext() + from opendox.runtime import bundle as bundle_mod + + return bundle_mod.isolated_from_libpq_environment() + + def main(argv: list[str] | None = None) -> int: """Parse and dispatch, and NEVER let a traceback be the whole answer. @@ -1222,7 +1298,8 @@ def main(argv: list[str] | None = None) -> int: """ args = build_parser().parse_args(argv) try: - return int(args.func(args)) + with _isolated_when_local(): + return int(args.func(args)) except SystemExit: raise except Exception as exc: # noqa: BLE001 diff --git a/src/opendox/runtime/config.py b/src/opendox/runtime/config.py index 8f169165..b220a6d5 100644 --- a/src/opendox/runtime/config.py +++ b/src/opendox/runtime/config.py @@ -26,14 +26,18 @@ from __future__ import annotations +import importlib.metadata import ipaddress import os import re import shlex +import sys +import tomllib import urllib.parse from collections.abc import Mapping from dataclasses import dataclass -from pathlib import Path +from pathlib import Path, PurePath +from typing import Any #: The environment prefix. One string, so a rename is one edit. PREFIX = "OPENDOX_" @@ -97,6 +101,18 @@ class Setting: "neither the install is hosted, so a hosted install with no issuer " "refuses rather than falling into local mode (13.4, 13.5)", ), + # WHERE A LOCAL INSTALL KEEPS ITS OWN STATE (plan 034 T072; #1144 13.1): + # the bundled PostgreSQL server's data directory and its Unix socket. No + # default string, because the default is COMPUTED, per user — see + # `state_dir`. A hosted install never reads it. + Setting( + PREFIX + "STATE_DIR", None, False, False, + "the directory a LOCAL install owns: the bundled PostgreSQL server's " + "data directory and its Unix socket live under it, and the server " + "listens on that socket and on no TCP port (13.1). Unset, it is " + "`$XDG_STATE_HOME/opendox`, else `~/.local/state/opendox`. A hosted " + "install never reads it", + ), Setting( PREFIX + "OIDC_ISSUER", None, True, False, "the Keycloak broker's issuer, pinned: a token from any other issuer " @@ -172,7 +188,13 @@ class Setting: ), Setting( PREFIX + "MIGRATIONS_DIR", "migrations", False, False, - "the ordered-SQL directory, repository-root-relative", + "the ordered-SQL directory, repository-root-relative. Unset, a HOSTED " + "install uses `migrations` where the working directory holds one (a " + "checkout, or the image's /app), as it always has, and otherwise the " + "copy this installation carries; a LOCAL install uses ONLY the copy " + "this installation carries and never the working directory's (plan " + "034 T072), so launching it from a checkout of somebody else's " + "repository cannot run that repository's SQL", ), Setting( PREFIX + "PROJECT_REPOSITORY_ROOT", "var/projects", False, False, @@ -224,6 +246,7 @@ class RuntimeSettings: database_url: str migration_database_url: str | None install_mode: str + state_dir: Path oidc_issuer: str oidc_audience: str oidc_jwks_url: str | None @@ -246,6 +269,7 @@ def __repr__(self) -> str: "migration_database_url=" f"{'' if self.migration_database_url else 'None'}, " f"install_mode={self.install_mode!r}, " + f"state_dir={str(self.state_dir)!r}, " # REDACTED TOO, and not because `load_settings` allows userinfo # here — it refuses it. A `RuntimeSettings` built by hand, in a # test or by a future caller, does not go through that door, and @@ -1582,13 +1606,335 @@ def _refuse_two_dsns_that_select_different_schemas( #: is REFUSED, by name (a holder reading on openxFactory#656, plan 034 T070, #: the same fail-closed reading as the disagreeing flag and setting): an issuer #: next to `local` says a broker was meant, and honouring `local` over it would -#: silently drop the authentication the operator configured. T072 adds the two -#: DSNs, which the local install supplies itself (13.1). -HOSTED_ONLY_SETTINGS: tuple[str, ...] = ( +#: silently drop the authentication the operator configured. +#: +#: AND THE TWO DSNs (plan 034 T072; the same holder reading): a local install +#: SUPPLIES BOTH ITSELF, from the server it bundles (#1144 13.1, "so no +#: pre-existing service can stand in for it"), so an operator's DSN beside +#: `local` is either about to be silently overridden or is another server +#: trying to stand in for the bundled one. Neither is accepted. +#: +#: TWO CLASSES, TWO REASONS (Copilot review of openDox-code#69): the broker +#: settings are refused because honouring `local` would drop an +#: authentication, and the DSNs because the local install supplies its own +#: database. Each refusal says its own reason. +BROKER_SETTINGS: tuple[str, ...] = ( PREFIX + "OIDC_ISSUER", PREFIX + "OIDC_AUDIENCE", PREFIX + "OIDC_JWKS_URL", ) +OPERATOR_DATABASE_SETTINGS: tuple[str, ...] = ( + PREFIX + "DATABASE_URL", + PREFIX + "MIGRATION_DATABASE_URL", +) +HOSTED_ONLY_SETTINGS: tuple[str, ...] = BROKER_SETTINGS + OPERATOR_DATABASE_SETTINGS + +#: THE BUNDLED SERVER'S IDENTITY (plan 034 T072; #1144 13.1, as T007 batch H's +#: addendum reads). The data directory and the socket directory live under the +#: install's own `OPENDOX_STATE_DIR`, at these paths — SHORT ONES, because a +#: Unix socket's whole path is bounded by the kernel (`sun_path`) and the +#: socket file is `/.s.PGSQL.`. +BUNDLE_DATA_DIR = PurePath("postgres", "data") +BUNDLE_SOCKET_DIR = PurePath("postgres", "run") +#: The port NUMBER, which names the socket file and opens NO TCP port: the +#: server is started with `listen_addresses` empty (13.1: "on NO TCP port"). +BUNDLE_PORT = 5432 +#: The two identities 13.3 keeps apart, and the one database. The MIGRATION +#: identity owns the database and every table it creates; the SERVED identity +#: is the least-privileged role the API reads and writes as, granted what +#: `deploy/compose/init-runtime-role.sh` grants the compose stack's role of the +#: same name. Two DSNs, two users, never one pasted twice (13.3). +BUNDLE_OWNER_ROLE = "opendox" +BUNDLE_SERVED_ROLE = "opendox_runtime" +BUNDLE_DATABASE = "opendox" + +#: The longest socket path the kernel takes, in bytes: `sizeof(sun_path)` less +#: its terminating NUL — 108 on Linux, 104 on macOS and the BSDs. PostgreSQL +#: refuses a longer one at startup; this refuses it at configuration, naming +#: the setting that made it long. +UNIX_SOCKET_PATH_MAX = 107 if sys.platform.startswith("linux") else 103 + + +@dataclass(frozen=True) +class DatabaseBundle: + """Where a LOCAL install's bundled PostgreSQL server lives, and its DSNs. + + Pure path and string arithmetic over `OPENDOX_STATE_DIR`, so `load_settings` + can name both DSNs without starting anything, and `runtime status` in a + second process derives the SAME ones and finds the same server. + `opendox.runtime.bundle` is what starts and stops it. + """ + + state_dir: Path + + @property + def data_dir(self) -> Path: + return self.state_dir / BUNDLE_DATA_DIR + + @property + def socket_dir(self) -> Path: + return self.state_dir / BUNDLE_SOCKET_DIR + + @property + def socket_path(self) -> Path: + return self.socket_dir / f".s.PGSQL.{BUNDLE_PORT}" + + def dsn(self, role: str) -> str: + """A DSN for `role` over the bundle's Unix socket, and never TCP. + + `host` is the socket DIRECTORY (libpq's rule for a value that starts + with `/`), percent-encoded so a state directory holding a space or a + `&` is still one value; `port` is spelled so a `PGPORT` in the + environment cannot send libpq to a different socket file. NO + PASSWORD, because there is none to give: the server authenticates a + Unix-socket connection by PEER (RULED openxFactory#656 `5916000030` + item 3). The kernel reports the connecting process's uid, and + `pg_ident.conf` maps this install's OS user, and nobody else, to the + two roles. The socket's directory is 0700, the server opens no TCP + port, and a host connection is rejected outright. SonarCloud's S2115 + ("add password protection") is ACCEPTED on this line for that reason, + with the same ruling as its authority. + """ + host = urllib.parse.quote(str(self.socket_dir), safe="/") + return (f"postgresql://{role}@/{BUNDLE_DATABASE}" + f"?host={host}&port={BUNDLE_PORT}") + + @property + def served_dsn(self) -> str: + return self.dsn(BUNDLE_SERVED_ROLE) + + @property + def migration_dsn(self) -> str: + return self.dsn(BUNDLE_OWNER_ROLE) + + +def state_dir(env: Mapping[str, str] | None = None) -> Path: + """The install's own state directory: `OPENDOX_STATE_DIR`, or the per-user one. + + ABSOLUTE, or refused: the document server that starts the bundled server + and a `runtime status` run from another directory must derive the same + socket, and a relative value would give each its own. Unset, it is + `$XDG_STATE_HOME/opendox` where that is absolute (the XDG rule ignores a + relative one), and otherwise `~/.local/state/opendox`. + """ + env = os.environ if env is None else env + setting = PREFIX + "STATE_DIR" + raw = env.get(setting, "").strip() + if raw: + try: + path = Path(raw).expanduser() + except RuntimeError: + # `~nosuchuser/...`: `expanduser` raises rather than answering, and + # a setting is refused by name, never by a traceback (Copilot + # review of openDox-code#69). + raise ConfigurationError( + f"{setting} is {raw!r}, whose `~` names no user this system " + "knows, so it expands to no directory. Name the state " + "directory absolutely") from None + if ".." in path.parts: + # PARENT TRAVERSAL IS REFUSED, so the path the bundle checks is + # the one the kernel walks: `a/../b` names `b` lexically and + # something else wherever `a` is a symbolic link (Copilot review + # of openDox-code#69). + raise ConfigurationError( + f"{setting} is {raw!r}, which climbs out through `..`. Name " + "the state directory directly") + if not path.is_absolute(): + raise ConfigurationError( + f"{setting} is {raw!r}, which is not an absolute path. The " + "document server that starts the bundled PostgreSQL server " + "and a `runtime status` run from another directory must find " + "the same socket, so the state directory is named absolutely") + return path + xdg = env.get("XDG_STATE_HOME", "").strip() + if xdg and Path(xdg).is_absolute(): + if ".." in Path(xdg).parts: + raise ConfigurationError( + f"{setting} is unset and XDG_STATE_HOME is {xdg!r}, which " + f"climbs out through `..`. Set {setting}, or XDG_STATE_HOME, " + "to the directory itself") + return Path(xdg) / "opendox" + try: + home = Path.home() + except RuntimeError: + raise ConfigurationError( + f"{setting} is unset and this process has no home directory to " + "put the default under (no HOME, and no password entry for the " + f"user). Set {setting} to an absolute path") from None + if not home.is_absolute(): + # `Path.home()` returns HOME as given, and a relative one would give + # the serving process and a `runtime status` run from another + # directory two different sockets (Copilot review of openDox-code#69). + raise ConfigurationError( + f"{setting} is unset and HOME is {str(home)!r}, which is not an " + "absolute path, so the default state directory would depend on " + f"the working directory. Set {setting} to an absolute path, or " + "HOME to one") + if ".." in home.parts: + raise ConfigurationError( + f"{setting} is unset and HOME is {str(home)!r}, which climbs out " + f"through `..`. Set {setting} to the directory itself") + return home / ".local" / "state" / "opendox" + + +def database_bundle(state: Path) -> DatabaseBundle: + """The bundle under `state`, refusing a socket path the kernel cannot bind. + + A COMMA IS REFUSED, wherever the state directory came from + (`OPENDOX_STATE_DIR`, `XDG_STATE_HOME` or the home directory), because + both ends read the socket directory as a LIST. PostgreSQL splits `-k` + (`unix_socket_directories`) on commas, and libpq splits a `host` on them + once the DSN's percent-encoding is decoded. So `…/a,…/b` made the server + put its sockets in two directories nothing here had checked, one of + them anyone could write, while the checked 0700 one stayed empty + (adversarial review of openDox-code#69). The value is not repeated: + the refusal names where it came from, and that is enough to find it. + """ + bundle = DatabaseBundle(state_dir=state) + if "," in str(state): + raise ConfigurationError( + "the state directory's path holds a `,`. PostgreSQL reads its " + "socket directories, and libpq its hosts, as comma-separated " + "lists, so a comma would split this install's one socket " + "directory into two it never checked. Choose a state directory " + f"without one: {PREFIX}STATE_DIR or, where that is unset, " + "XDG_STATE_HOME or HOME") + length = len(os.fsencode(str(bundle.socket_path))) + if length > UNIX_SOCKET_PATH_MAX: + raise ConfigurationError( + f"{PREFIX}STATE_DIR is too long for the bundled server's Unix " + f"socket: {bundle.socket_path} is {length} bytes and this kernel " + f"takes at most {UNIX_SOCKET_PATH_MAX}. The socket must live under " + "the install's own state directory (13.1), so choose a shorter " + f"{PREFIX}STATE_DIR") + return bundle + + +#: WHERE AN INSTALLED WHEEL KEEPS ITS MIGRATIONS (plan 034 T072). The +#: repository's `migrations/` stays where it is — the image copies it to +#: `/app/migrations` and runs from `/app` — and `pyproject.toml` maps the same +#: files into the wheel's data directory under this path, so an install run +#: outside any checkout still has the migrations it applies. The canonical +#: digest gate (`migrations.verify_canonical_digest`) is what proves any copy +#: found this way is the pinned one. +PACKAGED_MIGRATIONS = PurePath("share", "opendox", "migrations") + + +def _source_tree_migrations(module_file: Path) -> Path | None: + """The `migrations/` of the SOURCE TREE `module_file` was imported from. + + `module_file` is this module (`src/opendox/runtime/config.py`), so the + tree is three directories up: a checkout run with `src/` on the path, or + an editable install, neither of which installs data files. It counts only + where it really is that tree: `src/` is the directory the package sits + in, and the root's `pyproject.toml` names THIS project. A wheel's + `site-packages`, or a `--target` directory that happens to sit inside + some other checkout, is neither. + """ + here = module_file.resolve() + package_parent, root = here.parents[2], here.parents[3] + if package_parent.name != "src": + return None + try: + project = tomllib.loads( + (root / "pyproject.toml").read_text(encoding="utf-8")) + except (OSError, tomllib.TOMLDecodeError): + return None + if project.get("project", {}).get("name") != "opendox": + return None + candidate = root / "migrations" + return candidate if candidate.is_dir() else None + + +def _distribution_migrations(module_file: Path, + distribution: Any | None = None) -> Path | None: + """The migrations the INSTALLED DISTRIBUTION of `module_file` carries. + + From the distribution's own `RECORD`, wherever its install scheme put the + data files — and ONLY when that distribution is the one `module_file` was + loaded from. A name lookup alone is not: with a checkout's `src/` on the + path, `distribution("opendox")` can find an older wheel installed beside + it, and that wheel's migrations are another version's (Copilot review of + openDox-code#69). + """ + if distribution is None: + try: + distribution = importlib.metadata.distribution("opendox") + except importlib.metadata.PackageNotFoundError: + return None + files = list(distribution.files or ()) + here = module_file.resolve() + tail = here.parts[-3:] # ("opendox", "runtime", "config.py") + if not any(PurePath(entry).parts[-3:] == tail + and Path(entry.locate()).resolve() == here for entry in files): + return None + for entry in files: + parts = PurePath(entry).parts + if (entry.name.endswith(".sql") + and tuple(parts[-4:-1]) == PACKAGED_MIGRATIONS.parts): + return Path(entry.locate()).resolve().parent + return None + + +def installation_migrations_dir() -> Path | None: + """The migrations THIS INSTALLATION carries, or `None` where it carries none. + + Tied to the code that is running, never to a name or to the working + directory. First the source tree this module was imported from (a + checkout, or an editable install). Then the installed distribution that + this module belongs to, from its `RECORD` (a wheel install). A real wheel + install has no `pyproject.toml` beside its package, so it falls through to + its own `RECORD` (Copilot review of openDox-code#69). + """ + module_file = Path(__file__) + return (_source_tree_migrations(module_file) + or _distribution_migrations(module_file)) + + +def migrations_dir(env: Mapping[str, str] | None = None, *, + local: bool = False) -> Path: + """`OPENDOX_MIGRATIONS_DIR`, or where this install's migrations are. + + SET, it is used as given, in either shape: executing another directory's + SQL is something an operator says, not something a directory implies. + + UNSET, the shape decides. + * A LOCAL install uses ONLY the copy this installation carries + (`installation_migrations_dir`), never the working directory's. Its + entry point runs every migration it finds as the bundled server's + OWNER, and the canonical gate pins `0001` alone. So a `migrations/` + in whatever directory a user launches it from, holding the pinned + `0001` and SQL of its own, would otherwise be executed (Copilot + review of openDox-code#69). An installation that carries none is + refused, naming the setting. + * A HOSTED install keeps today's default, unchanged (13.6): `migrations` + where the working directory holds one (the image's `/app`, a + checkout), and otherwise the copy this installation carries. Where + neither exists, it is still `migrations`, and the canonical gate + refuses it by name. The compose file and the Kubernetes manifests + set the variable explicitly, so they read nothing implicit. + """ + env = os.environ if env is None else env + setting = PREFIX + "MIGRATIONS_DIR" + raw = env.get(setting, "").strip() + if raw: + return Path(raw) + if local: + found = installation_migrations_dir() + if found is None: + raise ConfigurationError( + f"{setting} is unset, and this installation carries no " + "migrations of its own: a wheel install has them under " + f"`{PACKAGED_MIGRATIONS}` in its data directory, and a " + "checkout has `migrations/` beside `src/`. A LOCAL install " + "never reads the working directory's `migrations/`, because " + "its entry point runs them as the bundled server's owner. " + f"Reinstall the package, or name the directory in {setting}") + return found + here = Path("migrations") + if here.is_dir(): + return here + return installation_migrations_dir() or here def install_mode(env: Mapping[str, str] | None = None, *, @@ -1666,26 +2012,79 @@ def refuse_what_a_local_install_cannot_be(env: Mapping[str, str]) -> None: two must not come to disagree about what a local install is. """ _refuse_hosted_only_settings(env) + _refuse_another_identity_than_the_bundles(env) refuse_a_non_loopback_local_bind( PREFIX + "BIND_HOST", _optional(env, _by_name(PREFIX + "BIND_HOST")) or "127.0.0.1") +def _refuse_another_identity_than_the_bundles(env: Mapping[str, str]) -> None: + """A LOCAL install's served role and database are the bundle's own. + + The bundled server is bootstrapped with `BUNDLE_SERVED_ROLE` as the served + identity, granted the default DML, and its served DSN connects as that + role to `BUNDLE_DATABASE`. `OPENDOX_RUNTIME_PG_ROLE` names the role the + migration run NARROWS on the ledger. So a different one, the existing + `pg_read_all_data` say, let a start succeed while `opendox_runtime` kept + INSERT, UPDATE and DELETE on `opendox_schema_migrations`, and the + ledger's protection was broken (Copilot review of openDox-code#69). + `OPENDOX_SERVED_DATABASE` is the same kind of declaration about the same + identity. Each may be set only to the bundle's own name, and anything + else is refused by name. + """ + for name, own in ((PREFIX + "RUNTIME_PG_ROLE", BUNDLE_SERVED_ROLE), + (PREFIX + "SERVED_DATABASE", BUNDLE_DATABASE)): + given = env.get(name, "").strip() + if given and given != own: + raise ConfigurationError( + f"{name} is {given!r}, and a LOCAL install's is {own!r}: the " + "bundled server is bootstrapped with that served identity, its " + "served DSN connects as it, and the migration run narrows " + "exactly the one this names on the migration ledger. Another " + "would leave the bundle's own served role able to rewrite the " + f"ledger. Unset {name}, or set it to {own!r}") + + +def _named(given: list[str]) -> str: + return f"{' and '.join(given)} {'are' if len(given) > 1 else 'is'} set" + + def _refuse_hosted_only_settings(env: Mapping[str, str]) -> None: - """Every setting in `HOSTED_ONLY_SETTINGS` given beside `local`, named.""" - given = [name for name in HOSTED_ONLY_SETTINGS - if env.get(name, "").strip()] - if given: - raise ConfigurationError( - f"{' and '.join(given)} {'are' if len(given) > 1 else 'is'} set, " - "and this is a LOCAL install, which has no broker and reads " - f"{'none of them' if len(given) > 1 else 'none'}. A broker " + """Every setting in `HOSTED_ONLY_SETTINGS` given beside `local`, named. + + Each CLASS carries its own reason (Copilot review of openDox-code#69). + A broker setting says a hosted install was meant, and a DSN says another + database was meant. A local run with only `OPENDOX_DATABASE_URL` is told + about its database, not about an authentication it never configured. + The values are never repeated: a DSN carries a password. + """ + def given(names: tuple[str, ...]) -> list[str]: + return [name for name in names if env.get(name, "").strip()] + + brokers, databases = given(BROKER_SETTINGS), given(OPERATOR_DATABASE_SETTINGS) + if not (brokers or databases): + return + reasons = [] + if brokers: + reasons.append( + f"{_named(brokers)}, and a LOCAL install has no broker and reads " + f"{'none of them' if len(brokers) > 1 else 'none'}. A broker " "setting beside the local mode says a HOSTED install was meant, " "and honouring `local` over it would silently drop that " - f"authentication: unset {'them' if len(given) > 1 else 'it'} for " - f"a local install, or drop {LOCAL_FLAG} / " - f"{PREFIX}INSTALL_MODE=local for a hosted one (the values are not " - "repeated here)") + "authentication") + if databases: + reasons.append( + f"{_named(databases)}, and a LOCAL install supplies BOTH of its " + "DSNs itself, from the PostgreSQL server it bundles under " + f"{PREFIX}STATE_DIR (13.1). An operator's DSN beside the local " + "mode would either be silently overridden or be another server " + "standing in for the bundled one, and neither is accepted") + every = brokers + databases + raise ConfigurationError( + "; and ".join(reasons) + f". Unset {'them' if len(every) > 1 else 'it'} " + f"for a local install, or drop {LOCAL_FLAG} / " + f"{PREFIX}INSTALL_MODE=local for a hosted one (the values are not " + "repeated here)") def require_the_hosted_issuer(env: Mapping[str, str] | None = None) -> None: @@ -1716,6 +2115,19 @@ def require_the_hosted_issuer(env: Mapping[str, str] | None = None) -> None: f"{PREFIX}INSTALL_MODE=local") +def _hosted_state_dir(env: Mapping[str, str]) -> Path: + """A HOSTED install's `state_dir`: reported, never read, never refused. + + A hosted install has no bundled server, so a value it will never use is + not a reason for it to refuse to start (13.6: otherwise unchanged). It is + still reported, so `status` describes the whole declared setting list. + """ + try: + return state_dir(env) + except ConfigurationError: + return Path(env.get(PREFIX + "STATE_DIR", "").strip()) + + def load_settings(env: Mapping[str, str] | None = None, *, local_flag: bool = False) -> RuntimeSettings: """Resolve :class:`RuntimeSettings` from `env` (default `os.environ`). @@ -1758,8 +2170,20 @@ def load_settings(env: Mapping[str, str] | None = None, *, if local: refuse_what_a_local_install_cannot_be(env) algorithms = _algorithms(env) - served = _require(env, _by_name(PREFIX + "DATABASE_URL")) - migration = _optional(env, _by_name(PREFIX + "MIGRATION_DATABASE_URL")) + # A LOCAL INSTALL SUPPLIES BOTH DSNs ITSELF (plan 034 T072; #1144 13.1), + # from the server it bundles under its own state directory, and an + # operator's DSN beside it was refused above. The two it supplies are two + # users over one socket, so T071's three checks below pass them for the + # reason they exist: one dialect, one database, and never one credential + # in both settings. + state = state_dir(env) if local else _hosted_state_dir(env) + if local: + bundle = database_bundle(state) + served: str = bundle.served_dsn + migration: str | None = bundle.migration_dsn + else: + served = _require(env, _by_name(PREFIX + "DATABASE_URL")) + migration = _optional(env, _by_name(PREFIX + "MIGRATION_DATABASE_URL")) # THE DIALECT FIRST: a scheme this module cannot parse as PostgreSQL is not # yet a DSN worth comparing at all. A no-op on an ABSENT migration DSN — # see `_refuse_non_postgresql_dsn`. @@ -1786,6 +2210,7 @@ def load_settings(env: Mapping[str, str] | None = None, *, database_url=served, migration_database_url=migration, install_mode=mode, + state_dir=state, oidc_issuer="" if local else _broker_url( env, _by_name(PREFIX + "OIDC_ISSUER"), required=True, is_a_base_url=True) or "", @@ -1798,11 +2223,15 @@ def load_settings(env: Mapping[str, str] | None = None, *, oidc_leeway_seconds=_positive_int(env, _by_name(PREFIX + "OIDC_LEEWAY_SECONDS")), bind_host=bind_host, bind_port=_positive_int(env, _by_name(PREFIX + "BIND_PORT")), - runtime_pg_role=_role_name(env), + # THE BUNDLE'S OWN NAMES where the operator declares none: the served + # role the migration narrows and verifies, and the database it may + # touch. An operator's declaration still wins, and a wrong one is + # refused by the migration run's own guards, as on a hosted install. + runtime_pg_role=_role_name(env) or (BUNDLE_SERVED_ROLE if local else None), served_schema=_served_schema(env), - served_database=_served_database(env), + served_database=_served_database(env) or (BUNDLE_DATABASE if local else None), publish_openapi=_boolean(env, _by_name(PREFIX + "PUBLISH_OPENAPI")), - migrations_dir=Path(_optional(env, _by_name(PREFIX + "MIGRATIONS_DIR")) or "migrations"), + migrations_dir=migrations_dir(env, local=local), project_repository_root=Path( _optional(env, _by_name(PREFIX + "PROJECT_REPOSITORY_ROOT")) or "var/projects" ), @@ -1835,13 +2264,21 @@ def load_migration_settings(env: Mapping[str, str] | None = None) -> RuntimeSett # (plan 034 T070; Copilot review of openDox-code#67). A migration run is # part of the same install as the served one, so `runtime migrate` and # `runtime reset` refuse what `load_settings` and `generate-and-open` - # refuse beside `local` — a broker setting, or a non-loopback - # `OPENDOX_BIND_HOST` — rather than accepting it in the one loader that - # never reads it. + # refuse beside `local` — a broker setting, an operator's DSN, or a + # non-loopback `OPENDOX_BIND_HOST` — rather than accepting it in the one + # loader that never reads it. mode = install_mode(env) - if mode == INSTALL_MODE_LOCAL: + local = mode == INSTALL_MODE_LOCAL + if local: + # THE BUNDLE'S OWNER, over its socket (plan 034 T072): a local install + # supplies its migration DSN as it supplies the served one, and an + # operator's beside it is refused, exactly as `load_settings` refuses. refuse_what_a_local_install_cannot_be(env) - dsn = env.get(PREFIX + "MIGRATION_DATABASE_URL", "").strip() + state = state_dir(env) + dsn = database_bundle(state).migration_dsn + else: + state = _hosted_state_dir(env) + dsn = env.get(PREFIX + "MIGRATION_DATABASE_URL", "").strip() if not dsn: raise ConfigurationError( f"{PREFIX}MIGRATION_DATABASE_URL is required to apply migrations; " @@ -1859,9 +2296,12 @@ def load_migration_settings(env: Mapping[str, str] | None = None) -> RuntimeSett database_url=dsn, migration_database_url=dsn, # READ ABOVE, SO AN UNRECOGNISED VALUE IS REFUSED HERE TOO (plan 034 - # T070). It changes nothing else a migration run does; the broker - # fields below are sentinels in either shape. + # T070): a migration run is part of the same install and one reading + # of the selector serves every verb. It changes nothing else a + # migration run does; the broker fields below are sentinels in either + # shape. install_mode=mode, + state_dir=state, oidc_issuer=MIGRATION_SENTINEL_ISSUER, oidc_audience=MIGRATION_SENTINEL_AUDIENCE, oidc_jwks_url=None, @@ -1870,12 +2310,11 @@ def load_migration_settings(env: Mapping[str, str] | None = None) -> RuntimeSett oidc_leeway_seconds=1, bind_host="127.0.0.1", bind_port=1, - runtime_pg_role=_role_name(env), + runtime_pg_role=_role_name(env) or (BUNDLE_SERVED_ROLE if local else None), served_schema=_served_schema(env), - served_database=_served_database(env), + served_database=_served_database(env) or (BUNDLE_DATABASE if local else None), publish_openapi=False, - migrations_dir=Path( - _optional(env, _by_name(PREFIX + "MIGRATIONS_DIR")) or "migrations"), + migrations_dir=migrations_dir(env, local=local), project_repository_root=Path( _optional(env, _by_name(PREFIX + "PROJECT_REPOSITORY_ROOT")) or "var/projects"), diff --git a/tests/standalone_child.py b/tests/standalone_child.py index 34bbda01..e9e23534 100644 --- a/tests/standalone_child.py +++ b/tests/standalone_child.py @@ -25,7 +25,14 @@ environment, so an `OPENDOX_INSTALL_MODE=hosted` or a broker issuer the runner happens to export cannot make a `generate-and-open --local` child refuse before the case it exists for (plan 034 T070; Copilot review of - openDox-code#67). + openDox-code#67). The one setting given back is the next one. +* ITS STATE DIRECTORY IS ITS OWN. A `generate-and-open --local` child starts + the local install's bundled PostgreSQL server (plan 034 T072) under + `OPENDOX_STATE_DIR`, whose default is the USER's own state directory. So + every child is given a fresh, short, private one (`Child.state_dir`, under + `/tmp` because a Unix socket's whole path is bounded), and it is removed + once the child is stopped. No case ever initializes a database in the home + directory of whoever runs the suite. * CTRL-C REACHES IT AS IT WOULD AT A TERMINAL, whatever the runner's own disposition. The same `sitecustomize` sets SIGINT back to Python's KeyboardInterrupt handler. A runner started as a background job @@ -51,6 +58,7 @@ import signal import subprocess import sys +import tempfile import threading import time from pathlib import Path @@ -154,6 +162,9 @@ def __init__(self, workdir: Path, module: str, *args: str) -> None: env["PYTHONPATH"] = os.pathsep.join( [str(blocker), *filter(None, [env.get("PYTHONPATH")])]) env[REFUSED_LOG_ENV] = str(self.refused_log) + self.state_dir = Path(tempfile.mkdtemp( + prefix="odx-child-", dir="/tmp" if os.path.isdir("/tmp") else None)) + env["OPENDOX_STATE_DIR"] = str(self.state_dir) self.argv = [sys.executable, "-m", module, *args] verb = args[0] if args and not args[0].startswith("-") else "" self.label = f"python -m {module} {verb}".strip() @@ -230,6 +241,7 @@ def kill(self) -> None: if self.process.poll() is None: self.process.kill() self.process.wait(timeout=STOP_DEADLINE_SECONDS) + shutil.rmtree(self.state_dir, ignore_errors=True) def _join(self) -> None: for pump in self._pumps: diff --git a/tests/test_doxbench_entrypoint.py b/tests/test_doxbench_entrypoint.py index cd0b8dca..e41659f2 100644 --- a/tests/test_doxbench_entrypoint.py +++ b/tests/test_doxbench_entrypoint.py @@ -51,6 +51,7 @@ import os import shutil import subprocess +import tempfile from pathlib import Path import pytest @@ -146,6 +147,39 @@ def _capture(*args, **kwargs): # from the shell would be refused beside the local mode, by design. for name in runtime_config.SETTING_NAMES: monkeypatch.delenv(name, raising=False) + # ITS OWN SHORT STATE DIRECTORY, so the configuration never reads the + # runner's: a local install's default is under `XDG_STATE_HOME` or the + # home directory, and one too long for a Unix socket is refused (13.1) + # before the entrypoint is reached. Nothing is made in it, because the + # database is stood in below, and it is removed after the case. + state = Path(tempfile.mkdtemp(prefix="odx-e-", + dir="/tmp" if os.path.isdir("/tmp") else None)) + monkeypatch.setenv(runtime_config.PREFIX + "STATE_DIR", str(state)) + # AND THE LOCAL INSTALL'S DATABASE IS STOOD IN, with a tripwire of its own + # (plan 034 T072). A local `generate-and-open` starts its bundled + # PostgreSQL server before it serves, and these cases are about the model + # port the entrypoint declares, which reads nothing from the store (R1Q16 + # (ii)). `tests_runtime/test_bundled_postgres.py` starts the real one, on + # this same entry point, and owns every assertion about it. + bundles = [] + + class _StandInBundle: + applied: list = [] + + def __init__(self, settings): + assert settings.install_mode == runtime_config.INSTALL_MODE_LOCAL + bundles.append(self) + + def start(self): + return self + + def stop(self): + pass + + def report(self): + return {"data_dir": None, "socket_dir": "(stood in)", "pid": None} + + monkeypatch.setattr(cli_mod.bundle_mod, "BundledServer", _StandInBundle) args = cli_mod.build_parser().parse_args([ "generate-and-open", runtime_config.LOCAL_FLAG, @@ -157,10 +191,14 @@ def _capture(*args, **kwargs): "--model-session-root", str(session_root), "--no-validate", "--no-open", "--no-serve", ]) - rc = cli_mod.cmd_generate_and_open(args, opener=lambda url: None) - assert rc == 0, "the entrypoint did not complete" - assert built, "the entrypoint never reached build_server" - yield _handler_class(built[-1]), spawned, session_root + try: + rc = cli_mod.cmd_generate_and_open(args, opener=lambda url: None) + assert rc == 0, "the entrypoint did not complete" + assert len(bundles) == 1, "a LOCAL entrypoint run must own one database" + assert built, "the entrypoint never reached build_server" + yield _handler_class(built[-1]), spawned, session_root + finally: + shutil.rmtree(state, ignore_errors=True) # -------------------------------------------------------------------------- diff --git a/tests/test_install_mode_entrypoint.py b/tests/test_install_mode_entrypoint.py index 9c4c6203..63356fdb 100644 --- a/tests/test_install_mode_entrypoint.py +++ b/tests/test_install_mode_entrypoint.py @@ -172,31 +172,33 @@ def _args(*extra: str): def test_the_local_shape_needs_no_broker_and_no_setting_at_all() -> None: - """13.4: `local` needs no broker. Resolved with an EMPTY environment.""" - assert cli_mod._resolve_install_shape( - _args(runtime_config.LOCAL_FLAG), env={}) == \ - runtime_config.INSTALL_MODE_LOCAL - assert cli_mod._resolve_install_shape( - _args(), env={MODE: "local"}) == runtime_config.INSTALL_MODE_LOCAL + """13.4: `local` needs no broker. Resolved with an EMPTY environment, and + (plan 034 T072) with both DSNs supplied by the install itself.""" + for args, env in ((_args(runtime_config.LOCAL_FLAG), {}), + (_args(), {MODE: "local"})): + settings = cli_mod._resolve_install_shape(args, env=env) + assert settings.install_mode == runtime_config.INSTALL_MODE_LOCAL + assert settings.oidc_issuer == "" + assert settings.database_url != settings.migration_database_url @pytest.mark.parametrize("host", sorted(serve_mod.LOOPBACK_HOSTS)) def test_the_local_shape_accepts_each_loopback_host(host: str) -> None: assert cli_mod._resolve_install_shape( - _args(runtime_config.LOCAL_FLAG, "--host", host), env={}) == \ - runtime_config.INSTALL_MODE_LOCAL + _args(runtime_config.LOCAL_FLAG, "--host", host), env={} + ).install_mode == runtime_config.INSTALL_MODE_LOCAL def test_a_complete_hosted_configuration_resolves_hosted_unchanged() -> None: """13.6: a hosted install with its broker configured serves as before.""" env = {**HOSTED_WITHOUT_ISSUER, PREFIX + "OIDC_ISSUER": "https://issuer.example.invalid/realms/x"} - assert cli_mod._resolve_install_shape(_args(), env=env) == \ + assert cli_mod._resolve_install_shape(_args(), env=env).install_mode == \ runtime_config.INSTALL_MODE_HOSTED # a hosted document server may still bind beyond loopback, as it always # could: the loopback rule is the LOCAL mode's assert cli_mod._resolve_install_shape( - _args("--host", "0.0.0.0"), env=env) == \ + _args("--host", "0.0.0.0"), env=env).install_mode == \ runtime_config.INSTALL_MODE_HOSTED diff --git a/tests/test_projection_seams.py b/tests/test_projection_seams.py index 02c5ec65..01bad88b 100644 --- a/tests/test_projection_seams.py +++ b/tests/test_projection_seams.py @@ -46,6 +46,7 @@ import stat import subprocess import sys +import tempfile import textwrap import threading import types @@ -1397,24 +1398,52 @@ def test_generate_and_open_refuses_an_empty_source_option_before_its_run_dir( first, as the doxBench entrypoint fixture does: an exported `OPENDOX_INSTALL_MODE=hosted` or broker issuer would otherwise make `--local` refuse before the empty option is reached (Copilot review of - openDox-code#67).""" - from opendox.runtime.config import SETTING_NAMES + openDox-code#67). And the refusal comes before the local install's + bundled server (plan 034 T072): a refused option costs no database + start. A tripwire stands in for the server, so a regression neither + starts one nor passes.""" + from opendox.runtime.config import PREFIX, SETTING_NAMES for name in SETTING_NAMES: monkeypatch.delenv(name, raising=False) + # Its own short state directory, so the configuration never reads the + # runner's state home, whose default may be too long for a Unix socket + # (13.1) and refused before the option is reached. The tripwire below + # means nothing is made in it. + state = Path(tempfile.mkdtemp(prefix="odx-e-", + dir="/tmp" if os.path.isdir("/tmp") else None)) + monkeypatch.setenv(PREFIX + "STATE_DIR", str(state)) + started: list = [] + + class _Tripwire: + def __init__(self, settings) -> None: + started.append(settings) + + def start(self): + raise AssertionError("the bundled server was started for a " + "refused source option") + + def stop(self) -> None: + pass + + monkeypatch.setattr(cli.bundle_mod, "BundledServer", _Tripwire) calls: list = [] _declaring_generator(calls) repo = _repository(tmp_path) run_dir = tmp_path / "run" # `--local`: the single-user install. Since plan 034 T070 an unflagged run # is HOSTED, and its issuer refusal would come first. - rc = cli.main(["generate-and-open", "--local", "--repo-root", str(repo), - "--repository", "garden", "--run-dir", str(run_dir), - "--no-open", "--no-serve", "--possibles", ""]) + try: + rc = cli.main(["generate-and-open", "--local", "--repo-root", str(repo), + "--repository", "garden", "--run-dir", str(run_dir), + "--no-open", "--no-serve", "--possibles", ""]) + finally: + shutil.rmtree(state, ignore_errors=True) assert rc == 1 assert ("generate-and-open refused: --possibles was given an empty path" in capsys.readouterr().err) assert calls == [] and not run_dir.exists() + assert started == [], "a bundled server was built for a refused option" def test_a_root_openDoxs_predicate_refuses_is_refused_with_its_message(tmp_path, capsys) -> None: diff --git a/tests/test_standalone_generate_path.py b/tests/test_standalone_generate_path.py index a199de51..567484b7 100644 --- a/tests/test_standalone_generate_path.py +++ b/tests/test_standalone_generate_path.py @@ -285,11 +285,17 @@ def test_generate_and_open_starts_a_server_that_answers_with_no_sibling(tmp_path child.wait_for_line(_SERVING) assert child.process.poll() is None, "the server exited after saying it serves" _assert_the_server_answers(base, run_dir / "snapshot.json", repo) + # THE LOCAL INSTALL'S DATABASE IS THE CHILD'S OWN (plan 034 T072): its + # bundled server was started under the private state directory the + # harness gave this child, never under the user's. + assert (child.state_dir / "postgres" / "data" / "PG_VERSION").is_file(), \ + "the bundled server was not started under the child's state dir" assert child.interrupt() == 0, child.stderr_text() finally: child.kill() _assert_the_port_is_closed(base) assert child.refused() == [], child.refused() + assert not child.state_dir.exists(), "the child's state dir outlived it" # --------------------------------------------------------------------------- @@ -327,27 +333,35 @@ def test_a_child_inherits_none_of_the_runners_runtime_settings( """The runner exports a HOSTED install's settings, and the child sees none of them (plan 034 T070; Copilot review of openDox-code#67). The `--local` cases above would otherwise refuse before they reach what they - test, for a reason that is the runner's configuration and not theirs.""" + test, for a reason that is the runner's configuration and not theirs. + The one runtime setting a child does see is the state directory the + harness gives it (plan 034 T072), never the runner's own.""" from opendox.runtime.config import PREFIX, SETTING_NAMES + state_setting = PREFIX + "STATE_DIR" exported = {PREFIX + "INSTALL_MODE": "hosted", PREFIX + "OIDC_ISSUER": "https://issuer.example.invalid/realms/x", PREFIX + "OIDC_AUDIENCE": "fixture", - PREFIX + "DATABASE_URL": "postgresql://s@127.0.0.1:1/x"} + PREFIX + "DATABASE_URL": "postgresql://s@127.0.0.1:1/x", + state_setting: str(tmp_path / "runners-own-state")} for name, value in exported.items(): monkeypatch.setenv(name, value) blocker = tmp_path / "sibling-blocker" blocker.mkdir() - (blocker / "t070_env_probe.py").write_text(textwrap.dedent(""" + (blocker / "t070_env_probe.py").write_text(textwrap.dedent(f""" import json, os - print(json.dumps(sorted(n for n in os.environ if n.startswith("OPENDOX_"))), - flush=True) + print(json.dumps([sorted(n for n in os.environ if n.startswith("OPENDOX_")), + os.environ.get({state_setting!r})]), flush=True) """), encoding="utf-8") - child, status = run_module(tmp_path, "t070_env_probe") + child = Child(tmp_path, "t070_env_probe") + state_dir = child.state_dir + status = child.wait() assert status == 0, child.stderr_text() - seen = set(json.loads(child.stdout_text().strip().splitlines()[-1])) - assert not seen & set(SETTING_NAMES), sorted(seen & set(SETTING_NAMES)) + names, state_value = json.loads(child.stdout_text().strip().splitlines()[-1]) + assert set(names) & set(SETTING_NAMES) == {state_setting}, names + assert state_value == str(state_dir) != exported[state_setting] assert set(exported) <= set(SETTING_NAMES) + assert not state_dir.exists(), "the child's state directory outlived it" def test_a_child_that_ignores_the_interrupt_is_killed_at_the_deadline( diff --git a/tests_runtime/test_bundled_postgres.py b/tests_runtime/test_bundled_postgres.py new file mode 100644 index 00000000..3157c37d --- /dev/null +++ b/tests_runtime/test_bundled_postgres.py @@ -0,0 +1,835 @@ +"""The LOCAL install's bundled PostgreSQL server (plan 034 T072; #1144 13.1, as +T007 batch H's addendum reads; RULED R1Q16 (i)-(iv), `5850003126`). + +T072's falsifier is F13.1's TCP-listener block, which reads the kernel's +socket table at run time, and its `runtime status` block. Both are run here +against a server the REAL entry point started: `python -m opendox.cli +generate-and-open --local` over a fresh repository copied from T050's +`tests/fixtures/plain-documents`, launched in the background as F13.1 launches +it, reached over HTTP, asked about by a second process, and stopped with a +signal. Nothing is stood in: the corpus root check, the generation, the +validator and the serve loop are phase 2's own, landed on this stack's base +(T054 to T058), so the stand-in driver this module once launched is gone. Where F13.1 reads the +server's pid from `caps.json`, these cases read the same pid from +`runtime status`'s `database_bundle`. `/capabilities`' `install` block is +T073's, and nothing here pretends it exists. + +R1Q16, each part asserted: + (i) the server is a CHILD of the entry point's process (its `PPid`); + (ii) it is started AND migrated, the ledger holding every migration; + (iii) the `local` extra carries it, and the `test` extra joins the extra; + (iv) it stops with the entry point: on SIGTERM, which the serve loop reads + as Ctrl-C, and — the backstop — on SIGKILL, through the parent-death + signal. + +And the migrations gap the holder assigned to T072: a WHEEL install, run from +a directory that is not a checkout, migrates its bundled server from the copy +the wheel carries. + +NOT SKIPPED IN CI. These cases need the `local` extra's server, which the +`test` extra installs; under `CI` its absence is a FAILURE, as a missing +`postgres:16` service is for the DB-backed suites (`conftest._skip_or_fail`'s +rule, restated below), because `validate.yml` pins the skip count exactly. +""" + +from __future__ import annotations + +import json +import os +import re +import selectors +import shutil +import signal +import stat +import subprocess +import sys +import sysconfig +import tempfile +import time +import tomllib +import urllib.request +from pathlib import Path + +import pytest + +from opendox.runtime import bundle as bundle_mod +from opendox.runtime import cli +from opendox.runtime import config +from opendox.runtime.config import PREFIX + +ROOT = Path(__file__).resolve().parents[1] +SRC = ROOT / "src" +#: T050's fixture, which F13.1's preamble copies into a fresh repository. +PLAIN_DOCUMENTS = ROOT / "tests" / "fixtures" / "plain-documents" +MODE = PREFIX + "INSTALL_MODE" +STATE = PREFIX + "STATE_DIR" + + +def _in_ci() -> bool: + return os.environ.get("CI", "").strip().lower() in {"1", "true", "yes", "on"} + + +@pytest.fixture(scope="module", autouse=True) +def _the_server_is_installed() -> None: + """The `local` extra's binaries, or this module's refusal to pass silently.""" + try: + bundle_mod.server_binaries() + except bundle_mod.BundleRefused as exc: + if _in_ci(): + pytest.fail(f"CI is set, so the bundled-server suite must RUN: {exc}", + pytrace=False) + pytest.skip(str(exc)) + if hasattr(os, "geteuid") and os.geteuid() == 0: # pragma: no cover + pytest.fail("the bundled server refuses root; run the suite as a user") + + +@pytest.fixture() +def state_dir(): + """A state directory nothing else has touched, with a SHORT path. + + Short because the socket's whole path is bounded by the kernel, and + pytest's own `tmp_path` grows with the test's name. Removed afterwards, + once any server on it has been checked stopped. + """ + base = "/tmp" if os.path.isdir("/tmp") else None + path = Path(tempfile.mkdtemp(prefix="odx-", dir=base)) + yield path + pid = bundle_mod.running_pid(config.DatabaseBundle(path)) + if pid is not None: # pragma: no cover + os.kill(pid, signal.SIGKILL) + shutil.rmtree(path, ignore_errors=True) + + +def _clean_env(**extra: str) -> dict[str, str]: + env = {name: value for name, value in os.environ.items() + if name not in config.SETTING_NAMES and not name.startswith("PG")} + env["PYTHONPATH"] = os.pathsep.join( + [str(SRC), env.get("PYTHONPATH", "")]).rstrip(os.pathsep) + env.update(extra) + return env + + +def _parent_of(pid: int) -> int: + for line in Path(f"/proc/{pid}/status").read_text().splitlines(): + if line.startswith("PPid:"): + return int(line.split()[1]) + raise AssertionError(f"no PPid for {pid}") # pragma: no cover + + +def _tcp_listeners(pid: int) -> list[tuple[str, str]]: + """F13.1's TCP-listener block, verbatim in substance: every socket the + process holds, looked up in the KERNEL's TCP tables, never self-report.""" + inodes = set() + for fd in os.listdir(f"/proc/{pid}/fd"): + try: + m = re.match(r"socket:\[(\d+)\]", os.readlink(f"/proc/{pid}/fd/{fd}")) + except OSError: + continue + if m: + inodes.add(m.group(1)) + return [(tbl, row.split()[1]) for tbl in ("/proc/net/tcp", "/proc/net/tcp6") + for row in open(tbl).read().splitlines()[1:] + if row.split()[3] == "0A" and row.split()[9] in inodes] + + +def _wait_gone(pid: int, seconds: float = 30.0) -> bool: + deadline = time.monotonic() + seconds + while time.monotonic() < deadline: + try: + os.kill(pid, 0) + except ProcessLookupError: + return True + # a zombie is gone as a server: reaped by init once its parent is + try: + if "State:\tZ" in Path(f"/proc/{pid}/status").read_text(): + return True + except OSError: + return True + time.sleep(0.1) + return False + + +def _first_url(child: subprocess.Popen, seconds: float) -> str | None: + """The first `http://` line `child` prints within `seconds`, or `None`. + + BOUNDED BY THE DEADLINE, not by the child: a blocking `readline()` waits + for as long as a child that is alive and silent stays so, which is exactly + the startup failure this has to diagnose (Copilot review of + openDox-code#69). So the pipe is polled with a selector, only up to the + time left, and read in whatever pieces arrive. + """ + deadline = time.monotonic() + seconds + pending = b"" + with selectors.DefaultSelector() as selector: + selector.register(child.stdout, selectors.EVENT_READ) + while (remaining := deadline - time.monotonic()) > 0: + if not selector.select(timeout=remaining): + return None # the deadline + chunk = os.read(child.stdout.fileno(), 65536) + if not chunk: + return None # the child closed it + pending += chunk + *lines, pending = pending.split(b"\n") + for line in lines: + if line.startswith(b"http://"): + return line.strip().decode() + return None + + +def _launch(corpus: Path, state: Path, run_dir: Path, + **extra: str) -> tuple[subprocess.Popen, str]: + """`generate-and-open --local` in the BACKGROUND, and the URL it serves.""" + child = subprocess.Popen( + [sys.executable, "-m", "opendox.cli", "generate-and-open", + config.LOCAL_FLAG, "--repo-root", str(corpus), "--repository", + "fixture", "--run-dir", str(run_dir), "--no-open", "--port", "0"], + env=_clean_env(**{STATE: str(state)}, **extra), cwd=ROOT, + stdout=subprocess.PIPE, stderr=subprocess.PIPE) + url = _first_url(child, 90) + if url is None: + child.kill() + _out, err = child.communicate(timeout=30) + raise AssertionError( + f"the entry point never served: {err.decode(errors='replace')[-2000:]}") + return child, url + + +def test_the_launch_helper_is_bounded_by_its_deadline_not_by_the_child() -> None: + """A child that is alive and silent does not hold the helper past its + deadline; one that prints the URL is read, however the pieces arrive.""" + silent = subprocess.Popen([sys.executable, "-c", "import time; time.sleep(60)"], + stdout=subprocess.PIPE) + try: + began = time.monotonic() + assert _first_url(silent, 1.0) is None + assert time.monotonic() - began < 10 + finally: + silent.kill() + silent.communicate(timeout=10) + talker = subprocess.Popen( + [sys.executable, "-c", "import sys, time; sys.stdout.write(' serving '); " + "sys.stdout.flush(); time.sleep(0.2); print('x'); " + "print('http://127.0.0.1:1/index.html', flush=True); time.sleep(60)"], + stdout=subprocess.PIPE) + try: + assert _first_url(talker, 30) == "http://127.0.0.1:1/index.html" + finally: + talker.kill() + talker.communicate(timeout=10) + + +def _status(state: Path, **extra: str) -> tuple[int, dict]: + """`OPENDOX_INSTALL_MODE=local opendox-runtime runtime status`, a SECOND process.""" + done = subprocess.run( + [sys.executable, "-m", "opendox.runtime.cli", "runtime", "status", + "--probe-timeout", "10"], + env=_clean_env(**{MODE: "local", STATE: str(state)}, **extra), cwd=ROOT, + capture_output=True, text=True, timeout=60) + return done.returncode, json.loads(done.stdout) + + +@pytest.fixture() +def corpus(tmp_path: Path) -> Path: + """F13.1's preamble: T050's `plain-documents` copied into a FRESH git + repository, committed under the fixture's own identity and none of the + user's git configuration.""" + root = tmp_path / PLAIN_DOCUMENTS.name + shutil.copytree(PLAIN_DOCUMENTS, root) + env = {name: value for name, value in os.environ.items() + if not name.startswith("GIT_")} + env.update({"GIT_AUTHOR_NAME": "fixture", + "GIT_AUTHOR_EMAIL": "fixture@example.invalid", + "GIT_COMMITTER_NAME": "fixture", + "GIT_COMMITTER_EMAIL": "fixture@example.invalid", + "GIT_CONFIG_GLOBAL": os.devnull, "GIT_CONFIG_SYSTEM": os.devnull}) + for argv in (["git", "-c", "init.defaultBranch=main", "init", "-q"], + ["git", "add", "-A"], ["git", "commit", "-qm", "fixture"]): + subprocess.run(argv, cwd=root, env=env, check=True, capture_output=True) + return root + + +# -- (iii): the packaging ---------------------------------------------------- + + +def test_the_local_extra_carries_the_runtime_and_the_server_and_test_joins_it( +) -> None: + project = tomllib.loads((ROOT / "pyproject.toml").read_text()) + extras = project["project"]["optional-dependencies"] + assert "opendox[runtime]" in extras["local"] + # the carrier RULED on openxFactory#656 `5916000030` item 2 + assert any(req.startswith("pixeltable-pgserver") for req in extras["local"]) + assert not any(req.startswith("pgserver") for req in extras["local"]) + # WITH A CEILING (adversarial review of #69): the carrier's `pginstall/` + # major is a property of the user's data, so no later release moves it + # without this package's say. The lock's pin sits inside the range. + assert "pixeltable-pgserver>=0.6.0,<0.7" in extras["local"], extras["local"] + assert "opendox[local]" in extras["test"], ( + "F9.1 installs `.[test]` alone; without the local extra there, this " + "suite could not start the server it tests") + lock = (ROOT / "constraints-cpython312-linux.txt").read_text() + assert re.search(r"(?m)^pixeltable-pgserver==0\.6\.\d+$", lock), \ + "the lock does not pin pixeltable-pgserver inside >=0.6.0,<0.7" + assert not re.search(r"(?m)^pgserver==", lock), "the lock still pins pgserver" + files = project["tool"]["setuptools"]["data-files"] + assert files == {"share/opendox/migrations": ["migrations/*.sql"]} + # and the packaging notes name functions that exist (Copilot review of #69) + for name in re.findall(r"opendox\.runtime\.config\.(\w+)", + (ROOT / "pyproject.toml").read_text()): + assert hasattr(config, name), ( + f"pyproject.toml points readers at config.{name}, which does not exist") + + +# -- the layout, before anything starts ---------------------------------------- + + +def test_the_two_dsns_are_two_users_over_the_one_socket(state_dir: Path) -> None: + settings = config.load_settings({MODE: "local", STATE: str(state_dir)}) + bundle = config.database_bundle(state_dir) + assert settings.database_url == bundle.served_dsn + assert settings.migration_database_url == bundle.migration_dsn + assert settings.database_url != settings.migration_database_url # 13.3 + for dsn in (settings.database_url, settings.migration_database_url): + assert dsn.startswith("postgresql://") # 13.2 + assert f"host={bundle.socket_dir}" in dsn and "port=5432" in dsn + assert config.user_named_by(settings.database_url) == config.BUNDLE_SERVED_ROLE + assert config.user_named_by(settings.migration_database_url) == \ + config.BUNDLE_OWNER_ROLE + assert bundle.data_dir.parent == bundle.socket_dir.parent + assert bundle.data_dir.is_relative_to(state_dir) + assert bundle.socket_dir.is_relative_to(state_dir) + + +def test_a_state_dir_too_long_for_a_unix_socket_is_refused_naming_it() -> None: + long = "/tmp/" + "x" * 120 + with pytest.raises(config.ConfigurationError) as caught: + config.load_settings({MODE: "local", STATE: long}) + assert STATE in str(caught.value) and "socket" in str(caught.value) + + +@pytest.mark.parametrize("variable", [STATE, "XDG_STATE_HOME", "HOME"]) +def test_a_comma_in_the_state_dir_is_refused_without_repeating_it( + monkeypatch, variable: str) -> None: + """PostgreSQL splits its socket directories, and libpq its hosts, on a + comma, so one would split the socket's one checked directory into two + unchecked ones (adversarial review of #69). Refused at configuration, + from whichever setting it came, and the value is not repeated.""" + value = "/tmp/odx-c1,/tmp/odx-c2-secretish" + env = {MODE: "local"} + if variable == "HOME": + monkeypatch.setenv("HOME", value) # `Path.home()` reads the process's + else: + env[variable] = value + with pytest.raises(config.ConfigurationError) as caught: + config.load_settings(env) + message = str(caught.value) + assert "`,`" in message and STATE in message, message + assert "secretish" not in message and value not in message, message + with pytest.raises(config.ConfigurationError): + config.load_migration_settings(env) + + +def test_a_relative_state_dir_is_refused_naming_it() -> None: + with pytest.raises(config.ConfigurationError) as caught: + config.load_settings({MODE: "local", STATE: "var/state"}) + assert STATE in str(caught.value) + + +def test_the_default_state_dir_is_the_users_own(monkeypatch) -> None: + home = config.state_dir({"HOME": "/ignored"}) + assert home.name == "opendox" and home.is_absolute() + assert config.state_dir({"XDG_STATE_HOME": "/srv/state"}) == \ + Path("/srv/state/opendox") + # the XDG rule: a relative value is ignored, never joined onto the cwd + assert config.state_dir({"XDG_STATE_HOME": "relative"}) == \ + Path.home() / ".local" / "state" / "opendox" + + +# -- 13.1 and R1Q16 (i), (ii), (iv): F13.1's two blocks, on the real entry point + + +# LINUX'S `/proc`, FOR THE KERNEL'S OWN ANSWERS (Copilot review of #69): the +# server's parent, its TCP listeners and `status`'s pid are read from it, so +# a POSIX platform without it skips this case rather than failing it. +# F13.1 names Linux's socket table; the lifecycle it shares with every POSIX +# platform is held by the cases around it. +@pytest.mark.skipif(not Path("/proc/self").exists(), reason="asks Linux's /proc") +def test_the_entry_point_owns_a_migrated_server_with_no_tcp_listener( + corpus: Path, state_dir: Path, tmp_path: Path) -> None: + """F13.1's `runtime status` block and its TCP-listener block, against the + server `generate-and-open --local` started in the background; then + `kill "$SERVER"; wait`, and the server is gone with it.""" + server, url = _launch(corpus, state_dir, tmp_path / "run") + try: + with urllib.request.urlopen(url, timeout=10) as answer: # ready + assert answer.status == 200 + # -- F13.1's `runtime status` block -------------------------------- + code, status = _status(state_dir) + assert status.get("database") == "reachable", \ + f"no bundled database answered: {status}" + assert status.get("applied_migrations") and \ + not status.get("pending_migrations"), f"not migrated: {status}" + state = os.path.realpath(state_dir) + bundle = status.get("database_bundle") or {} + for key in ("data_dir", "socket_dir"): + got = os.path.realpath(bundle.get(key, "")) + assert got.startswith(state + os.sep), \ + f"{key} {got!r} is not under the install's state dir {state!r}" + # a local install has no broker, and that is not a fault (T070) + assert status["broker_keys"] == "not configured (local mode)" + assert code == 0 and status["ok"] is True, status + # -- F13.1's TCP-listener block, the pid from `database_bundle` ----- + pid = bundle.get("pid") + assert isinstance(pid, int), f"the bundle reports no server pid: {pid!r}" + assert not _tcp_listeners(pid), \ + f"the bundled server listens on TCP: {_tcp_listeners(pid)}" + # -- R1Q16 (i): the server is the document server's own child ------ + assert _parent_of(pid) == server.pid, ( + "the bundled server is not a child of the process serving the " + "document surface") + mode = os.stat(bundle["socket_dir"]).st_mode & 0o777 + assert mode == 0o700, f"the socket directory is {mode:o}" + finally: + server.send_signal(signal.SIGTERM) # `kill "$SERVER"` + server.communicate(timeout=60) # `wait "$SERVER"` + # -- R1Q16 (iv): it stopped with the entry point, and cleanly ---------- + assert server.returncode == 0, server.returncode + assert _wait_gone(pid), "the bundled server outlived its entry point" + assert config.DatabaseBundle(state_dir).data_dir.joinpath("PG_VERSION").is_file(), \ + "the data directory must survive a stop: it is the install's database" + + +#: libpq defaults that, READ, would move the bundle's connections: to an +#: unroutable TCP address (TEST-NET-1, RFC 5737), through a service that does +#: not exist, and into a schema that does not either. +HOSTILE_LIBPQ = {"PGHOSTADDR": "192.0.2.1", "PGSERVICE": "no-such-service-odx", + "PGOPTIONS": "-c search_path=nowhere"} + + +def test_libpq_defaults_in_the_environment_never_reach_the_bundle( + corpus: Path, state_dir: Path, tmp_path: Path) -> None: + """`PGHOSTADDR` outranks a DSN's socket `host`, `PGSERVICE` fills + parameters from a service file, and `PGOPTIONS` sets the session's + parameters. With all three set, the entry point still starts, migrates + and serves ITS OWN server, and `runtime status` in a second process still + finds it (Copilot review of openDox-code#69).""" + server, url = _launch(corpus, state_dir, tmp_path / "run", **HOSTILE_LIBPQ) + try: + code, status = _status(state_dir, **HOSTILE_LIBPQ) + assert status.get("database") == "reachable", status + assert status.get("applied_migrations") and \ + not status.get("pending_migrations"), status + assert code == 0 and status["ok"] is True, status + finally: + server.send_signal(signal.SIGTERM) + server.communicate(timeout=60) + assert server.returncode == 0, server.returncode + + +@pytest.mark.skipif(not sys.platform.startswith("linux"), + reason="PR_SET_PDEATHSIG is Linux's") +def test_the_server_stops_even_when_the_entry_point_is_killed_outright( + corpus: Path, state_dir: Path, tmp_path: Path) -> None: + """(iv)'s backstop: SIGKILL gives the entry point no chance to stop + anything, and the parent-death signal stops the server all the same.""" + server, _url = _launch(corpus, state_dir, tmp_path / "run") + pid = bundle_mod.running_pid(config.DatabaseBundle(state_dir)) + assert isinstance(pid, int) + server.kill() + server.communicate(timeout=30) + assert _wait_gone(pid), "the bundled server outlived a SIGKILLed entry point" + + +def test_a_second_entry_point_on_the_same_state_dir_is_refused( + state_dir: Path) -> None: + settings = config.load_settings({MODE: "local", STATE: str(state_dir)}) + first = bundle_mod.BundledServer(settings).start() + try: + with pytest.raises(bundle_mod.BundleRefused) as caught: + bundle_mod.BundledServer(settings).start() + assert str(first.report()["pid"]) in str(caught.value) + # and a restart of the one that owns it re-migrates nothing + assert first.applied == ["0001", "0002"] + finally: + first.stop() + again = bundle_mod.BundledServer(settings).start() + try: + assert again.applied == [] + finally: + again.stop() + + +@pytest.mark.skipif(not Path("/proc/self").exists(), reason="asks /proc") +def test_a_stale_lock_naming_a_recycled_pid_does_not_hold_the_bundle( + state_dir: Path) -> None: + """A server that did not stop cleanly leaves its `postmaster.pid`, and the + kernel gives its pid to something else, here a process with `postgres` in + its argv. `status` does not report that process, and the next start + starts, with the proven-stale lock removed (Copilot review of + openDox-code#69).""" + settings = config.load_settings({MODE: "local", STATE: str(state_dir)}) + bundle_mod.BundledServer(settings).start().stop() # a real cluster + bundle = config.DatabaseBundle(state_dir) + decoy = subprocess.Popen( + [sys.executable, "-c", "import time; time.sleep(60)", "postgres"], + cwd=bundle.data_dir, stdout=subprocess.DEVNULL) + try: + (bundle.data_dir / "postmaster.pid").write_text( + f"{decoy.pid}\n{bundle.data_dir}\n", encoding="utf-8") + assert bundle_mod.report(bundle)["pid"] is None + server = bundle_mod.BundledServer(settings).start() + try: + assert bundle_mod.running_pid(bundle) == server.report()["pid"] + finally: + server.stop() + finally: + decoy.kill() + decoy.wait(timeout=10) + + +def test_readiness_is_this_childs_server_not_a_winners_socket( + state_dir: Path) -> None: + """Two entry points racing from an idle state both launch. The loser's + `postgres` lives a moment before it refuses the winner's lock, while the + winner's socket already answers. The loser must wait for ITS child, and + be refused when that child exits, not connect to the winner and carry on + as if it owned a database (Copilot review of openDox-code#69). The + loser's child is stood in by a process that lives three seconds.""" + settings = config.load_settings({MODE: "local", STATE: str(state_dir)}) + with bundle_mod.BundledServer(settings): + loser = bundle_mod.BundledServer(settings) + loser.process = subprocess.Popen( + [sys.executable, "-c", "import time; time.sleep(3)"]) + began = time.monotonic() + try: + with pytest.raises(bundle_mod.BundleRefused) as caught: + loser._wait_until_ready() + finally: + loser.process.kill() + loser.process.wait(timeout=10) + assert "exited during start" in str(caught.value), caught.value + assert time.monotonic() - began < bundle_mod.START_TIMEOUT_SECONDS + + +# -- peer authentication (RULED openxFactory#656 `5916000030` item 3) ---------- + + +def _owner(server) -> "object": + import psycopg + + return psycopg.connect(server.bundle.migration_dsn, autocommit=True) + + +def test_the_bundle_authenticates_by_peer_through_the_one_map( + state_dir: Path) -> None: + """The server's OWN reading of its two files, from `pg_hba_file_rules` + and `pg_ident_file_mappings`, and the method each connection really + used, from `system_user`. There is one local rule, peer through the + `opendox` map. Host is rejected. There is no `trust` anywhere. The map + admits this OS user as the two roles and names no other OS user.""" + user = bundle_mod.os_user() + settings = config.load_settings({MODE: "local", STATE: str(state_dir)}) + with bundle_mod.BundledServer(settings) as server: + with _owner(server) as conn: + rules = conn.execute( + "select type, database, user_name, auth_method, options, error " + "from pg_hba_file_rules order by rule_number").fetchall() + mappings = conn.execute( + "select map_name, sys_name, pg_username, error " + "from pg_ident_file_mappings order by map_number").fetchall() + for dsn, role in ((server.bundle.migration_dsn, config.BUNDLE_OWNER_ROLE), + (server.bundle.served_dsn, config.BUNDLE_SERVED_ROLE)): + import psycopg + + with psycopg.connect(dsn) as conn: + assert conn.execute("select current_user, system_user").fetchone() \ + == (role, f"peer:{user}") + assert rules == [ + ("local", ["all"], ["all"], "peer", [f"map={bundle_mod.IDENT_MAP}"], None), + ("host", ["all"], ["all"], "reject", None, None), + ("host", ["all"], ["all"], "reject", None, None)], rules + assert mappings == [ + (bundle_mod.IDENT_MAP, user, config.BUNDLE_OWNER_ROLE, None), + (bundle_mod.IDENT_MAP, user, config.BUNDLE_SERVED_ROLE, None)], mappings + + +@pytest.mark.parametrize("user", ["DOMAIN\\alice", "alice\\", "a\\\\b"]) +def test_the_server_reads_a_backslash_in_the_map_literally( + state_dir: Path, user: str) -> None: + """A user name holding a backslash, the shape an NSS or AD account takes + (`DOMAIN\\alice`), is written into `pg_ident.conf` as it is. PostgreSQL + 16 reads a quoted field's backslash LITERALLY: its tokenizer treats a + backslash specially only at the end of a line, as a continuation, and + never inside quotes. So the name is NOT escaped, and escaping it would + map a different name (Copilot review of #69, which suggested escaping, + answered by measurement). The server's own reading of the file is + asked, for each shape.""" + settings = config.load_settings({MODE: "local", STATE: str(state_dir)}) + with bundle_mod.BundledServer(settings) as server: + data = server.bundle.data_dir + (data / "pg_ident.conf").write_text( + bundle_mod.authentication_files(user)["pg_ident.conf"], encoding="utf-8") + try: + with _owner(server) as conn: + mappings = conn.execute( + "select sys_name, pg_username, error from pg_ident_file_mappings " + "order by map_number").fetchall() + finally: + bundle_mod.write_authentication(data, bundle_mod.os_user()) + assert mappings == [(user, config.BUNDLE_OWNER_ROLE, None), + (user, config.BUNDLE_SERVED_ROLE, None)], mappings + + +def test_a_role_outside_the_map_is_refused_even_for_this_os_user( + state_dir: Path) -> None: + """The MAP decides, not the socket. The same OS user, over the same + 0700 socket, asking for a role the map does not name, is refused by + peer authentication. A suite that does not run as root cannot connect + as a second OS user. What stands for that case is the map itself, read + back above, which names this user and no other.""" + import psycopg + from psycopg import sql + + settings = config.load_settings({MODE: "local", STATE: str(state_dir)}) + with bundle_mod.BundledServer(settings) as server: + with _owner(server) as conn: + conn.execute(sql.SQL("create role {} login").format( + sql.Identifier("odx_stranger"))) + with pytest.raises(psycopg.OperationalError) as caught: + psycopg.connect(server.bundle.dsn("odx_stranger")).close() + assert "peer authentication failed" in str(caught.value).lower(), caught.value + + +@pytest.mark.parametrize("mode", ["database", "true"]) +def test_a_replication_connection_is_refused_logical_or_physical( + state_dir: Path, mode: str) -> None: + """A PHYSICAL replication connection (`replication=true`) matches no + rule in `pg_hba.conf`. A LOGICAL one (`replication=database`) names a + database, and the one local rule admitted it as `peer:`, where + `IDENTIFY_SYSTEM` answered (adversarial review of #69). The server + starts no WAL sender (`max_wal_senders=0`), so both are refused, as the + owner role and over the same socket.""" + import psycopg + from psycopg.conninfo import make_conninfo + + settings = config.load_settings({MODE: "local", STATE: str(state_dir)}) + with bundle_mod.BundledServer(settings) as server: + with pytest.raises(psycopg.OperationalError) as caught: + psycopg.connect(make_conninfo(server.bundle.migration_dsn, + replication=mode)).close() + with _owner(server) as conn: # an ordinary one still is + senders = conn.execute("show max_wal_senders").fetchone()[0] + message = str(caught.value).lower() + assert ("max_wal_senders" in message if mode == "database" + else ("max_wal_senders" in message or "no pg_hba.conf entry" in message)), message + assert senders == "0", senders + + +def test_an_older_trust_cluster_is_brought_back_to_peer_on_start( + state_dir: Path) -> None: + """A data directory an earlier build initialized with `trust` (or a file + edited by hand) is put back to the one configuration before the next + launch, so it never serves as trust.""" + settings = config.load_settings({MODE: "local", STATE: str(state_dir)}) + bundle_mod.BundledServer(settings).start().stop() + data = config.DatabaseBundle(state_dir).data_dir + (data / "pg_hba.conf").write_text("local all all trust\n", encoding="utf-8") + (data / "pg_ident.conf").write_text("", encoding="utf-8") + with bundle_mod.BundledServer(settings) as server: + import psycopg + + with psycopg.connect(server.bundle.served_dsn) as conn: + method = conn.execute("select system_user").fetchone()[0] + expected = bundle_mod.authentication_files(bundle_mod.os_user()) + for name, content in expected.items(): + assert (data / name).read_text(encoding="utf-8") == content, name + assert stat.S_IMODE((data / name).stat().st_mode) == 0o600, name + assert method == f"peer:{bundle_mod.os_user()}", method + + +def test_a_cluster_whose_configuration_points_elsewhere_runs_on_its_own_files( + state_dir: Path, tmp_path: Path) -> None: + """An existing cluster's `postgresql.conf` can point `hba_file` and + `ident_file` at outside files, a `trust` one say, and `data_directory` + at a cluster that is not this install's (Copilot review of #69). The + launch pins all three on the command line, which outranks the file. So + the server reads the two files this install wrote, from its own data + directory, and every connection is still peer.""" + settings = config.load_settings({MODE: "local", STATE: str(state_dir)}) + bundle_mod.BundledServer(settings).start().stop() + data = config.DatabaseBundle(state_dir).data_dir + outside = tmp_path / "outside" + outside.mkdir() + (outside / "trust_hba.conf").write_text("local all all trust\n", encoding="utf-8") + (outside / "ident.conf").write_text("", encoding="utf-8") + with (data / "postgresql.conf").open("a", encoding="utf-8") as conf: + conf.write(f"\nhba_file = '{outside / 'trust_hba.conf'}'\n" + f"ident_file = '{outside / 'ident.conf'}'\n" + f"data_directory = '{outside / 'no-such-cluster'}'\n") + with bundle_mod.BundledServer(settings) as server: + import psycopg + + with _owner(server) as conn: + shown = {name: conn.execute(f"show {name}").fetchone()[0] + for name in ("data_directory", "hba_file", "ident_file")} + with psycopg.connect(server.bundle.served_dsn) as conn: + method = conn.execute("select system_user").fetchone()[0] + assert shown == {"data_directory": str(data), + "hba_file": str(data / "pg_hba.conf"), + "ident_file": str(data / "pg_ident.conf")}, shown + assert method == f"peer:{bundle_mod.os_user()}", method + + +def _verb(state: Path, *verb: str) -> tuple[int, dict]: + """`OPENDOX_INSTALL_MODE=local opendox-runtime runtime `, a SECOND process.""" + done = subprocess.run( + [sys.executable, "-m", "opendox.runtime.cli", "runtime", *verb], + env=_clean_env(**{MODE: "local", STATE: str(state)}), cwd=ROOT, + capture_output=True, text=True, timeout=60) + return done.returncode, json.loads(done.stdout) + + +@pytest.mark.parametrize("shape", ["linked-run", "linked-data", "open-state", + "no-server"]) +def test_the_local_verbs_connect_only_to_their_own_verified_server( + state_dir: Path, shape: str) -> None: + """The adversarial review of #69 (L2): a second state tree whose + `postgres/run` is a link to a RUNNING bundle's socket directory, in a + state directory every user could write. `status` and `migrate` ran on + it as the owner role, against the other bundle's server, while a start + refused the same tree. Each verb now asks the tree check and a live + server of its OWN data directory before any connection. A valid tree + with no server is answered the same way, by name and unconnected. And + `status` reports no pid for a tree it refuses, a `data` linked to the + running bundle's included (Copilot review of #69).""" + settings = config.load_settings({MODE: "local", STATE: str(state_dir)}) + other = Path(tempfile.mkdtemp(prefix="odx-o-", dir="/tmp" if os.path.isdir("/tmp") else None)) + try: + (other / "postgres").mkdir(mode=0o700) + if shape == "linked-data": + (other / "postgres" / "data").symlink_to( + config.DatabaseBundle(state_dir).data_dir) + else: + (other / "postgres" / "data").mkdir(mode=0o700) + if shape in {"no-server", "linked-data"}: + (other / "postgres" / "run").mkdir(mode=0o700) + else: + (other / "postgres" / "run").symlink_to( + config.DatabaseBundle(state_dir).socket_dir) + if shape == "open-state": + other.chmod(0o777) + with bundle_mod.BundledServer(settings): + status_code, status = _status(other) + migrate_code, migrate = _verb(other, "migrate") + reset_code, reset = _verb(other, "reset", "--confirm", + cli.RESET_CONFIRMATION) + # the positive control: the running bundle's own verbs connect, + # and its schema was not dropped through the other tree's link + own_code, own = _status(state_dir) + finally: + other.chmod(0o700) + shutil.rmtree(other, ignore_errors=True) + expected = {"linked-run": "is a symbolic link", + "linked-data": "is a symbolic link", + "open-state": "writable by every user", + "no-server": "no bundled server is running"}[shape] + assert status_code == 1 and status["database"].startswith("not probed: "), status + assert status["database_bundle"]["pid"] is None, status["database_bundle"] + assert expected in status["database"], status + assert migrate_code == 1, migrate + assert migrate["refusal"] == "local-bundle-unverified", migrate + assert expected in migrate["message"], migrate + assert reset_code == 1 and reset["refusal"] == "local-bundle-unverified", reset + assert expected in reset["message"], reset + assert own["database"] == "reachable" and own_code == 0, own + assert own["database_bundle"]["pid"] is not None, own["database_bundle"] + assert own["applied_migrations"] and not own["pending_migrations"], own + + +def test_migrate_under_the_local_mode_uses_the_bundle_and_refuses_a_dsn( + state_dir: Path) -> None: + """`runtime migrate` is part of the same install: it reads the bundle's + owner DSN, and an operator's migration DSN beside `local` is refused.""" + settings = config.load_settings({MODE: "local", STATE: str(state_dir)}) + with bundle_mod.BundledServer(settings): + done = subprocess.run( + [sys.executable, "-m", "opendox.runtime.cli", "runtime", "migrate"], + env=_clean_env(**{MODE: "local", STATE: str(state_dir)}), cwd=ROOT, + capture_output=True, text=True, timeout=60) + evidence = json.loads(done.stdout) + assert done.returncode == 0 and evidence["applied"] == [], evidence + done = subprocess.run( + [sys.executable, "-m", "opendox.runtime.cli", "runtime", "migrate"], + env=_clean_env(**{MODE: "local", STATE: str(state_dir), + PREFIX + "MIGRATION_DATABASE_URL": + "postgresql://m:hunter2@db.invalid/x"}), + cwd=ROOT, capture_output=True, text=True, timeout=60) + evidence = json.loads(done.stdout) + assert evidence["refusal"] == "configuration", evidence + assert PREFIX + "MIGRATION_DATABASE_URL" in evidence["message"] + assert "hunter2" not in done.stdout + + +# -- the migrations gap: a wheel, outside any checkout -------------------------- + + +def test_a_wheel_install_migrates_its_bundled_server_outside_a_checkout( + state_dir: Path, tmp_path: Path) -> None: + """Build this package's wheel, install it OUTSIDE the checkout, run it from + a directory with no `migrations/`, and migrate the bundled server from the + copy the wheel carries (the holder's assignment to T072).""" + source = tmp_path / "source" + source.mkdir() + for name in ("pyproject.toml", "src", "migrations"): + item = ROOT / name + (shutil.copytree if item.is_dir() else shutil.copy2)( + item, source / name, **({"ignore": shutil.ignore_patterns( + "__pycache__", "*.egg-info")} if item.is_dir() else {})) + wheels = tmp_path / "wheels" + built = subprocess.run( + [sys.executable, "-m", "pip", "wheel", "--no-deps", "--no-index", + "--no-build-isolation", "-q", "-w", str(wheels), str(source)], + capture_output=True, text=True, timeout=300) + assert built.returncode == 0, built.stderr[-3000:] + (wheel,) = wheels.glob("opendox-*.whl") + prefix = tmp_path / "prefix" + # `--ignore-installed` IS LOAD-BEARING: without it pip treats the suite's + # own (editable) `opendox` as the installed copy of the same project and + # UNINSTALLS it before writing the new one under `--prefix` — measured, it + # emptied the environment this very suite runs in. + installed = subprocess.run( + [sys.executable, "-m", "pip", "install", "--no-deps", "--no-index", + "--ignore-installed", "-q", "--prefix", str(prefix), str(wheel)], + capture_output=True, text=True, timeout=300) + assert installed.returncode == 0, installed.stderr[-3000:] + assert "uninstall" not in (installed.stdout + installed.stderr).lower() + # ...and the environment this suite runs in still has its own install + still = subprocess.run( + [sys.executable, "-c", "import importlib.metadata as m; " + "print(m.distribution('opendox').version)"], + env=_clean_env(PYTHONPATH=""), capture_output=True, text=True, timeout=60) + assert still.returncode == 0, "installing the wheel removed the suite's opendox" + site = Path(sysconfig.get_path("purelib", vars={"base": str(prefix), + "platbase": str(prefix)})) + elsewhere = tmp_path / "elsewhere" + elsewhere.mkdir() + program = f""" +import json, sys +from pathlib import Path +import opendox +from opendox.runtime import bundle, config +prefix = Path({str(prefix)!r}).resolve() +assert Path(opendox.__file__).resolve().is_relative_to(prefix), opendox.__file__ +assert not Path("migrations").exists() +settings = config.load_settings() +found = Path(settings.migrations_dir).resolve() +assert found.is_relative_to(prefix / "share" / "opendox"), found +with bundle.BundledServer(settings) as server: + print(json.dumps({{"applied": server.applied, "dir": str(found)}})) +""" + env = _clean_env(**{MODE: "local", STATE: str(state_dir)}) + env["PYTHONPATH"] = str(site) + done = subprocess.run([sys.executable, "-c", program], cwd=elsewhere, + env=env, capture_output=True, text=True, timeout=120) + assert done.returncode == 0, done.stderr[-3000:] + result = json.loads(done.stdout.strip().splitlines()[-1]) + assert result["applied"] == ["0001", "0002"], result diff --git a/tests_runtime/test_install_mode.py b/tests_runtime/test_install_mode.py index 0c5477a4..3cbbf4bb 100644 --- a/tests_runtime/test_install_mode.py +++ b/tests_runtime/test_install_mode.py @@ -10,10 +10,10 @@ The exception is `test_runtime_status_of_a_healthy_local_install_exits_zero`, which is DB-BACKED (Copilot review of openDox-code#67). A healthy local -`status` exits 0 only against a database that answers, so that case takes the -suite's `postgres_dsn` and `database` fixtures (`tests_runtime/conftest.py`). -Like every DB-backed case, it is skipped where there is no Postgres, and it -fails under CI. +`status` exits 0 only against a database that answers. A local install refuses +an operator's DSN (T072), so that case starts the local install's OWN bundled +server on a fresh state directory. It needs the `local` extra's binaries, +which the `test` extra installs. WHAT IS RULED AND WHAT IS READ, so a reviewer can tell them apart: @@ -70,6 +70,11 @@ HOSTED = {**DSNS, PREFIX + "OIDC_ISSUER": "https://issuer.example.invalid/realms/fixture", PREFIX + "OIDC_AUDIENCE": "fixture"} +#: A LOCAL install's whole environment: the selector and a state directory. +#: No DSN — the local install supplies both from the server it bundles +#: (plan 034 T072), and one given beside it is refused. Nothing is created +#: there: `load_settings` derives paths, and starts nothing. +LOCAL = {MODE: "local", PREFIX + "STATE_DIR": "/nonexistent/opendox-state"} def _refusal(env: dict, **kwargs) -> str: @@ -198,7 +203,7 @@ def test_the_hosted_mode_is_unchanged() -> None: @pytest.mark.parametrize("selection", ["setting", "flag"]) def test_a_local_install_needs_no_broker(selection: str) -> None: - env = dict(DSNS) + env = {PREFIX + "STATE_DIR": LOCAL[PREFIX + "STATE_DIR"]} kwargs = {} if selection == "setting": env[MODE] = "local" @@ -221,13 +226,19 @@ def test_a_broker_setting_beside_the_local_mode_is_refused_by_name( """A holder reading (#656, T070): a broker setting says hosted was meant.""" assert set(HOSTED_ONLY_SETTINGS) == {PREFIX + "OIDC_ISSUER", PREFIX + "OIDC_AUDIENCE", - PREFIX + "OIDC_JWKS_URL"} - secret = "https://svc:hunter2@broker.example.invalid/realms/x" - message = _refusal({**DSNS, MODE: "local", name: secret}) + PREFIX + "OIDC_JWKS_URL", + # and T072's two: the local install + # supplies both DSNs itself (13.1) + PREFIX + "DATABASE_URL", + PREFIX + "MIGRATION_DATABASE_URL"} + secret = ("https://svc:hunter2@broker.example.invalid/realms/x" + if "OIDC" in name else "postgresql://u:hunter2@db.invalid/x") + message = _refusal({**LOCAL, name: secret}) assert name in message, message assert "hunter2" not in message, "the value must not be repeated" # and the flag spelling of the same selection refuses it the same way - assert name in _refusal({**DSNS, name: secret}, local_flag=True) + assert name in _refusal({PREFIX + "STATE_DIR": LOCAL[PREFIX + "STATE_DIR"], + name: secret}, local_flag=True) def test_every_broker_setting_given_is_named_at_once() -> None: @@ -239,8 +250,7 @@ def test_every_broker_setting_given_is_named_at_once() -> None: @pytest.mark.parametrize("host", sorted(LOCAL_BIND_HOSTS)) def test_a_local_install_binds_each_loopback_spelling(host: str) -> None: - settings = load_settings({**DSNS, MODE: "local", - PREFIX + "BIND_HOST": host}) + settings = load_settings({**LOCAL, PREFIX + "BIND_HOST": host}) assert settings.bind_host == host @@ -251,7 +261,7 @@ def test_a_local_install_refuses_a_non_loopback_bind_naming_the_rule( """13.4: loopback ONLY, and no opt-in. `127.0.0.2` is refused too: the document server does not treat it as loopback (`serve.LOOPBACK_HOSTS`), and the mode makes the SAME judgement at its own boundary.""" - message = _refusal({**DSNS, MODE: "local", PREFIX + "BIND_HOST": host}) + message = _refusal({**LOCAL, PREFIX + "BIND_HOST": host}) assert PREFIX + "BIND_HOST" in message, message assert "loopback" in message.lower(), message assert "no opt-in" in message.lower(), message @@ -291,9 +301,8 @@ def run(self) -> None: stub.Config, stub.Server = _Config, _Server scrubbed.setitem(sys.modules, "uvicorn", stub) scrubbed.setattr(app_module, "create_app", lambda **kwargs: object()) - for name, value in DSNS.items(): + for name, value in LOCAL.items(): scrubbed.setenv(name, value) - scrubbed.setenv(MODE, "local") code, evidence = _run(["runtime", "serve"]) assert served == [], "the API was started for a LOCAL install" assert code == 1 @@ -314,9 +323,8 @@ def _no_broker(_settings): "install, which has no broker") monkeypatch.setattr(oidc, "build_verifier", _no_broker) - for name, value in DSNS.items(): + for name, value in LOCAL.items(): scrubbed.setenv(name, value) - scrubbed.setenv(MODE, "local") code, evidence = _run(["runtime", "status", "--probe-timeout", "0.2"]) assert evidence.get("refusal") is None, evidence assert evidence["broker_keys"] == "not configured (local mode)" @@ -324,8 +332,13 @@ def _no_broker(_settings): assert evidence["settings"][MODE] == INSTALL_MODE_LOCAL assert evidence["settings"][PREFIX + "OIDC_ISSUER"] == "" assert evidence["settings"][PREFIX + "OIDC_JWKS_URL"] == "" - # the database half (port 1, unreachable) is the ONLY reason `ok` is false - assert evidence["database"].startswith("unreachable"), evidence + # the database half is the ONLY reason `ok` is false: no bundled server + # is running on this (nonexistent) state directory, and `status` reports + # that rather than starting one. It says so WITHOUT connecting: a local + # socket is judged before it is asked (adversarial review of #69). + assert evidence["database"].startswith( + "not probed: no bundled server is running"), evidence + assert evidence["database_bundle"]["pid"] is None, evidence assert code == 1 @@ -339,9 +352,11 @@ def test_migrate_and_reset_refuse_a_broker_setting_beside_the_local_mode( scrubbed, verb: list[str], name: str) -> None: """The migration loader asks what every other loader asks of `local` (Copilot review of openDox-code#67): refused at CONFIGURATION, before any - database is reached — `reset` included, confirmation and all.""" - scrubbed.setenv(PREFIX + "MIGRATION_DATABASE_URL", DSNS[PREFIX + "MIGRATION_DATABASE_URL"]) - scrubbed.setenv(MODE, "local") + database is reached — `reset` included, confirmation and all. The local + shape supplies its own migration DSN (T072), so the broker setting is the + only fault.""" + for local_name, value in LOCAL.items(): + scrubbed.setenv(local_name, value) scrubbed.setenv(name, "https://issuer.example.invalid/realms/x" if name != PREFIX + "OIDC_AUDIENCE" else "fixture") code, evidence = _run(verb) @@ -352,8 +367,8 @@ def test_migrate_and_reset_refuse_a_broker_setting_beside_the_local_mode( def test_migrate_refuses_a_non_loopback_bind_beside_the_local_mode( scrubbed) -> None: - scrubbed.setenv(PREFIX + "MIGRATION_DATABASE_URL", DSNS[PREFIX + "MIGRATION_DATABASE_URL"]) - scrubbed.setenv(MODE, "local") + for name, value in LOCAL.items(): + scrubbed.setenv(name, value) scrubbed.setenv(PREFIX + "BIND_HOST", "0.0.0.0") code, evidence = _run(["runtime", "migrate"]) assert code == 1 and evidence["refusal"] == "configuration", evidence @@ -361,15 +376,23 @@ def test_migrate_refuses_a_non_loopback_bind_beside_the_local_mode( def test_runtime_status_of_a_healthy_local_install_exits_zero( - scrubbed, monkeypatch: pytest.MonkeyPatch, postgres_dsn: str, - database) -> None: + scrubbed, monkeypatch: pytest.MonkeyPatch) -> None: """THE EXIT CODE IS THE DATABASE'S VERDICT ALONE (Copilot review of openDox-code#67). A migrated, reachable database is the only thing a local install's `status` needs, and with it the verb exits 0. The broker is reported as not configured and is never probed, so F13.1's `set -e` survives. The other local cases force a database fault and exit 1, so they cannot tell a broker counted as a fault from a database that failed. + + The database is the local install's OWN (T072): a local install refuses + an operator's DSN, so the case starts the bundled server on a fresh state + directory and asks `status` about it. """ + import shutil + import tempfile + from pathlib import Path + + from opendox.runtime import bundle as bundle_mod from opendox.runtime import oidc def _no_broker(_settings): @@ -377,16 +400,14 @@ def _no_broker(_settings): "install, which has no broker") monkeypatch.setattr(oidc, "build_verifier", _no_broker) - # `make_conninfo`, NOT a `?options=` suffix: `OPENDOX_TEST_DATABASE_URL` - # may be libpq's keyword/value form as well as a URI, and a suffix on - # `… dbname=opendox` names the database `opendox?options=…` instead of - # selecting the schema (Copilot review of openDox-code#67). - from psycopg.conninfo import make_conninfo - - scrubbed.setenv(PREFIX + "DATABASE_URL", make_conninfo( - postgres_dsn, options=f"-c search_path={database.schema},public")) - scrubbed.setenv(MODE, "local") - code, evidence = _run(["runtime", "status", "--probe-timeout", "5"]) + state = Path(tempfile.mkdtemp(prefix="odx-s-", dir="/tmp")) + try: + scrubbed.setenv(MODE, "local") + scrubbed.setenv(PREFIX + "STATE_DIR", str(state)) + with bundle_mod.BundledServer(load_settings()): + code, evidence = _run(["runtime", "status", "--probe-timeout", "10"]) + finally: + shutil.rmtree(state, ignore_errors=True) assert evidence["database"] == "reachable", evidence assert evidence["pending_migrations"] == [], evidence assert not evidence["migration_drift"], evidence @@ -407,7 +428,7 @@ def test_runtime_status_without_the_runtime_extra_reports_the_broker_by_mode( import sys scrubbed.setitem(sys.modules, "opendox.runtime.db", None) - for name, value in (HOSTED if mode == INSTALL_MODE_HOSTED else DSNS).items(): + for name, value in (HOSTED if mode == INSTALL_MODE_HOSTED else LOCAL).items(): scrubbed.setenv(name, value) scrubbed.setenv(MODE, mode) code, evidence = _run(["runtime", "status", "--probe-timeout", "0.2"]) @@ -418,9 +439,12 @@ def test_runtime_status_without_the_runtime_extra_reports_the_broker_by_mode( assert evidence["broker_keys"] == "not configured (local mode)", evidence assert "broker_discovery" in evidence, evidence assert evidence["broker_discovery"] is None + # and the bundle is reported before the early return (T072) + assert evidence["database_bundle"] is not None, evidence else: assert evidence["broker_keys"] == "not probed", evidence assert "broker_discovery" not in evidence, evidence + assert evidence["database_bundle"] is None, evidence def test_runtime_status_reports_the_hosted_mode_it_loaded(scrubbed) -> None: diff --git a/tests_runtime/test_local_lifecycle.py b/tests_runtime/test_local_lifecycle.py new file mode 100644 index 00000000..b4692375 --- /dev/null +++ b/tests_runtime/test_local_lifecycle.py @@ -0,0 +1,1332 @@ +"""The LOCAL install's lifecycle, hardened (plan 034 T072; Copilot review of +openDox-code#69). + +Hermetic: nothing here starts a real PostgreSQL server. The bundle's binaries +are stood in by small scripts where a phase must fail, and the entry point's +bundle is stood in where the case is the entry point's own handling. The real +server's cases are `test_bundled_postgres.py`'s. + + * which migrations a local install runs: this installation's own, never the + working directory's, and never another installation's found by name; + * a stale `postmaster.pid` is believed only for this data directory's + postmaster, asked of the kernel; + * `initdb` never leaves a half-built data directory; + * every phase of a start fails as the one named refusal, never a traceback; + * an interrupt anywhere in the local lifecycle is a clean stop. +""" + +from __future__ import annotations + +import os +import shutil +import signal +import stat +import subprocess +import sys +import tempfile +import time +from pathlib import Path + +import pytest + +from opendox import cli as cli_mod +from opendox.runtime import bundle as bundle_mod +from opendox.runtime import config +from opendox.runtime.config import PREFIX + +ROOT = Path(__file__).resolve().parents[1] +MODE = PREFIX + "INSTALL_MODE" +STATE = PREFIX + "STATE_DIR" +MIGRATIONS = PREFIX + "MIGRATIONS_DIR" + + +@pytest.fixture() +def scrubbed(monkeypatch: pytest.MonkeyPatch) -> pytest.MonkeyPatch: + for name in config.SETTING_NAMES: + monkeypatch.delenv(name, raising=False) + return monkeypatch + + +@pytest.fixture() +def short_state(): + """A state directory with a SHORT path (the socket's path is bounded).""" + path = Path(tempfile.mkdtemp(prefix="odx-l-", dir="/tmp")) + yield path + for directory in path.rglob("*"): + if directory.is_dir(): + directory.chmod(0o700) + shutil.rmtree(path, ignore_errors=True) + + +def _local(state: Path) -> config.RuntimeSettings: + return config.load_settings({MODE: "local", STATE: str(state)}) + + +# -- which migrations a local install runs ------------------------------------ + + +@pytest.fixture() +def hostile_cwd(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Path: + """A working directory whose `migrations/` holds the pinned `0001` AND SQL + of its own: what a checkout of somebody else's repository can hold.""" + here = tmp_path / "somebody-elses-checkout" + (here / "migrations").mkdir(parents=True) + for sql in (ROOT / "migrations").glob("0001*.sql"): + shutil.copy2(sql, here / "migrations" / sql.name) + (here / "migrations" / "0099_not_this_products.sql").write_text( + "create table stolen (secret text);\n", encoding="utf-8") + monkeypatch.chdir(here) + return here + + +def test_a_local_install_never_runs_the_working_directorys_migrations( + hostile_cwd: Path, short_state: Path) -> None: + settings = _local(short_state) + found = Path(settings.migrations_dir).resolve() + assert not found.is_relative_to(hostile_cwd.resolve()), found + assert found == (ROOT / "migrations").resolve(), found + migrate = config.load_migration_settings({MODE: "local", STATE: str(short_state)}) + assert Path(migrate.migrations_dir).resolve() == found + + +def test_a_hosted_install_keeps_its_working_directory_default( + hostile_cwd: Path) -> None: + """13.6: the hosted default is unchanged, and the deploy files set the + variable explicitly.""" + assert config.migrations_dir({}) == Path("migrations") + + +def test_an_explicit_migrations_dir_is_used_as_given_in_either_shape( + tmp_path: Path) -> None: + chosen = str(tmp_path / "chosen") + assert config.migrations_dir({MIGRATIONS: chosen}, local=True) == Path(chosen) + assert config.migrations_dir({MIGRATIONS: chosen}) == Path(chosen) + + +def test_a_local_installation_that_carries_none_is_refused_naming_the_setting( + monkeypatch: pytest.MonkeyPatch, hostile_cwd: Path) -> None: + monkeypatch.setattr(config, "installation_migrations_dir", lambda: None) + with pytest.raises(config.ConfigurationError) as caught: + config.migrations_dir({}, local=True) + assert MIGRATIONS in str(caught.value), caught.value + + +class _FakeFile: + """One `importlib.metadata` RECORD entry, located under `base`.""" + + def __init__(self, base: Path, relative: str) -> None: + self.base, self.relative = base, relative + self.name = Path(relative).name + self.parts = Path(relative).parts + + def __fspath__(self) -> str: + return self.relative + + def locate(self) -> Path: + return self.base / self.relative + + +class _FakeDistribution: + def __init__(self, base: Path) -> None: + site = "lib/python3/site-packages" + self.files = [ + _FakeFile(base, f"{site}/opendox/runtime/config.py"), + _FakeFile(base, "share/opendox/migrations/0001_older.sql"), + ] + + +def test_another_installations_record_is_not_this_ones(tmp_path: Path) -> None: + """A distribution found by NAME whose RECORD does not hold the running + module (an older wheel beside a checkout) is not this installation.""" + older = _FakeDistribution(tmp_path / "older-wheel") + assert config._distribution_migrations(Path(config.__file__), older) is None + + +def test_the_record_of_the_distribution_that_holds_the_module_is_used( + tmp_path: Path) -> None: + base = tmp_path / "wheel" + dist = _FakeDistribution(base) + module = dist.files[0].locate() + module.parent.mkdir(parents=True) + module.write_text("# stand-in\n", encoding="utf-8") + assert config._distribution_migrations(module, dist) == \ + (base / "share/opendox/migrations").resolve() + + +def test_the_source_tree_wins_over_a_wheel_found_by_name( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + """A checkout run with `src/` on the path, beside an older wheel: the + checkout's own migrations, not the wheel's (Copilot review of + openDox-code#69).""" + older = _FakeDistribution(tmp_path / "older-wheel") + monkeypatch.setattr(config.importlib.metadata, "distribution", + lambda name: older) + assert config.installation_migrations_dir() == ROOT / "migrations" + + +def test_the_source_tree_is_asked_before_any_record( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + """The documented ORDER, pinned: even a RECORD that does hold the running + module is asked only after the source tree it was imported from.""" + class _Holding: + files = [_FakeFile(ROOT / "src", "opendox/runtime/config.py"), + _FakeFile(tmp_path, "share/opendox/migrations/0001_other.sql")] + + assert config._distribution_migrations(Path(config.__file__), _Holding()) == \ + (tmp_path / "share/opendox/migrations").resolve() + monkeypatch.setattr(config.importlib.metadata, "distribution", + lambda name: _Holding()) + assert config.installation_migrations_dir() == ROOT / "migrations" + + +def test_a_package_outside_src_is_not_a_source_tree(tmp_path: Path) -> None: + """A `--target` install inside some other checkout: its parent has a + `pyproject.toml` and a `migrations/`, and it is still not this tree.""" + target = tmp_path / "checkout" / "vendor" / "opendox" / "runtime" + target.mkdir(parents=True) + (tmp_path / "checkout" / "pyproject.toml").write_text( + '[project]\nname = "opendox"\n', encoding="utf-8") + (tmp_path / "checkout" / "migrations").mkdir() + assert config._source_tree_migrations(target / "config.py") is None + + +# -- a stale postmaster.pid --------------------------------------------------- + + +def _decoy(executable: str, cwd: Path, *argv: str) -> subprocess.Popen: + return subprocess.Popen([executable, *argv], cwd=cwd, + stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + + +def _lock(bundle: config.DatabaseBundle, pid: int) -> None: + bundle.data_dir.mkdir(parents=True, exist_ok=True) + (bundle.data_dir / "postmaster.pid").write_text( + f"{pid}\n{bundle.data_dir}\n", encoding="utf-8") + + +needs_proc = pytest.mark.skipif(not bundle_mod.PROC.joinpath("self").exists(), + reason="asks /proc") + + +@needs_proc +def test_a_recycled_pid_is_not_believed_even_with_postgres_in_its_argv( + short_state: Path) -> None: + """Even IN the data directory, and with `postgres -D ` in its + argv: the executable is not `postgres`, so it is not this server.""" + bundle = config.DatabaseBundle(short_state) + bundle.data_dir.mkdir(parents=True) + decoy = _decoy(sys.executable, bundle.data_dir, "-c", + "import time; time.sleep(60)", "postgres", "-D", + str(bundle.data_dir)) + try: + _lock(bundle, decoy.pid) + assert bundle_mod.running_pid(bundle) is None + assert bundle_mod.report(bundle)["pid"] is None + # and the start is not refused over it: the stale lock is removed + bundle_mod._remove_a_proven_stale_lock(bundle) + assert not (bundle.data_dir / "postmaster.pid").exists() + finally: + decoy.kill() + decoy.wait(timeout=10) + + +@needs_proc +def test_a_postgres_serving_another_directory_is_not_this_server( + short_state: Path, tmp_path: Path) -> None: + """The exact pair: an executable NAMED `postgres` is not enough, it must + be running in THIS data directory. And the positive control: the same + decoy, run in the data directory, is believed.""" + sleeper = shutil.which("sleep") + assert sleeper + named = tmp_path / "bin" / "postgres" + named.parent.mkdir() + shutil.copy2(sleeper, named) + bundle = config.DatabaseBundle(short_state) + bundle.data_dir.mkdir(parents=True) + elsewhere = tmp_path / "another-data-dir" + elsewhere.mkdir() + for cwd, believed in ((elsewhere, False), (bundle.data_dir, True)): + decoy = _decoy(str(named), cwd, "60") + try: + _lock(bundle, decoy.pid) + got = bundle_mod.running_pid(bundle) + assert got == (decoy.pid if believed else None), (cwd, got) + finally: + decoy.kill() + decoy.wait(timeout=10) + + +def test_a_pid_nothing_can_describe_is_not_believed_and_its_lock_is_kept( + short_state: Path, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + """No `/proc` (macOS, the BSDs): nothing can say what the pid is, so it + is not reported as this server, and its lock is NOT removed, since + nothing proved it stale. PostgreSQL's own interlock is left to judge a + start. The decoy would be believed if it could be described (the case + above, where it is).""" + monkeypatch.setattr(bundle_mod, "PROC", tmp_path / "no-proc") + sleeper = shutil.which("sleep") + assert sleeper + named = tmp_path / "bin" / "postgres" + named.parent.mkdir() + shutil.copy2(sleeper, named) + bundle = config.DatabaseBundle(short_state) + bundle.data_dir.mkdir(parents=True) + decoy = _decoy(str(named), bundle.data_dir, "60") + try: + _lock(bundle, decoy.pid) + assert bundle_mod.running_pid(bundle) is None + bundle_mod._remove_a_proven_stale_lock(bundle) + assert (bundle.data_dir / "postmaster.pid").exists() + finally: + decoy.kill() + decoy.wait(timeout=10) + + +@needs_proc +def test_a_process_that_exits_before_it_is_described_is_not_believed( + short_state: Path, monkeypatch: pytest.MonkeyPatch) -> None: + """Alive at the signal check, gone by the time it is described: that is + a stale lock, never proof of a server.""" + bundle = config.DatabaseBundle(short_state) + gone = subprocess.Popen(["true"]) + gone.wait(timeout=10) + _lock(bundle, gone.pid) + monkeypatch.setattr(bundle_mod.os, "kill", lambda pid, sig: None) + assert bundle_mod.running_pid(bundle) is None + + +def test_another_users_process_is_not_this_server(short_state: Path) -> None: + """A pid the kernel will not let this user signal belongs to another user, + and this bundle's server runs as the owner of its 0700 data directory. + Pid 1 is another user's on an ordinary host and in CI. Where the suite + runs as pid 1's owner, the pid is still not a postgres in this directory. + Either way it is not believed.""" + bundle = config.DatabaseBundle(short_state) + _lock(bundle, 1) + assert bundle_mod.running_pid(bundle) is None + + +# -- the state directory, refused by name -------------------------------------- + + +def test_a_state_dir_naming_an_unknown_user_is_refused_by_name() -> None: + raw = "~no-such-user-odx-8f3a/state" + with pytest.raises(config.ConfigurationError) as caught: + config.state_dir({STATE: raw}) + assert STATE in str(caught.value), caught.value + with pytest.raises(config.ConfigurationError): + config.load_settings({MODE: "local", STATE: raw}) + # a HOSTED install never reads it, and is not refused over it (13.6) + hosted = config.load_settings({ + STATE: raw, PREFIX + "DATABASE_URL": "postgresql://s@127.0.0.1:1/x", + PREFIX + "OIDC_ISSUER": "https://issuer.example.invalid/realms/x", + PREFIX + "OIDC_AUDIENCE": "fixture"}) + assert hosted.install_mode == config.INSTALL_MODE_HOSTED + + +def test_no_home_for_the_default_state_dir_is_refused_by_name( + monkeypatch: pytest.MonkeyPatch) -> None: + def _no_home(): + raise RuntimeError("Could not determine home directory.") + + monkeypatch.setattr(config.Path, "home", staticmethod(_no_home)) + with pytest.raises(config.ConfigurationError) as caught: + config.state_dir({}) + assert STATE in str(caught.value), caught.value + + +def test_a_relative_home_for_the_default_state_dir_is_refused_by_name( + monkeypatch: pytest.MonkeyPatch) -> None: + """`Path.home()` returns HOME as given; a relative one would put the + socket wherever each process happens to run (Copilot review of #69).""" + monkeypatch.setenv("HOME", "relative-home") + with pytest.raises(config.ConfigurationError) as caught: + config.state_dir({}) + assert STATE in str(caught.value) and "HOME" in str(caught.value) + # an absolute XDG_STATE_HOME still answers without HOME at all + assert config.state_dir({"XDG_STATE_HOME": "/srv/state"}) == \ + Path("/srv/state/opendox") + + +# -- libpq's environment, out of reach --------------------------------------- + + +def test_libpq_defaults_are_lifted_for_the_duration_and_put_back( + monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("PGHOSTADDR", "192.0.2.1") + monkeypatch.setenv("PGSERVICE", "no-such-service-odx") + monkeypatch.setenv("OPENDOX_NOT_LIBPQ", "kept") + with bundle_mod.isolated_from_libpq_environment(): + assert not [name for name in os.environ if name.startswith("PG")] + assert os.environ["OPENDOX_NOT_LIBPQ"] == "kept" + assert os.environ["PGHOSTADDR"] == "192.0.2.1" + assert os.environ["PGSERVICE"] == "no-such-service-odx" + + +def test_the_runtime_cli_isolates_only_a_local_install( + monkeypatch: pytest.MonkeyPatch, scrubbed) -> None: + from opendox.runtime import cli as runtime_cli + + seen: dict = {} + + def _verb(args) -> int: + seen["PGHOSTADDR"] = os.environ.get("PGHOSTADDR") + return 0 + + monkeypatch.setenv("PGHOSTADDR", "192.0.2.1") + parser = runtime_cli.build_parser() + monkeypatch.setattr(runtime_cli, "build_parser", lambda: parser) + real_parse = parser.parse_args + + def _parse(argv=None): + args = real_parse(argv) + args.func = _verb + return args + + monkeypatch.setattr(parser, "parse_args", _parse) + scrubbed.setenv(MODE, "local") + assert runtime_cli.main(["runtime", "status"]) == 0 + assert seen["PGHOSTADDR"] is None, "a local verb saw PGHOSTADDR" + scrubbed.setenv(MODE, "hosted") + assert runtime_cli.main(["runtime", "status"]) == 0 + assert seen["PGHOSTADDR"] == "192.0.2.1", "a hosted verb lost its libpq setting" + assert os.environ["PGHOSTADDR"] == "192.0.2.1" + + +# -- the socket's path, this user's to change ----------------------------------- + + +def _prepared(monkeypatch, tmp_path: Path, state: Path) -> bundle_mod.BundledServer: + binaries = _binaries(tmp_path, initdb='echo "initdb reached" >&2; exit 1') + monkeypatch.setattr(bundle_mod, "server_binaries", lambda: binaries) + return bundle_mod.BundledServer(_local(state)) + + +def test_a_state_dir_others_can_write_is_refused( + monkeypatch, tmp_path: Path, short_state: Path) -> None: + short_state.chmod(0o777) + server = _prepared(monkeypatch, tmp_path, short_state) + with pytest.raises(bundle_mod.BundleRefused) as caught: + server.start() + assert "writable by every user" in str(caught.value), caught.value + assert str(short_state) in str(caught.value) + + +@pytest.mark.parametrize("which", ["postgres", "run"]) +def test_a_symlink_inside_the_state_tree_is_refused_and_its_target_untouched( + monkeypatch, tmp_path: Path, short_state: Path, which: str) -> None: + target = tmp_path / "somewhere-else" + target.mkdir(mode=0o755) + target.chmod(0o755) + if which == "postgres": + (short_state / "postgres").symlink_to(target) + else: + (short_state / "postgres").mkdir(mode=0o700) + (short_state / "postgres" / "run").symlink_to(target) + server = _prepared(monkeypatch, tmp_path, short_state) + with pytest.raises(bundle_mod.BundleRefused) as caught: + server.start() + assert "symbolic link" in str(caught.value), caught.value + assert stat.S_IMODE(target.stat().st_mode) == 0o755, "the link's target was re-moded" + + +def test_an_ancestor_every_user_can_write_without_the_sticky_bit_is_refused( + monkeypatch, tmp_path: Path, short_state: Path) -> None: + open_dir = short_state / "open" + open_dir.mkdir() + open_dir.chmod(0o777) + server = _prepared(monkeypatch, tmp_path, open_dir / "state") + with pytest.raises(bundle_mod.BundleRefused) as caught: + server.start() + assert str(open_dir) in str(caught.value) and "not sticky" in str(caught.value) + + +def test_a_sticky_ancestor_is_accepted( + monkeypatch, tmp_path: Path, short_state: Path) -> None: + """The positive control, `/tmp`'s shape: every user can write it, and it + is sticky. The start gets past the tree and reaches `initdb`, which the + stand-in fails on purpose.""" + sticky = short_state / "sticky" + sticky.mkdir() + sticky.chmod(0o1777) + server = _prepared(monkeypatch, tmp_path, sticky / "state") + with pytest.raises(bundle_mod.BundleRefused) as caught: + server.start() + assert "initdb" in str(caught.value), caught.value + + +def test_a_group_writable_ancestor_is_refused_even_for_this_users_group( + monkeypatch, tmp_path: Path, short_state: Path) -> None: + """A group is other users, the user's own primary group included (Copilot + review of #69): a 0775 ancestor is refused whatever its group.""" + group = short_state / "group" + group.mkdir() + group.chmod(0o775) + assert group.stat().st_gid == os.getgid() # this user's own group + server = _prepared(monkeypatch, tmp_path, group / "state") + with pytest.raises(bundle_mod.BundleRefused) as caught: + server.start() + assert str(group) in str(caught.value) and "its group" in str(caught.value) + + +def test_a_link_in_a_directory_others_can_write_is_refused( + monkeypatch, tmp_path: Path, short_state: Path) -> None: + """The configured path goes through a symbolic link, and the link sits + in a directory every user can write. The link's TARGET is private, and + it is still refused, because anyone could replace the link (Copilot + review of #69).""" + private = short_state / "private" + private.mkdir(mode=0o700) + open_dir = short_state / "open" + open_dir.mkdir() + open_dir.chmod(0o777) + (open_dir / "link").symlink_to(private) + server = _prepared(monkeypatch, tmp_path, open_dir / "link" / "state") + with pytest.raises(bundle_mod.BundleRefused) as caught: + server.start() + assert str(open_dir) in str(caught.value) and "not sticky" in str(caught.value) + + +def test_this_users_own_link_to_a_private_directory_is_accepted( + monkeypatch, tmp_path: Path, short_state: Path) -> None: + private = short_state / "private" + private.mkdir(mode=0o700) + (short_state / "link").symlink_to(private) + server = _prepared(monkeypatch, tmp_path, short_state / "link" / "state") + with pytest.raises(bundle_mod.BundleRefused) as caught: + server.start() + assert "initdb" in str(caught.value), caught.value + + +def _foreign_lstat(monkeypatch, *links: Path) -> None: + """`os.lstat` answers that each of `links` belongs to another user. A + non-root suite cannot create another user's link, so it is stood in for + those paths only.""" + real_lstat = os.lstat + + def _lstat(path, *args, **kwargs): + info = real_lstat(path, *args, **kwargs) + if Path(path) in links: + fields = list(info) + fields[4] = os.getuid() + 4242 # st_uid + return os.stat_result(fields) + return info + + monkeypatch.setattr(bundle_mod.os, "lstat", _lstat) + + +def test_a_link_another_user_owns_is_refused( + monkeypatch, tmp_path: Path, short_state: Path) -> None: + """Another user's link could be pointed elsewhere after the check. A + non-root suite cannot create one, so its `lstat` is stood in, for that + one path only.""" + private = short_state / "private" + private.mkdir(mode=0o700) + link = short_state / "link" + link.symlink_to(private) + _foreign_lstat(monkeypatch, link) + server = _prepared(monkeypatch, tmp_path, link / "state") + with pytest.raises(bundle_mod.BundleRefused) as caught: + server.start() + assert str(link) in str(caught.value) and "symbolic link owned by" in str(caught.value) + + +@pytest.mark.parametrize("umask", [0o002, 0o200]) +def test_missing_directories_are_created_0700_whatever_the_umask( + monkeypatch, tmp_path: Path, short_state: Path, umask: int) -> None: + """A fresh default path creates the directories above the state tree + too. Under umask 0002 `mkdir(parents=True)` would make them 0775, and + the tree check would then refuse what this install had just made + (Copilot review of #69). A umask that takes the owner's own bits would + leave them unusable. Each is created exactly 0700, and the start reaches + `initdb`, which the stand-in fails on purpose.""" + state = short_state / "a" / "b" / "state" + server = _prepared(monkeypatch, tmp_path, state) + previous = os.umask(umask) + try: + with pytest.raises(bundle_mod.BundleRefused) as caught: + server.start() + # The narrowed umask is the start's alone: the user's is put back. + assert os.umask(umask) == umask, "the start left its own umask in place" + finally: + os.umask(previous) + assert "initdb" in str(caught.value), caught.value + for directory in (short_state / "a", short_state / "a" / "b", state, + state / "postgres", state / "postgres" / "run"): + assert stat.S_IMODE(directory.stat().st_mode) == 0o700, directory + + +@pytest.mark.parametrize("shape", ["foreign-link", "foreign-broken-link", + "open-ancestor", "link-in-open-dir"]) +def test_nothing_is_made_through_a_path_the_tree_check_refuses( + monkeypatch, tmp_path: Path, short_state: Path, shape: str) -> None: + """What exists is judged BEFORE anything is created (Copilot review of + #69). Before, the missing `postgres/run` was made through the path first + and refused only after, so it was created in a foreign link's target, or + beneath a directory every user can write.""" + private = short_state / "private" + private.mkdir(mode=0o700) + open_dir = short_state / "open" + open_dir.mkdir() + open_dir.chmod(0o777) + if shape in {"foreign-link", "foreign-broken-link"}: + target = private if shape == "foreign-link" else private / "gone" + state = short_state / "link" + state.symlink_to(target) + _foreign_lstat(monkeypatch, state) + expected, made = "symbolic link owned by", target / "postgres" + elif shape == "open-ancestor": + state = open_dir / "state" + expected, made = "not sticky", state + else: + (open_dir / "link").symlink_to(private) + state = open_dir / "link" / "state" + expected, made = "not sticky", private / "state" + server = _prepared(monkeypatch, tmp_path, state) + with pytest.raises(bundle_mod.BundleRefused) as caught: + server.start() + assert expected in str(caught.value), caught.value + assert not os.path.lexists(made), f"{made} was made before the refusal" + + +@pytest.mark.parametrize("shape", ["link-to-open", "open", "link-to-open-state", + "sticky-state"]) +def test_the_directory_creation_starts_from_judges_its_own_descriptor( + tmp_path: Path, short_state: Path, shape: str) -> None: + """`_make_private_directories` judges the directory it opens, by its + descriptor, before its first `mkdir` (adversarial review of #69). It is + asked here directly, with no path-wise check in front of it, so its own + guard is what is measured: a base that every user can write, reached + through a link or not, makes nothing beneath it. The state directory is + judged as the install's OWN, so even a sticky one is refused, where a + sticky ANCESTOR (`/tmp`'s shape) is accepted.""" + watched = short_state / "open" + watched.mkdir() + watched.chmod(0o1777 if shape == "sticky-state" else 0o777) + if shape == "link-to-open": + (short_state / "link").symlink_to(watched) + state = short_state / "link" / "state" + elif shape == "open": + state = watched / "state" + elif shape == "link-to-open-state": + (short_state / "link").symlink_to(watched) + state = short_state / "link" + else: + state = watched + with pytest.raises(bundle_mod.BundleRefused) as caught: + bundle_mod._make_private_directories(state / "postgres" / "run", state=state) + assert "writable by every user" in str(caught.value), caught.value + assert list(watched.iterdir()) == [], "made beneath an unsafe base" + + +@pytest.mark.parametrize("shape", ["link", "foreign-directory"]) +def test_a_name_put_in_the_way_first_is_refused_never_followed( + monkeypatch, tmp_path: Path, short_state: Path, shape: str) -> None: + """The race in a sticky directory, `/tmp`'s shape (Copilot review of + #69): every user can create a name there, so another user can put the + state directory's name in place between the check and the `mkdir`. The + stand-in `mkdir` plays that user, once. The name is refused, and nothing + is made through it or beneath it, nor is it re-moded.""" + sticky = short_state / "sticky" + sticky.mkdir() + sticky.chmod(0o1777) + state = sticky / "state" + target = tmp_path / "somewhere-else" + target.mkdir(mode=0o755) + target.chmod(0o755) + real_mkdir, real_fstat = os.mkdir, os.fstat + planted: dict = {} + + def _mkdir(path, mode=0o777, *, dir_fd=None): + if dir_fd is not None and path == state.name and not planted: + if shape == "link": + os.symlink(target, path, dir_fd=dir_fd) + else: + real_mkdir(path, 0o755, dir_fd=dir_fd) + os.chmod(state, 0o755) + planted["inode"] = os.lstat(state).st_ino + raise FileExistsError(path) + return real_mkdir(path, mode, dir_fd=dir_fd) + + def _fstat(descriptor): + info = real_fstat(descriptor) + if info.st_ino == planted.get("inode"): + fields = list(info) + fields[4] = os.getuid() + 4242 # st_uid + return os.stat_result(fields) + return info + + monkeypatch.setattr(bundle_mod.os, "mkdir", _mkdir) + monkeypatch.setattr(bundle_mod.os, "fstat", _fstat) + server = _prepared(monkeypatch, tmp_path, state) + with pytest.raises(bundle_mod.BundleRefused) as caught: + server.start() + assert planted, "the stand-in never ran: the state directory was not made by descriptor" + message = str(caught.value) + assert str(state) in message, message + assert ("is a symbolic link" if shape == "link" + else "is owned by uid") in message, message + beneath = target if shape == "link" else state + assert not (beneath / "postgres").exists(), "made beneath the planted name" + assert stat.S_IMODE(os.stat(beneath).st_mode) == 0o755, "the planted name was re-moded" + + +@pytest.mark.parametrize("shape", ["link", "broken-link", "open"]) +def test_a_data_directory_that_is_not_this_installs_own_is_refused( + monkeypatch, tmp_path: Path, short_state: Path, shape: str) -> None: + """An existing `data` joins the tree check (Copilot review of #69): as a + link to a cluster elsewhere, as a broken link, or as a directory others + can write. It is refused before anything is written into it or launched + on it.""" + elsewhere = tmp_path / "cluster-elsewhere" + (short_state / "postgres").mkdir(mode=0o700) + data = short_state / "postgres" / "data" + if shape in {"link", "open"}: + target = elsewhere if shape == "link" else data + target.mkdir(mode=0o700) + (target / "PG_VERSION").write_text("16\n", encoding="utf-8") + if shape in {"link", "broken-link"}: + data.symlink_to(elsewhere) + if shape == "open": + data.chmod(0o777) + server = _prepared(monkeypatch, tmp_path, short_state) + with pytest.raises(bundle_mod.BundleRefused) as caught: + server.start() + message = str(caught.value) + assert str(data) in message, message + assert ("symbolic link" if shape != "open" else "writable by every user") \ + in message, message + if shape == "link": + assert not (elsewhere / "pg_hba.conf").exists(), "wrote into the link's target" + + +@pytest.mark.parametrize("variable", [STATE, "XDG_STATE_HOME"]) +def test_parent_traversal_in_the_state_path_is_refused(variable: str) -> None: + value = "/tmp/odx-a/../odx-b" + with pytest.raises(config.ConfigurationError) as caught: + config.state_dir({variable: value}) + assert "`..`" in str(caught.value), caught.value + + +# -- the two refusal classes, each with its own reason -------------------------- + + +@pytest.mark.parametrize("setting, own, other", [ + ("RUNTIME_PG_ROLE", config.BUNDLE_SERVED_ROLE, "pg_read_all_data"), + ("SERVED_DATABASE", config.BUNDLE_DATABASE, "postgres"), +]) +@pytest.mark.parametrize("loader", ["load_settings", "load_migration_settings"]) +def test_a_served_identity_other_than_the_bundles_is_refused( + setting: str, own: str, other: str, loader: str) -> None: + """`OPENDOX_RUNTIME_PG_ROLE` names the role the migration run narrows on + the ledger, and a local install's served DSN connects as the bundle's own + (Copilot review of #69). Another role, `pg_read_all_data` say, left + `opendox_runtime` able to rewrite `opendox_schema_migrations`. So under + `local` each of the two identity settings may name only the bundle's + own, and the bundle's own name is accepted, as before.""" + name = config.PREFIX + setting + load = getattr(config, loader) + base = {MODE: "local", STATE: "/tmp/odx-identity"} + with pytest.raises(config.ConfigurationError) as caught: + load({**base, name: other}) + assert name in str(caught.value) and repr(own) in str(caught.value), caught.value + settings = load({**base, name: own}) + assert (settings.runtime_pg_role if setting == "RUNTIME_PG_ROLE" + else settings.served_database) == own + + +def test_a_dsn_beside_local_is_refused_for_the_database_not_a_broker() -> None: + with pytest.raises(config.ConfigurationError) as caught: + config.load_settings({MODE: "local", + PREFIX + "DATABASE_URL": "postgresql://s@h.invalid/x"}) + message = str(caught.value) + assert PREFIX + "DATABASE_URL" in message and "bundles" in message, message + assert "broker" not in message and "authentication" not in message, message + + +def test_a_broker_setting_beside_local_is_refused_for_the_broker() -> None: + with pytest.raises(config.ConfigurationError) as caught: + config.load_settings({MODE: "local", + PREFIX + "OIDC_AUDIENCE": "fixture"}) + message = str(caught.value) + assert PREFIX + "OIDC_AUDIENCE" in message, message + assert "broker" in message and "authentication" in message, message + assert "bundles" not in message, message + + +def test_both_classes_together_name_both_reasons() -> None: + with pytest.raises(config.ConfigurationError) as caught: + config.load_settings({ + MODE: "local", PREFIX + "OIDC_ISSUER": "https://i.invalid/r/x", + PREFIX + "MIGRATION_DATABASE_URL": "postgresql://m:hunter2@h.invalid/x"}) + message = str(caught.value) + for fragment in (PREFIX + "OIDC_ISSUER", PREFIX + "MIGRATION_DATABASE_URL", + "authentication", "bundles"): + assert fragment in message, (fragment, message) + assert "hunter2" not in message + + +@pytest.mark.parametrize("found", ["absent", "no-binaries", "outside-it", + "dir-outside-it", "not-executable"]) +def test_a_missing_server_package_is_the_named_refusal( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path, found: str) -> None: + """No `pixeltable-pgserver` distribution at all; one whose file list + names no binaries; or one whose listed binaries resolve outside it, + file by file or through a linked `bin` directory; or binaries that are + listed and inside it but cannot be executed. + Each is the one refusal naming the `local` extra, never a traceback, + and nothing is taken from anywhere but the distribution's own files + (Copilot review of #69).""" + root = tmp_path / "site" + bin_dir = root / bundle_mod.SERVER_PACKAGE / "pginstall" / "bin" + bin_dir.mkdir(parents=True) + elsewhere = tmp_path / "elsewhere" + elsewhere.mkdir() + for name in ("initdb", "postgres"): + for directory in (bin_dir, elsewhere): + (directory / name).write_text("#!/bin/sh\nexit 0\n", encoding="utf-8") + (directory / name).chmod(0o755) + if found == "outside-it": + for name in ("initdb", "postgres"): + (bin_dir / name).unlink() + (bin_dir / name).symlink_to(elsewhere / name) + elif found == "dir-outside-it": + shutil.rmtree(bin_dir) + bin_dir.symlink_to(elsewhere) + elif found == "not-executable": + for name in ("initdb", "postgres"): + (bin_dir / name).chmod(0o644) + + class _Distribution: + files = ([] if found == "no-binaries" else + [f"{bundle_mod.SERVER_PACKAGE}/pginstall/bin/{name}" + for name in ("initdb", "postgres")]) + + def locate_file(self, entry): + return root / str(entry) + + monkeypatch.setattr(bundle_mod.metadata, "distributions", + lambda **kwargs: [] if found == "absent" else [_Distribution()]) + with pytest.raises(bundle_mod.BundleRefused) as caught: + bundle_mod.server_binaries() + message = str(caught.value) + assert ('opendox[local]' if found == "absent" else "reinstall the `local` extra") \ + in message, message + + +def test_the_server_is_never_taken_from_the_working_directory( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + """`python -m opendox.cli` puts the working directory first on + `sys.path`, and a corpus checkout is where it runs (Copilot review of + #69). A checkout holding an importable `pixeltable_pgserver` with + executable binaries, AND a forged `.dist-info` naming them, is still not + where the server comes from: the search path leaves the working + directory out, as `''` and as its own path.""" + shadow = tmp_path / "checkout" + package = shadow / bundle_mod.SERVER_PACKAGE + bin_dir = package / "pginstall" / "bin" + bin_dir.mkdir(parents=True) + (package / "__init__.py").write_text("", encoding="utf-8") + for name in ("initdb", "postgres"): + (bin_dir / name).write_text("#!/bin/sh\necho planted\n", encoding="utf-8") + (bin_dir / name).chmod(0o755) + info = shadow / "pixeltable_pgserver-99.0.dist-info" + info.mkdir() + (info / "METADATA").write_text( + "Metadata-Version: 2.1\nName: pixeltable-pgserver\nVersion: 99.0\n", + encoding="utf-8") + (info / "RECORD").write_text("".join( + f"{bundle_mod.SERVER_PACKAGE}/pginstall/bin/{name},,\n" + for name in ("initdb", "postgres")), encoding="utf-8") + monkeypatch.chdir(shadow) + monkeypatch.syspath_prepend(str(shadow)) + monkeypatch.setattr(bundle_mod.sys, "path", ["", *bundle_mod.sys.path]) + assert str(shadow) not in bundle_mod._distribution_search_path() + assert "" not in bundle_mod._distribution_search_path() + try: + found = bundle_mod.server_binaries() + except bundle_mod.BundleRefused: + return # no carrier installed: nothing taken either + assert not found.resolve().is_relative_to(shadow.resolve()), found + + +# -- peer authentication, hermetic ---------------------------------------------- + + +def test_the_authentication_files_admit_one_os_user_as_the_two_roles() -> None: + files = bundle_mod.authentication_files("alice") + active = {name: [line.split() for line in text.splitlines() + if line.strip() and not line.startswith("#")] + for name, text in files.items()} + assert active["pg_hba.conf"] == [ + ["local", "all", "all", "peer", f"map={bundle_mod.IDENT_MAP}"], + ["host", "all", "all", "0.0.0.0/0", "reject"], + ["host", "all", "all", "::/0", "reject"]] + assert active["pg_ident.conf"] == [ + [bundle_mod.IDENT_MAP, '"alice"', config.BUNDLE_OWNER_ROLE], + [bundle_mod.IDENT_MAP, '"alice"', config.BUNDLE_SERVED_ROLE]] + assert "trust" not in " ".join(" ".join(r) for rows in active.values() for r in rows) + # A BACKSLASH IS WRITTEN AS IT IS, never escaped: PostgreSQL 16 reads a + # quoted field's backslash literally (`test_bundled_postgres.py` asks + # the server's own reading; Copilot review of #69). + domain = bundle_mod.authentication_files("DOMAIN\\alice")["pg_ident.conf"] + assert f'{bundle_mod.IDENT_MAP} "DOMAIN\\alice" {config.BUNDLE_OWNER_ROLE}' \ + in domain.splitlines(), domain + + +def test_the_files_are_written_0600_and_replace_what_was_there( + tmp_path: Path) -> None: + (tmp_path / "pg_hba.conf").write_text("local all all trust\n") + bundle_mod.write_authentication(tmp_path, "alice") + for name, content in bundle_mod.authentication_files("alice").items(): + assert (tmp_path / name).read_text(encoding="utf-8") == content + assert stat.S_IMODE((tmp_path / name).stat().st_mode) == 0o600 + assert sorted(p.name for p in tmp_path.iterdir()) == ["pg_hba.conf", "pg_ident.conf"] + + +@pytest.mark.parametrize("shape", ["umask", "stale-0644", "stale-link"]) +def test_the_files_are_exactly_0600_whatever_the_umask_or_a_stale_temporary( + tmp_path: Path, shape: str) -> None: + """The 0600 is SET, not asked for (Copilot review of #69). A umask that + takes the owner's write bit would leave a 0400 file. A temporary file an + interrupted start left at the same name would keep its own mode, 0644, + and a link left there would be written through to its target.""" + data = tmp_path / "data" + data.mkdir(mode=0o700) + outside = tmp_path / "outside.conf" + outside.write_text("untouched\n", encoding="utf-8") + outside.chmod(0o644) + for name in bundle_mod.authentication_files("alice"): + stale = data / f".{name}.opendox-{os.getpid()}" + if shape == "stale-0644": + stale.write_text("local all all trust\n", encoding="utf-8") + stale.chmod(0o644) + elif shape == "stale-link": + stale.symlink_to(outside) + previous = os.umask(0o277 if shape == "umask" else 0o022) + try: + bundle_mod.write_authentication(data, "alice") + finally: + os.umask(previous) + for name, content in bundle_mod.authentication_files("alice").items(): + written = data / name + assert not written.is_symlink(), name + assert written.read_text(encoding="utf-8") == content, name + assert stat.S_IMODE(written.stat().st_mode) == 0o600, name + assert sorted(p.name for p in data.iterdir()) == ["pg_hba.conf", "pg_ident.conf"] + assert outside.read_text(encoding="utf-8") == "untouched\n" + assert stat.S_IMODE(outside.stat().st_mode) == 0o644, "a stale link's target was re-moded" + + +def test_a_link_put_at_the_temporary_name_after_its_removal_is_never_followed( + monkeypatch, tmp_path: Path) -> None: + """The temporary file is created EXCLUSIVELY and without following a + link, so a link that appears at its name between the removal and the + open is a refusal, never a file written through (Copilot review of + #69). The stand-in `unlink` plays that race once.""" + data = tmp_path / "data" + data.mkdir(mode=0o700) + outside = tmp_path / "outside.conf" + outside.write_text("untouched\n", encoding="utf-8") + outside.chmod(0o644) + real_unlink = os.unlink + raced: list = [] + + def _unlink(path, *args, **kwargs): + try: + real_unlink(path, *args, **kwargs) + finally: + if not raced and Path(path).name.startswith(".pg_hba.conf.opendox-"): + raced.append(path) + os.symlink(outside, path) + + monkeypatch.setattr(bundle_mod.os, "unlink", _unlink) + with pytest.raises(FileExistsError): + bundle_mod.write_authentication(data, "alice") + assert raced, "the stand-in never ran" + assert outside.read_text(encoding="utf-8") == "untouched\n" + assert stat.S_IMODE(outside.stat().st_mode) == 0o644, "the link's target was re-moded" + assert not (data / "pg_hba.conf").exists() + + +@pytest.mark.parametrize("name", ["/regex", 'quo"te', "has space", "hash#tag", ""]) +def test_an_os_user_name_the_map_cannot_hold_plainly_is_refused( + monkeypatch: pytest.MonkeyPatch, name: str) -> None: + import pwd + + class _Entry: + pw_name = name + + monkeypatch.setattr(pwd, "getpwuid", lambda uid: _Entry()) + with pytest.raises(bundle_mod.BundleRefused): + bundle_mod.os_user() + + +def test_a_uid_with_no_password_entry_is_refused_by_name( + monkeypatch: pytest.MonkeyPatch) -> None: + import pwd + + def _missing(uid): + raise KeyError(uid) + + monkeypatch.setattr(pwd, "getpwuid", _missing) + with pytest.raises(bundle_mod.BundleRefused) as caught: + bundle_mod.os_user() + assert "password database" in str(caught.value) + + +def test_initdb_is_asked_for_peer_and_host_reject( + monkeypatch, tmp_path: Path, short_state: Path) -> None: + """The flags `initdb` receives, recorded by a stand-in: local is `peer`, + host is `reject`, and `trust` is not asked for anywhere.""" + record = tmp_path / "initdb-argv" + server = _server(monkeypatch, tmp_path, short_state, initdb=( + f'echo "$@" > "{record}"\nexit 1')) + with pytest.raises(bundle_mod.BundleRefused): + server.start() + argv = record.read_text(encoding="utf-8").split() + assert "--auth-local=peer" in argv and "--auth-host=reject" in argv, argv + assert not [a for a in argv if "trust" in a], argv + + +# -- initdb, and every phase of a start --------------------------------------- + + +def _binaries(tmp_path: Path, *, initdb: str, postgres: str = "exit 0") -> Path: + """Stand-in `initdb` and `postgres` scripts, executable.""" + directory = tmp_path / "fake-bin" + directory.mkdir() + for name, body in (("initdb", initdb), ("postgres", postgres)): + script = directory / name + script.write_text(body if body.startswith("#!") else + f"#!/bin/sh\n{body}\n", encoding="utf-8") + script.chmod(0o755) + return directory + + +def _server(monkeypatch, tmp_path: Path, state: Path, **scripts) -> bundle_mod.BundledServer: + binaries = _binaries(tmp_path, **scripts) + monkeypatch.setattr(bundle_mod, "server_binaries", lambda: binaries) + return bundle_mod.BundledServer(_local(state)) + + +def _target_of_initdb() -> str: + """Shell: the directory `initdb -D ` was told to initialize.""" + return 'while [ "$1" != "-D" ]; do shift; done; T="$2"' + + +@pytest.mark.parametrize("says", ["17.2", "16.14", ""]) +def test_an_existing_cluster_is_opened_only_by_its_own_major( + monkeypatch, tmp_path: Path, short_state: Path, says: str) -> None: + """The server's `postgres --version` against the cluster's `PG_VERSION` + (adversarial review of #69). Another major, or a server that does not + say, is the named refusal, before anything is written into the data + directory. The same major goes on to the launch, which the stand-in + fails on purpose.""" + data = short_state / "postgres" / "data" + data.mkdir(parents=True, mode=0o700) + (short_state / "postgres").chmod(0o700) + (data / "PG_VERSION").write_text("16\n", encoding="utf-8") + answer = f'echo "postgres (PostgreSQL) {says}"' if says else "true" + server = _server(monkeypatch, tmp_path, short_state, initdb="exit 1", + postgres=f'if [ "$1" = --version ]; then {answer}; exit 0; fi\nexit 3') + with pytest.raises(bundle_mod.BundleRefused) as caught: + server.start() + message = str(caught.value) + if says == "17.2": + assert "PostgreSQL 16 cluster" in message and "PostgreSQL 17:" in message, message + elif says == "": + assert "does not say which PostgreSQL it is" in message, message + else: + assert "exited during start (exit 3)" in message, message + if says != "16.14": + assert not (data / "pg_hba.conf").exists(), "wrote into another major's cluster" + + +def _a_tree(state: Path) -> config.DatabaseBundle: + """A valid bundle tree under `state`, every directory 0700, no server.""" + bundle = config.DatabaseBundle(state) + for directory in (state / "postgres", bundle.data_dir, bundle.socket_dir): + directory.mkdir(mode=0o700, exist_ok=True) + directory.chmod(0o700) + return bundle + + +@pytest.mark.parametrize("shape", ["no-lock", "dead-pid", "not-postgres", + "no-proc-other-socket", "no-proc-this-socket", + "open-state"]) +def test_a_local_verb_connects_only_behind_a_verified_server( + monkeypatch, short_state: Path, shape: str) -> None: + """`bundle.refusal_before_connecting` (adversarial review of #69, L2): + the tree a start checks, then a live server of THIS data directory, + named by its own lock file, listening at THIS socket directory. Where + `/proc` cannot say what the pid is, the lock file's socket line still + binds the socket to the tree.""" + bundle = _a_tree(short_state) + live = os.getpid() # alive, and not a postgres + dead = 2 ** 22 + 17 # above the default pid_max + lock = bundle.data_dir / "postmaster.pid" + socket_line = (str(bundle.socket_dir) if shape != "no-proc-other-socket" + else "/tmp/somewhere-else") + if shape != "no-lock": + pid = dead if shape == "dead-pid" else live + lock.write_text(f"{pid}\n{bundle.data_dir}\n1\n5432\n{socket_line}\n\n", + encoding="utf-8") + if shape.startswith("no-proc"): + monkeypatch.setattr(bundle_mod, "PROC", short_state / "no-proc") + if shape == "open-state": + short_state.chmod(0o777) + reason = bundle_mod.refusal_before_connecting(bundle) + expected = {"no-lock": "no readable postmaster.pid", + "dead-pid": "not a live process", + "not-postgres": "is not the postgres serving", + "no-proc-other-socket": "listens at /tmp/somewhere-else", + "no-proc-this-socket": None, + "open-state": "writable by every user"}[shape] + if expected is None: + assert reason is None, reason + else: + assert reason is not None and expected in reason, reason + + +@pytest.mark.parametrize("lacking", ["O_NOFOLLOW", "getuid", "AF_UNIX", "dir_fd"]) +def test_a_platform_without_the_posix_primitives_is_the_named_refusal( + monkeypatch, tmp_path: Path, short_state: Path, lacking: str) -> None: + """The bundle rests on POSIX primitives, and the carrier ships Windows + wheels too (Copilot review of #69). Lacking any one, a start and the + local verbs' socket check both name the gap, never an `AttributeError`.""" + import socket as socket_mod + + if lacking == "AF_UNIX": + monkeypatch.delattr(socket_mod, "AF_UNIX") + elif lacking == "dir_fd": + monkeypatch.setattr(bundle_mod, "MKDIR_TAKES_DIR_FD", False) + else: + monkeypatch.delattr(bundle_mod.os, lacking) + expected = {"dir_fd": "mkdir with dir_fd", "AF_UNIX": "socket.AF_UNIX"}.get( + lacking, f"os.{lacking}") + server = _prepared(monkeypatch, tmp_path, short_state) + with pytest.raises(bundle_mod.BundleRefused) as caught: + server.start() + assert "needs a POSIX platform" in str(caught.value), caught.value + assert expected in str(caught.value), caught.value + assert not (short_state / "postgres").exists(), "made directories first" + reason = bundle_mod.refusal_before_connecting(config.DatabaseBundle(short_state)) + assert reason is not None and expected in reason, reason + + +def test_a_symlink_loop_in_the_state_path_is_a_reason_not_a_traceback( + tmp_path: Path, short_state: Path) -> None: + """`Path.resolve()` raises for a symbolic-link loop (`RuntimeError` on + Python 3.12), and the local verbs' socket check must still answer with + a reason (Copilot review of #69).""" + loop = short_state / "loop" + loop.symlink_to(loop) + reason = bundle_mod.refusal_before_connecting( + config.DatabaseBundle(loop / "state")) + assert reason is not None, reason + assert "could not be judged" in reason or "symbolic link" in reason, reason + + +def test_status_reports_no_pid_behind_a_tree_it_refuses( + monkeypatch, short_state: Path) -> None: + """A pid is reported only behind a verified tree (Copilot review of + #69): `running_pid` alone would answer for whatever `postgres/data` is, + another live bundle's included.""" + bundle = _a_tree(short_state) + monkeypatch.setattr(bundle_mod, "running_pid", lambda b: 4242) + monkeypatch.setattr(bundle_mod, "refusal_before_connecting", + lambda b: "refused, for the case") + assert bundle_mod.report(bundle)["pid"] is None + monkeypatch.setattr(bundle_mod, "refusal_before_connecting", lambda b: None) + assert bundle_mod.report(bundle)["pid"] == 4242 + + +def test_an_initdb_that_dies_midway_leaves_no_data_directory( + monkeypatch, tmp_path: Path, short_state: Path) -> None: + """The half-built cluster: `PG_VERSION` written, then the run fails. The + data directory must not exist afterwards, so the next start initializes + again instead of launching the remains (Copilot review of #69).""" + server = _server(monkeypatch, tmp_path, short_state, initdb=( + f'{_target_of_initdb()}\necho 16 > "$T/PG_VERSION"\n' + 'mkdir -p "$T/base"\necho "initdb: killed" >&2\nexit 1')) + with pytest.raises(bundle_mod.BundleRefused) as caught: + server.start() + assert "initdb" in str(caught.value) + data = server.bundle.data_dir + assert not data.exists(), sorted(p.name for p in data.iterdir()) + leftovers = list(data.parent.glob(f"{bundle_mod.BundledServer.ATTEMPT_PREFIX}*")) + assert leftovers == [], leftovers + + +def test_an_abandoned_attempt_is_removed_and_a_live_ones_is_kept( + monkeypatch, tmp_path: Path, short_state: Path) -> None: + server = _server(monkeypatch, tmp_path, short_state, initdb="exit 1") + parent = server.bundle.data_dir.parent + parent.mkdir(parents=True) + dead = subprocess.Popen(["true"]) + dead.wait(timeout=10) + abandoned = parent / f"{server.ATTEMPT_PREFIX}{dead.pid}-x" + live = parent / f"{server.ATTEMPT_PREFIX}{os.getpid()}-y" + for attempt in (abandoned, live): + attempt.mkdir() + (attempt / "PG_VERSION").write_text("16\n", encoding="utf-8") + with pytest.raises(bundle_mod.BundleRefused): + server.start() + assert not abandoned.exists() + assert live.exists(), "a concurrent start's attempt was removed" + + +def test_a_data_directory_that_is_not_a_cluster_is_refused_and_left_alone( + monkeypatch, tmp_path: Path, short_state: Path) -> None: + server = _server(monkeypatch, tmp_path, short_state, initdb="exit 0") + data = server.bundle.data_dir + data.mkdir(parents=True) + (data / "somebody-elses-file").write_text("keep me\n", encoding="utf-8") + with pytest.raises(bundle_mod.BundleRefused) as caught: + server.start() + assert str(data) in str(caught.value) and STATE in str(caught.value) + assert (data / "somebody-elses-file").read_text(encoding="utf-8") == "keep me\n" + + +def test_an_initdb_that_hangs_is_the_named_refusal_not_a_traceback( + monkeypatch, tmp_path: Path, short_state: Path) -> None: + monkeypatch.setattr(bundle_mod, "START_TIMEOUT_SECONDS", 0.5) + server = _server(monkeypatch, tmp_path, short_state, initdb="exec sleep 30") + with pytest.raises(bundle_mod.BundleRefused) as caught: + server.start() + assert "initializing its data directory: TimeoutExpired" in str(caught.value) + assert not server.bundle.data_dir.exists() + + +def test_a_launch_that_cannot_exec_is_the_named_refusal( + monkeypatch, tmp_path: Path, short_state: Path) -> None: + server = _server(monkeypatch, tmp_path, short_state, + initdb=f'{_target_of_initdb()}\necho 16 > "$T/PG_VERSION"', + postgres="#!/nonexistent/interpreter\n") + with pytest.raises(bundle_mod.BundleRefused) as caught: + server.start() + assert "launching it: FileNotFoundError" in str(caught.value), caught.value + assert server.process is None + + +@pytest.mark.skipif(not sys.platform.startswith("linux"), + reason="PR_SET_PDEATHSIG is Linux's") +def test_a_parent_death_signal_that_cannot_be_armed_starts_no_server( + monkeypatch, tmp_path: Path, short_state: Path) -> None: + """`ctypes` reports a failed `prctl` by returning `-1` (a seccomp denial, + say), never by raising, and a server started anyway would outlive an + entry point killed outright (Copilot review of #69). The stand-in C + library's `prctl` fails; the launch is the named refusal, and no server + process exists.""" + launched = tmp_path / "launched" + + class _Libc: + @staticmethod + def prctl(*args): + return -1 + + monkeypatch.setattr(bundle_mod.ctypes, "CDLL", lambda *a, **k: _Libc()) + server = _server(monkeypatch, tmp_path, short_state, + initdb=f'{_target_of_initdb()}\necho 16 > "$T/PG_VERSION"', + postgres=f'touch "{launched}"\nsleep 30') + with pytest.raises(bundle_mod.BundleRefused) as caught: + server.start() + assert "parent-death signal" in str(caught.value), caught.value + assert server.process is None + assert not launched.exists(), "the server ran without its parent-death signal" + + +def test_directories_it_cannot_make_are_the_named_refusal( + monkeypatch, tmp_path: Path, short_state: Path) -> None: + if hasattr(os, "geteuid") and os.geteuid() == 0: # pragma: no cover + pytest.skip("root ignores directory modes") + locked = short_state / "locked" + locked.mkdir(mode=0o500) + binaries = _binaries(tmp_path, initdb="exit 0") + monkeypatch.setattr(bundle_mod, "server_binaries", lambda: binaries) + server = bundle_mod.BundledServer(_local(locked / "state")) + with pytest.raises(bundle_mod.BundleRefused) as caught: + server.start() + assert "preparing its directories: PermissionError" in str(caught.value) + + +# -- an interrupt anywhere in the local lifecycle ------------------------------- + + +@pytest.fixture() +def local_entry(scrubbed, monkeypatch, tmp_path: Path): + """`cmd_generate_and_open --local` with its bundle stood in. The corpus + checks are not under test here and pass whatever the root.""" + scrubbed.setenv(MODE, "local") + scrubbed.setenv(STATE, "/tmp/odx-never-created") + monkeypatch.setattr(cli_mod, "_refuse_non_corpus_repo_root", lambda args: None) + monkeypatch.setattr(cli_mod, "_refuse_malformed_generated_at", lambda args: None) + stopped: list[int] = [] + behaviour: dict = {} + + class _StandIn: + applied: list = [] + + def __init__(self, settings) -> None: + pass + + def start(self): + behaviour.get("start", lambda: None)() + return self + + def stop(self) -> None: + stopped.append(1) + + def report(self) -> dict: + return {"data_dir": None, "socket_dir": "(stood in)", "pid": None} + + monkeypatch.setattr(cli_mod.bundle_mod, "BundledServer", _StandIn) + args = cli_mod.build_parser().parse_args( + ["generate-and-open", config.LOCAL_FLAG, "--repo-root", str(tmp_path), + "--repository", "fixture", "--no-open", "--no-serve"]) + return args, behaviour, stopped + + +def _entry(args) -> int: + """The entry point, with an ESCAPING interrupt turned into a failure: one + that reached pytest would abort the whole session, not fail this case.""" + try: + return cli_mod.cmd_generate_and_open(args, opener=lambda url: None) + except KeyboardInterrupt as escaped: + raise AssertionError( + f"the interrupt escaped the entry point ({type(escaped).__name__}), " + "which is a traceback and not a clean stop") from None + + +def _sigterm_to_self() -> None: + os.kill(os.getpid(), signal.SIGTERM) + for _ in range(500): # the handler runs between two bytecodes + time.sleep(0.01) + raise AssertionError("SIGTERM was not delivered") # pragma: no cover + + +def test_a_sigterm_while_the_bundle_starts_is_a_clean_stop( + local_entry, capsys) -> None: + args, behaviour, stopped = local_entry + before = signal.getsignal(signal.SIGTERM) + behaviour["start"] = _sigterm_to_self + rc = _entry(args) + assert rc == 128 + signal.SIGTERM + assert "interrupted (SIGTERM)" in capsys.readouterr().err + assert stopped == [1], "the bundle was not stopped" + assert signal.getsignal(signal.SIGTERM) == before, "the handler was not restored" + + +def test_a_ctrl_c_while_the_snapshot_is_generated_is_a_clean_stop( + local_entry, monkeypatch, capsys) -> None: + args, _behaviour, stopped = local_entry + + def _interrupted(args, *, opener): + raise KeyboardInterrupt + + monkeypatch.setattr(cli_mod, "_generate_and_open", _interrupted) + rc = _entry(args) + assert rc == 128 + signal.SIGINT + assert "interrupted (SIGINT)" in capsys.readouterr().err + assert stopped == [1] diff --git a/tests_runtime/test_runtime_surface.py b/tests_runtime/test_runtime_surface.py index ff6a73a3..5de1373c 100644 --- a/tests_runtime/test_runtime_surface.py +++ b/tests_runtime/test_runtime_surface.py @@ -50,6 +50,11 @@ # one would make openDox the only destination the corpus could not test. "opendox.runtime.local_git_adapter", "opendox.runtime.repository_act", + # THE LOCAL INSTALL'S BUNDLED SERVER (plan 034 T072). `opendox.cli` + # imports it at module level, and `opendox.cli` is what `opendox --help` + # runs on an install with no extra at all, so it connects through + # `psycopg` only inside the functions that start the server. + "opendox.runtime.bundle", ) #: The modules that legitimately need the `runtime` extra, and the only ones.