T072, 13.1: the bundled PostgreSQL server, the local install's own child (plan 034)

constraints-cpython312-linux.txt

@@ -30,6 +30,15 @@
 # two installs, no second list of names to drift.
 #
 # Resolved 2026-09-18 on cpython 3.12.3 / linux x86_64.
+# Extended 2026-09-30 on cpython 3.12.3 / linux x86_64, under the pins above
+# (`-c` this file), for plan 034 T072's `local` extra and the `test` extra's
+# `setuptools`: pgserver, its psutil, platformdirs and fasteners, and
+# setuptools are new; no earlier pin moved.
+# Re-resolved 2026-09-30 on cpython 3.12.3 / linux x86_64, in a clean
+# environment under the pins above less `pgserver`, when the carrier became
+# `pixeltable-pgserver` (RULED, openxFactory#656 `5916000030` item 2): the one
+# line that moved is `pgserver==0.1.4` -> `pixeltable-pgserver==0.6.0`, whose
+# own requirements were all pinned already.
 PyJWT==2.14.0
 PyYAML==6.0.3
 Pygments==2.21.0
@@ -41,14 +50,18 @@ cffi==2.1.1
 click==8.5.0
 cryptography==50.0.1
 fastapi==0.141.1
+fasteners==0.20
 h11==0.16.0
 httpcore==1.0.9
 httptools==0.8.0
 httpx==0.28.1
 idna==3.20
 iniconfig==2.3.0
 packaging==26.3
+pixeltable-pgserver==0.6.0
+platformdirs==4.12.2
 pluggy==1.6.0
+psutil==7.2.2
 psycopg-binary==3.3.6
 psycopg-pool==3.3.2
 psycopg==3.3.6
@@ -57,6 +70,7 @@ pydantic==2.13.5
 pydantic_core==2.46.5
 pytest==8.4.2
 python-dotenv==1.2.3
+setuptools==84.0.0
 starlette==1.6.0
 typing-inspection==0.4.4
 typing_extensions==4.16.0

deploy/compose/.env.example

@@ -57,6 +57,13 @@ OPENDOX_MIGRATION_DATABASE_URL=postgresql://opendox:change-me-local-only@postgre
 # broker settings below, so it is never selected here.
 OPENDOX_INSTALL_MODE=hosted
 
+# Where a LOCAL install keeps its bundled PostgreSQL server's data directory and
+# Unix socket (plan 034 T072; #1144 13.1). This package is hosted and never
+# reads it; it is here because every setting the runtime reads is named here.
+# Empty means the per-user default, `$XDG_STATE_HOME/opendox` or
+# `~/.local/state/opendox`.
+OPENDOX_STATE_DIR=
+
 # REQUIRED. The Keycloak broker's issuer, pinned (RULING Q2): a token from any
 # other issuer is refused rather than trusted.
 OPENDOX_OIDC_ISSUER=https://keycloak.example/realms/opendox

pyproject.toml

@@ -93,9 +93,62 @@ dependencies = [
 # (`tests_runtime/conftest.py`), and a skipped case does not make a whole
 # suite. `opendox[runtime]` names the extra below instead of copying its five
 # lines, so the runtime's dependency list is still declared once.
+#
+# AND THE LOCAL EXTRA, the same way and for the same reason (plan 034 T072;
+# R1Q16 (iii), `5850003126`): the local install's bundled PostgreSQL server is
+# what `tests_runtime/test_bundled_postgres.py` starts, so F9.1's `.[test]`
+# install still runs every case. `opendox[local]` carries `opendox[runtime]`;
+# both are named so the T036 reason above stays readable on its own.
+#
+# AND `setuptools`, the backend `[build-system]` names: the same test BUILDS
+# this package's wheel and installs it outside the checkout, to prove a wheel
+# carries its migrations (plan 034 T072). Declared here so that build runs
+# offline, with `--no-build-isolation`, in the environment the suite runs in.
+# `>=70.1` and not `[build-system]`'s `>=68`: 70.1 is the first release that
+# builds a wheel with no separate `wheel` package installed.
 test = [
     "pytest>=8,<9",
     "opendox[runtime]",
+    "opendox[local]",
+    "setuptools>=70.1",
+]
+
+# THE LOCAL EXTRA — the standalone install (plan 034 T072; #1144 13.1, as T007
+# batch H's addendum reads; RULED R1Q16 (iii), openxFactory#656 comment
+# `5850003126`): `pip install "opendox[local]"`, then
+# `opendox generate-and-open --local …`. It carries the runtime's packages and
+# the bundled server's own, and nothing else.
+#
+# `pixeltable-pgserver` IS THE SERVER'S CARRIER (RULED, openxFactory#656
+# comment `5916000030` item 2, Brett Heap 2026-09-30: "pixeltable-pgserver
+# (Recommended)"), and only its binaries are used (see `opendox/runtime/
+# bundle.py` for why its own manager is not). It is the maintained fork of
+# `pgserver`, and it ships PostgreSQL 16 as `initdb` and `postgres` inside the
+# wheel, built to link only libc and libz, so a host needs no system
+# PostgreSQL and no ICU. Measured 2026-09-30 on its 0.6.0 cp312 manylinux
+# wheel:
+#   * PostgreSQL 16.14 (`pginstall/bin/postgres --version`); the wheel also
+#     carries an 18.4 under `pginstall18/`, which this package does not use;
+#   * wheels for cp310 to cp314, so every interpreter `requires-python`
+#     admits today has one; `pgserver` 0.1.4 stopped at cp312 and 16.2;
+#   * linux wheels tagged manylinux_2_27/2_28 (glibc 2.27 and later), on
+#     x86_64 and aarch64, plus macOS and Windows;
+#   * about 24.7 MB per wheel, because it carries the two server majors;
+#   * Apache-2.0 for the package; the PostgreSQL License for the server.
+# `>=0.6.0` is the release this package has been exercised against, the same
+# rule every floor in this file is set by. `<0.7` IS A CEILING, which no other
+# requirement here carries, because this one is a DATABASE SERVER whose major
+# is a property of the user's data: `bundle.py` runs the PostgreSQL 16 under
+# `pginstall/`, and a cluster is opened only by the major that made it.
+# Upstream's own default is already 18, so a later release may move what
+# `pginstall/` holds; the ceiling keeps that a deliberate act of this package
+# (adversarial review of openDox-code#69), and `bundle.py` still refuses, by
+# name, a server whose major disagrees with the cluster's `PG_VERSION`. Its
+# own requirements (fasteners, platformdirs, psutil, typing-extensions) arrive
+# with it and are imported by nothing here.
+local = [
+    "opendox[runtime]",
+    "pixeltable-pgserver>=0.6.0,<0.7",
 ]
 
 # THE RUNTIME EXTRA — `split-opendox-two-layer-product` § 3.5, RULED Q2
@@ -206,6 +259,20 @@ py-modules = ["route_extension", "subcommand_extension"]
 [tool.setuptools.packages.find]
 where = ["src"]
 
+# THE MIGRATIONS, SHIPPED IN THE WHEEL (plan 034 T072). `migrations/` stays at
+# the repository root, where the image copies it (`deploy/compose/Dockerfile`:
+# `COPY migrations ./migrations`, run from `/app`) and every checkout run finds
+# it; this maps the same files into the wheel's data directory, so an install
+# run OUTSIDE a checkout still has the migrations its bundled server applies.
+# `opendox.runtime.config.installation_migrations_dir` finds them through the
+# own record of the installed distribution that holds the running module
+# (after the source tree, for a checkout), and the canonical digest gate is
+# what proves the copy found is the pinned one. A data directory and not package
+# data because package data must live inside the package, and the one tree is
+# not moved.
+[tool.setuptools.data-files]
+"share/opendox/migrations" = ["migrations/*.sql"]
+
 # THE SERVED BUNDLE, PACKAGED — § 3.4 slice S5, and it is a gap this slice had
 # to close to make RULED Q5 true rather than true-in-a-source-checkout.
 #

src/opendox/cli.py

@@ -19,6 +19,7 @@
 import argparse
 import json
 import os
+import signal
 import sys
 import tempfile
 import webbrowser
@@ -88,6 +89,12 @@
 # or serves anything. Stdlib-only, like `local_git_adapter` above, which
 # already imports it, so this adds no reach and no import weight.
 from opendox.runtime import config as runtime_config  # noqa: E402
+# THE LOCAL INSTALL'S BUNDLED POSTGRESQL SERVER (plan 034 T072; #1144 13.1,
+# R1Q16 (i)-(iv)): started as THIS process's child by `generate-and-open
+# --local`, and stopped with it. Stdlib-only at import, like `runtime_config`;
+# the driver is imported when the server is started, never here.
+from opendox.runtime import bundle as bundle_mod  # noqa: E402
+from opendox.runtime import migrations as migrations_mod  # noqa: E402
 from opendox.boundary import (  # noqa: E402
     BoundaryViolation, HumanGate, OutputBoundary,
 )
@@ -558,8 +565,8 @@
 
 
 def _resolve_install_shape(args: argparse.Namespace,
-                           env=None) -> str:
-    """The install shape this run serves as, or `ConfigurationError` naming why.
+                           env=None) -> runtime_config.RuntimeSettings:
+    """The settings this run serves with, or `ConfigurationError` naming why.
 
     `--local` and `OPENDOX_INSTALL_MODE` are resolved by
     `runtime_config.install_mode`, the one reading of the selector, which
@@ -569,27 +576,45 @@
       * LOCAL binds loopback only, with no opt-in: a non-loopback `--host` is
         refused naming the rule (13.4), and so is anything a local install
         cannot be (`refuse_what_a_local_install_cannot_be`: a broker setting
-        beside it, or a non-loopback `OPENDOX_BIND_HOST`). It needs no broker.
-        Its datastore is 13.1's, and arrives with T072.
+        beside it, a DSN beside it, or a non-loopback `OPENDOX_BIND_HOST`). It
+        needs no broker, and it supplies BOTH DSNs itself, from the server it
+        bundles under `OPENDOX_STATE_DIR` (13.1; plan 034 T072).
       * HOSTED, set or by default, refuses with no issuer, NAMING THE ISSUER
-        (13.5), and then loads the runtime's whole configuration, because the
-        serving process is the one whose settings are the install's (13.4a;
-        R1Q16 (i)). Otherwise unchanged (13.6).
+        (13.5), and then loads the runtime's whole configuration.
+        Otherwise unchanged (13.6).
+
+    Either way the result is the runtime's own `load_settings`, because the
+    serving process is the one whose settings are the install's (13.4a;
+    R1Q16 (i)).
 
     Asked before anything is scanned, minted or bound, so a refused run leaves
     nothing behind and exits at once rather than starting a server that a
     bound would have to kill (F13.1's `test "$rc" -ne 124`).
     """
     env = os.environ if env is None else env
-    mode = runtime_config.install_mode(
-        env, local_flag=bool(getattr(args, "local", False)))
+    local_flag = bool(getattr(args, "local", False))
+    mode = runtime_config.install_mode(env, local_flag=local_flag)
     if mode == runtime_config.INSTALL_MODE_LOCAL:
         runtime_config.refuse_a_non_loopback_local_bind("--host", args.host)
         runtime_config.refuse_what_a_local_install_cannot_be(env)
     else:
         runtime_config.require_the_hosted_issuer(env)
-        runtime_config.load_settings(env)
-    return mode
+    return runtime_config.load_settings(env, local_flag=local_flag)
+
+
+class _Terminated(KeyboardInterrupt):
+    """SIGTERM, raised as the interrupt the serve loop already stops cleanly on.
+
+    A subclass, so the serve loop's own `except KeyboardInterrupt` still ends
+    a served run with 0 (F13.1's `kill "$SERVER"; wait "$SERVER"`). An
+    interrupt that arrives BEFORE the serve loop can still say which signal
+    it was.
+    """
+
+
+def _terminate_as_interrupt(signum, frame):
+    """SIGTERM, read as the Ctrl-C the serve loop already stops cleanly on."""
+    raise _Terminated
 
 
 def cmd_generate_and_open(args: argparse.Namespace, *, opener=webbrowser.open) -> int:
@@ -602,12 +627,85 @@
     `OPENDOX_INSTALL_MODE=local`, selects the local single-user install, and
     with neither the install is hosted — see `_resolve_install_shape`. A
     refusal there is printed on stderr and the command exits 1, before any
-    other work."""
+    other work.
+
+    A LOCAL RUN OWNS ITS DATABASE (plan 034 T072; #1144 13.1, R1Q16 (i)-(iv)).
+    Once the corpus root and the anchors are known good, the bundled
+    PostgreSQL server is started as THIS process's child, bootstrapped and
+    migrated — starting and migrating it is all release 1 asks of it — and it
+    is stopped when this command returns, however it returns: a served run
+    ended by Ctrl-C or by SIGTERM (read here as the same interrupt), a
+    `--no-serve` run, or a failure. A refused start exits 1 on stderr, like
+    every other refusal of this verb."""
     try:
-        args.install_mode = _resolve_install_shape(args)
+        settings = _resolve_install_shape(args)
     except runtime_config.ConfigurationError as exc:
         print(f"generate-and-open refused: {exc}", file=sys.stderr)
         return 1
+    args.install_mode = settings.install_mode
+    args.runtime_settings = settings
+    if settings.install_mode != runtime_config.INSTALL_MODE_LOCAL:
+        return _generate_and_open(args, opener=opener)
+    # THE CHEAP REFUSALS FIRST, so a mistyped root never costs a database
+    # start: every one `_generate_and_open` asks before it mints its run
+    # directory, main's empty-source-option refusal (T055) included.
+    _refuse_non_corpus_repo_root(args)
+    _refuse_malformed_generated_at(args)
+    _refuse_empty_source_options(args)
+    server = bundle_mod.BundledServer(settings)
+    args.database_bundle = server
+    # NO `PG*` DEFAULT REACHES THE BUNDLE'S CONNECTIONS while this process
+    # runs its database (Copilot review of openDox-code#69): see
+    # `bundle.isolated_from_libpq_environment`.
+    with bundle_mod.isolated_from_libpq_environment():
+        return _run_the_local_lifecycle(args, server, opener=opener)
+
+
+def _run_the_local_lifecycle(args: argparse.Namespace, server, *, opener) -> int:
+    """Start the bundled server, generate and serve, and stop it, however
+    this ends: a served run ended by Ctrl-C or SIGTERM, a `--no-serve` run, a
+    refusal, a failure, or an interrupt before anything was served."""
+    try:
+        previous = signal.signal(signal.SIGTERM, _terminate_as_interrupt)
+    except ValueError:                  # not the main thread: no handler to own
+        previous = None
+    try:
+        try:
+            server.start()
+        except (bundle_mod.BundleRefused, runtime_config.ConfigurationError,
+                migrations_mod.MigrationError) as exc:
+            print(f"generate-and-open refused: {exc}", file=sys.stderr)
+            return 1
+        report = server.report()
+        print(f"  database {report['socket_dir']} (bundled, pid {report['pid']}, "
+              f"migrations applied now: {server.applied or 'none pending'})")
+        return _generate_and_open(args, opener=opener)
+    except KeyboardInterrupt as interrupt:
+        # AN INTERRUPT ANYWHERE IN THE LOCAL LIFECYCLE IS A CLEAN STOP (Copilot
+        # review of openDox-code#69). SIGTERM, or Ctrl-C, can arrive while the
+        # server is initializing or migrating, or while the snapshot is being
+        # generated, all before the serve loop's own handler. It is a stop
+        # that was asked for, so it is not a traceback: the `finally` below
+        # stops the bundled server and restores the handler. Nothing was
+        # served, so the exit is the signal's conventional status (128 + its
+        # number) and not 0.
+        signum = (signal.SIGTERM if isinstance(interrupt, _Terminated)
+                  else signal.SIGINT)
+        print(f"generate-and-open interrupted ({signum.name}) before it "
+              "served; its bundled PostgreSQL server stops with it",
+              file=sys.stderr)
+        return 128 + int(signum)
+    finally:
+        # THE HANDLER FIRST, so a second SIGTERM during the stop takes the
+        # default action at once; the parent-death signal still stops the
+        # server if this process goes before `stop()` has finished.
+        if previous is not None:
+            signal.signal(signal.SIGTERM, previous)
+        server.stop()
+
+
+def _generate_and_open(args: argparse.Namespace, *, opener) -> int:
+    """`generate-and-open`'s generate-then-serve half, once the install is known."""
     # Ahead of minting the run dir, so a refused root leaves not even an empty
     # temp directory behind. `_generate_and_write` is still the guard that MATTERS
     # (it is the one no caller can skip); these are the same checks, earlier.