diff --git a/pyproject.toml b/pyproject.toml index c8794fb1..452f5203 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -297,14 +297,27 @@ where = ["src"] # the hosted GET fallback by a packaging omission rather than by a deployment # choice. # -# `web/**` and not a list: the bundle is 41 files across one subdirectory and +# `web/**` and not a list: the bundle is 42 files across two subdirectories and # grows by feature, and a hand-kept list would drift the first time a view is # added. `tests/test_web_boundary.py`'s census is what holds the DIRECTORY # honest — every file under `src/opendox/web/` carries a declared row with a # class, and an unrowed file fails immediately — so the glob here ships exactly # what that census counts rather than whatever happens to be on disk. +# +# AND `web/**/.*` BESIDE IT (plan 034 T075; #1144 10.2): "ships exactly what +# that census counts" was not true of a wheel. setuptools expands each pattern +# with the standard library's `glob`, and a `*` or `**` there never matches a +# name that starts with a dot, so `web/**` alone shipped 41 of the census's 42 +# files: `vendor/.gitkeep`, a census row, never reached an install, and the +# installed `opendox generate-and-open --local` answered it 404. A pattern +# whose last part starts with the dot is the one that matches such a name, at +# any depth under `web/`. Measured with both builds an install can run, pip's +# isolated build and `--no-build-isolation` over the `test` extra's +# setuptools: 41 files without it, all 42 with it. +# `tests_runtime/test_served_bundle.py` holds the wheel to the tree's +# `src/opendox/web/`, and fetches every file from the installed entry point. [tool.setuptools.package-data] -opendox = ["web/**"] +opendox = ["web/**", "web/**/.*"] # THE VALIDATOR'S PACKAGED COPIES (plan 034 T057; #1144 7.1, as T007's batch G # amends it; R1Q12 (a), openxFactory#656 comment 5850003126). openDox's own # validator reads its spec leg's four schemas from `opendox/contracts/`, and diff --git a/tests/test_gate_loop_contributed.py b/tests/test_gate_loop_contributed.py index 252cbf67..d8b9fc6e 100644 --- a/tests/test_gate_loop_contributed.py +++ b/tests/test_gate_loop_contributed.py @@ -1025,11 +1025,22 @@ def test_the_bundle_glob_and_the_setuptools_floor_are_one_guarantee() -> None: Measured on a real build at `setuptools 84.0.0`: 40 entries under `opendox/web/`, 36 of them under `opendox/web/views/` — the whole census except `vendor/.gitkeep`, which `glob` skips as a dotfile and which exists - only to keep an otherwise-empty directory in git.""" + only to keep an otherwise-empty directory in git. + + Those figures are S5's, at its tree. PLAN 034 T075 (#1144 10.2: the + entry point serves the 42-file bundle) re-measured them, at the same + `setuptools 84.0.0`: `web/**` alone gave 41 entries, 37 of them under + `opendox/web/views/`, every census file but `vendor/.gitkeep` again. T075 + put `web/**/.*` beside the recursive pattern, because the census counts + that dotfile and an installed entry point answered it 404; with both, the + wheel carries all 42. The recursive pattern is still first and still the + one this guarantee is about; the line is held verbatim as it now reads, + and `tests_runtime/test_served_bundle.py` holds the wheel to the tree.""" text = (ROOT / "pyproject.toml").read_text(encoding="utf-8") - assert 'opendox = ["web/**"]' in text, ( + assert 'opendox = ["web/**", "web/**/.*"]' in text, ( "the bundle ships by a RECURSIVE glob; a single `*` would package " - "app.js without `web/views/`") + "app.js without `web/views/`; and its dotfile pattern beside it ships " + "the census's `vendor/.gitkeep` (plan 034 T075)") floor = re.search(r'requires\s*=\s*\["setuptools>=(\d+)', text) assert floor, "the build system must declare a setuptools floor at all" assert int(floor.group(1)) >= 63, ( diff --git a/tests/test_validator_input_set.py b/tests/test_validator_input_set.py index 763bae43..3ba756db 100644 --- a/tests/test_validator_input_set.py +++ b/tests/test_validator_input_set.py @@ -414,15 +414,16 @@ def test_the_package_data_ships_the_record_and_every_copy() -> None: """7.1 settles that the copies travel as PACKAGE DATA, so an install has them beside the validator. The package-data table names, under `opendox.contracts`, exactly the record and every schema copy the record - pins, and the bundle's own line is unchanged. (A wheel built without it + pins, and the bundle's own line is unchanged by 7.1. (A wheel built without it carries `opendox/contracts/__init__.py` alone, and its validator refuses, - naming the absent record.)""" + naming the absent record.) The bundle's line is the one plan 034 T075 + left it as: the recursive pattern and the dotfile one beside it.""" import fnmatch import tomllib table = tomllib.loads((ROOT / "pyproject.toml").read_text(encoding="utf-8")) data = table["tool"]["setuptools"]["package-data"] - assert data["opendox"] == ["web/**"] + assert data["opendox"] == ["web/**", "web/**/.*"] patterns = data["opendox.contracts"] shipped = sorted( relative for relative in (p.relative_to(PACKAGE).as_posix() diff --git a/tests_runtime/test_served_bundle.py b/tests_runtime/test_served_bundle.py new file mode 100644 index 00000000..23fb4347 --- /dev/null +++ b/tests_runtime/test_served_bundle.py @@ -0,0 +1,596 @@ +"""The door: an openDox-only install serves the whole web bundle (plan 034 +T075; #1144 10.2 and 10.2a). + +T075's falsifier is F10.1's fetch, as T007 batch H amends it: a `.[local]` +install, and `opendox generate-and-open --local …`, which starts the bundled +server (13.1; R1Q15 (b), R1Q16 (iii), `5850003126`). F10.1 fetches `/` and +reads ` bool: + return os.environ.get("CI", "").strip().lower() in {"1", "true", "yes", "on"} + + +@pytest.fixture(scope="module", autouse=True) +def _the_server_is_installed() -> None: + """The `local` extra's binaries, or this module's refusal to pass silently.""" + try: + bundle_mod.server_binaries() + except bundle_mod.BundleRefused as exc: + if _in_ci(): + pytest.fail(f"CI is set, so the served-bundle suite must RUN: {exc}", + pytrace=False) + pytest.skip(str(exc)) + if hasattr(os, "geteuid") and os.geteuid() == 0: # pragma: no cover + pytest.fail("the bundled server refuses root; run the suite as a user") + + +def _tree_files() -> set[str]: + """Every file under the tree's `src/opendox/web/`, as a bundle path.""" + return {p.relative_to(WEB).as_posix() for p in WEB.rglob("*") + if p.is_file() and "__pycache__" not in p.parts} + + +@pytest.fixture(scope="module") +def wheel(tmp_path_factory) -> Path: + """This checkout's wheel, built offline from a copy of what it packages.""" + work = tmp_path_factory.mktemp("t075-wheel") + source = work / "source" + source.mkdir() + for name in ("pyproject.toml", "src", "migrations"): + item = ROOT / name + if item.is_dir(): + shutil.copytree(item, source / name, ignore=shutil.ignore_patterns( + "__pycache__", "*.egg-info")) + else: + shutil.copy2(item, source / name) + wheels = work / "wheels" + built = subprocess.run( + [sys.executable, "-m", "pip", "wheel", "--no-deps", "--no-index", + "--no-build-isolation", "-q", "-w", str(wheels), str(source)], + capture_output=True, text=True, timeout=300) + assert built.returncode == 0, built.stderr[-3000:] + (found,) = wheels.glob("opendox-*.whl") + return found + + +@pytest.fixture(scope="module") +def installed(wheel: Path, tmp_path_factory) -> tuple[Path, Path]: + """The wheel installed under a prefix of its own: `(console script, site)`. + + `--ignore-installed` is load-bearing, as T072's case records: without it + pip uninstalls the suite's own editable `opendox` first.""" + prefix = tmp_path_factory.mktemp("t075-prefix") + done = subprocess.run( + [sys.executable, "-m", "pip", "install", "--no-deps", "--no-index", + "--ignore-installed", "-q", "--prefix", str(prefix), str(wheel)], + capture_output=True, text=True, timeout=300) + assert done.returncode == 0, done.stderr[-3000:] + assert "uninstall" not in (done.stdout + done.stderr).lower() + site = Path(sysconfig.get_path("purelib", vars={"base": str(prefix), + "platbase": str(prefix)})) + script = Path(sysconfig.get_path("scripts", vars={"base": str(prefix), + "platbase": str(prefix)})) + return script / "opendox", site + + +def _child_env(site: Path, **extra: str) -> dict[str, str]: + env = {name: value for name, value in os.environ.items() + if not name.startswith(SCRUBBED)} + env["PYTHONPATH"] = str(site) + env.pop("PYTHONUNBUFFERED", None) + env.update(extra) + return env + + +def _fresh_repository(parent: Path) -> Path: + """#1144's preamble: T050's fixture in a FRESH repository, one commit, as + the fixture's own identity and no user or system configuration.""" + root = parent / PLAIN_DOCUMENTS.name + shutil.copytree(PLAIN_DOCUMENTS, root) + env = {k: v for k, v in os.environ.items() if not k.startswith("GIT_")} + env.update({ + "GIT_AUTHOR_NAME": "fixture", "GIT_AUTHOR_EMAIL": "fixture@example.invalid", + "GIT_COMMITTER_NAME": "fixture", + "GIT_COMMITTER_EMAIL": "fixture@example.invalid", + "GIT_CONFIG_GLOBAL": os.devnull, "GIT_CONFIG_SYSTEM": os.devnull, + }) + for args in (("-c", "init.defaultBranch=main", "init", "-q"), + ("add", "-A"), ("commit", "-qm", "fixture")): + subprocess.run(["git", "-C", str(root), *args], check=True, + capture_output=True, env=env) + return root + + +@pytest.fixture() +def state_dir(): + """A SHORT state directory (the socket's path is bounded by the kernel), + removed afterwards once any server on it is checked stopped. + + THE REMOVAL IS RETRIED, BOUNDED. A server killed on a red path can leave a + backend still writing its WAL for a moment after the postmaster has gone, + and a single `rmtree` that races it leaves a data directory behind under + `/tmp` (seen once, on a mutant run, as `postgres/data/pg_wal/…`).""" + base = "/tmp" if os.path.isdir("/tmp") else None + path = Path(tempfile.mkdtemp(prefix="odx-t075-", dir=base)) + yield path + pid = bundle_mod.running_pid(config.DatabaseBundle(path)) + if pid is not None: # pragma: no cover + os.kill(pid, signal.SIGKILL) + deadline = time.monotonic() + STOP_SECONDS + while True: + shutil.rmtree(path, ignore_errors=True) + if not path.exists() or time.monotonic() > deadline: + break + time.sleep(0.2) # pragma: no cover + assert not path.exists(), f"the state directory outlived its test: {path}" + + +def _served_url(child: subprocess.Popen, said: Path, seconds: float) -> str | None: + """The URL line the entry point prints once it serves, read from the FILE + its standard output goes to, or `None` once `seconds` pass or it exits. + + A file and not a pipe: nothing has to drain it while the server runs, and + the entry point flushes that line before it blocks (plan 034 T056), so a + server that serves has said where by the time the line is looked for.""" + deadline = time.monotonic() + seconds + while time.monotonic() < deadline: + text = said.read_text(encoding="utf-8", errors="replace") + found = [line for line in text.splitlines() if line.startswith("http://")] + if found: + return found[0].strip() + if child.poll() is not None: + return None + time.sleep(0.2) + return None + + +def _get(base: tuple[str, int], path: str) -> tuple[int, str, bytes]: + connection = http.client.HTTPConnection(*base, timeout=30) + try: + connection.request("GET", path) + response = connection.getresponse() + return (response.status, response.getheader("Content-Type") or "", + response.read()) + finally: + connection.close() + + +def _essence(content_type: str) -> str: + return content_type.split(";", 1)[0].strip().lower() + + +# What a browser fetches from the bundle on its own: the page's `src`/`href` +# attributes, and each module's STATIC imports. A static import that names a +# missing file fails the whole module graph; a dynamic `import()` is how the +# bundle reaches what may be absent (10.2a), so those are read separately, and +# walked on only where the bundle carries the target. Both are read from the +# module with its COMMENTS removed (`_without_comments`): a multi-line import +# clause carries comments of its own, and an apostrophe or a quotation mark in +# one ended the clause before its `from` (Copilot review of openDox-code#73: +# `views/doxbench-editor.js`'s import of `./doxbench-state.js` was missed). +_PAGE_REFERENCE = re.compile(r"""\b(?:src|href)\s*=\s*["']([^"']+)["']""") +_STATIC_IMPORT = re.compile( + r"""^[ \t]*(?:import|export)\b[^;'"`]*?\bfrom[ \t]*(["'])([^"'\n]+)\1""" + r"""|^[ \t]*import[ \t]*(["'])([^"'\n]+)\3""", + re.MULTILINE) +_DYNAMIC_IMPORT = re.compile(r"""\bimport\(\s*(["'])(\.{1,2}/[^"'\n]+)\1\s*\)""") + + +#: Where a `/` opens a regular-expression literal rather than dividing: after +#: one of these characters, or after one of these words (the usual reading). +_REGEX_AFTER = frozenset("(,=:[!&|?{};+-*%<>~^") +_REGEX_AFTER_WORDS = frozenset({"return", "typeof", "case", "do", "else", + "in", "of", "void", "yield", "await", "delete", + "instanceof", "new", "throw"}) + + +def _without_comments(source: str) -> str: + """`source` with every JavaScript comment replaced by blank space (its line + breaks kept), and every string, template and regular-expression literal + kept as it is, so a quote or slash inside one is never read as code.""" + out: list[str] = [] + i, n = 0, len(source) + previous, word, in_word = "", "", False + while i < n: + ch, two = source[i], source[i:i + 2] + if two in ("//", "/*"): + if two == "//": + end = source.find("\n", i) + end = n if end == -1 else end + else: + end = source.find("*/", i + 2) + end = n if end == -1 else end + 2 + out.append("\n" * source.count("\n", i, end) or " ") + i = end + continue + regex = ch == "/" and (previous in _REGEX_AFTER or previous == "" + or word in _REGEX_AFTER_WORDS) + if ch in "'\"`" or regex: + j, in_class = i + 1, False + while j < n and (source[j] != ch or in_class): + if source[j] == "\\": + j += 1 + elif regex and source[j] == "[": + in_class = True + elif regex and source[j] == "]": + in_class = False + elif regex and source[j] == "\n": + break + j += 1 + out.append(source[i:j + 1]) + previous, word, in_word, i = ch, "", False, j + 1 + continue + out.append(ch) + if ch.isalnum() or ch in "_$": + word = word + ch if in_word else ch + previous, in_word = ch, True + else: + in_word = False + if not ch.isspace(): + previous, word = ch, "" + i += 1 + return "".join(out) + + +def _imports(path: str, text: str) -> tuple[set[str], set[str]]: + """`(static, dynamic)`: the bundle paths `path`'s module names in its static + imports and in its relative dynamic `import()` calls, comments removed.""" + code = _without_comments(text) + static = {_resolve(path, match.group(2) or match.group(4)) + for match in _STATIC_IMPORT.finditer(code) + if (match.group(2) or match.group(4)).startswith((".", "/"))} + dynamic = {_resolve(path, match.group(2)) + for match in _DYNAMIC_IMPORT.finditer(code)} + return static, dynamic + + +def _local(reference: str) -> bool: + return not re.match(r"^(?:[a-z][a-z0-9+.-]*:|//|#)", reference, re.I) + + +def _resolve(importer: str, reference: str) -> str: + target = posixpath.normpath(posixpath.join(posixpath.dirname(importer), + reference.split("?", 1)[0])) + assert not target.startswith("../"), (importer, reference) + return target + + +def _module_graph(read, carried: set[str]) -> tuple[set[str], set[str], set]: + """`(reached, dynamic, edges)`: every bundle path a browser fetches from + `/`, every relative dynamic `import()` target any reached module names, + and every `(importer, imported)` pair the walk followed. + + The walk starts at the page's references and follows each module's static + imports. A dynamic target the bundle CARRIES is a module a browser can + load too, so it is walked like any other (Copilot review of + openDox-code#73: `views/repo-selector.js` loads `views/projection-index.js` + that way); one the bundle does not carry, 10.2a's, stays a leaf. + `read(path)` answers a path's text.""" + page = "index.html" + queue = [_resolve(page, ref) for ref in _PAGE_REFERENCE.findall(read(page)) + if _local(ref)] + reached, dynamic, edges = {page}, set(), set() + while queue: + path = queue.pop() + if path in reached: + continue + reached.add(path) + if not path.endswith(".js"): + continue + static, named = _imports(path, read(path)) + dynamic |= named + for target in static | (named & carried): + edges.add((path, target)) + queue.append(target) + return reached, dynamic, edges + + +# --------------------------------------------------------------------------- +# 1 — the wheel carries the whole bundle +# --------------------------------------------------------------------------- + +def test_the_wheel_carries_every_web_file_the_tree_holds(wheel: Path) -> None: + """10.2's 42 files, all of them in the wheel `pip install ".[local]"` + installs, and nothing under `opendox/web/` that the tree does not hold.""" + tree = _tree_files() + assert len(tree) == BUNDLE_FILES, sorted(tree) + assert NOT_OWED not in tree # 10.2a + with zipfile.ZipFile(wheel) as archive: + carried = {name[len("opendox/web/"):] for name in archive.namelist() + if name.startswith("opendox/web/") and not name.endswith("/")} + assert sorted(tree - carried) == [], "the tree holds files the wheel omits" + assert sorted(carried - tree) == [], "the wheel carries files the tree does not" + + +# --------------------------------------------------------------------------- +# 2 — F10.1's fetch, over every file, from the installed local entry point +# --------------------------------------------------------------------------- + +def test_F10_1_fetch_the_installed_local_entry_point_serves_every_bundle_file( + installed: tuple[Path, Path], state_dir: Path, tmp_path: Path) -> None: + """`opendox generate-and-open --local --no-open`, as installed, from a + directory that is not a checkout: `/` is the bundle's page, every one of + the 42 files answers with the tree's bytes, the browser's module graph + closes inside the bundle, and 10.2a's file is the one that is absent.""" + script, site = installed + elsewhere = tmp_path / "elsewhere" + elsewhere.mkdir() + repo = _fresh_repository(tmp_path) + env = _child_env(site, **{PREFIX + "STATE_DIR": str(state_dir)}) + + # F10.1's preconditions, in the environment the server runs in: no + # sibling importable, the console script exists, and the `opendox` it + # runs is the installed one, whose bundle is the wheel's copy. + probe = subprocess.run( + [sys.executable, "-c", + "import importlib.util, sys\n" + f"present = [n for n in {SIBLINGS!r} if importlib.util.find_spec(n)]\n" + "if present:\n" + " print('importable siblings:', present, file=sys.stderr)\n" + f" sys.exit({SIBLING_PRESENT})\n" + "from opendox import cli\n" + "print(cli.WEB_DIR.resolve())\n"], + cwd=elsewhere, env=env, capture_output=True, text=True, timeout=60) + # Named, so a red here says WHICH sibling, and an import that fails for + # another reason is not reported as one. + assert probe.returncode != SIBLING_PRESENT, ( + f"a sibling is importable: {probe.stderr[-2000:]}") + assert probe.returncode == 0, probe.stderr[-2000:] + served_dir = Path(probe.stdout.strip().splitlines()[-1]) + assert served_dir.is_relative_to(site.resolve()), served_dir + helped = subprocess.run([str(script), "--help"], cwd=elsewhere, env=env, + capture_output=True, text=True, timeout=60) + assert helped.returncode == 0, helped.stderr[-2000:] + + # Both streams go to FILES: the server logs every request on standard + # error, and an unread pipe that fills would stall it mid-fetch. + out, err = tmp_path / "entry-point.stdout", tmp_path / "entry-point.stderr" + with out.open("wb") as stdout, err.open("wb") as stderr: + child = subprocess.Popen( + [str(script), "generate-and-open", "--local", + "--repo-root", str(repo), "--repository", "fixture", + "--run-dir", str(tmp_path / "run"), "--no-open", "--port", "0"], + cwd=elsewhere, env=env, stdout=stdout, stderr=stderr) + + def said() -> str: + return err.read_text(encoding="utf-8", errors="replace")[-3000:] + + try: + url = _served_url(child, out, START_SECONDS) + if url is None: + child.kill() + child.wait(timeout=STOP_SECONDS) + raise AssertionError(f"the installed entry point never served: {said()}") + match = re.match(r"^http://([0-9.]+):([0-9]+)/index\.html$", url) + assert match, url + base = (match.group(1), int(match.group(2))) + + # F10.1's fetch, as written: `/` answers, and it is really the bundle. + status, _kind, body = _get(base, "/") + assert status == 200, status + assert " str: + status, _kind, body = _get(base, "/" + path) + assert status == 200, (path, status) + return body.decode("utf-8") + + tree = _tree_files() + reached, dynamic, _edges = _module_graph(read, tree) + assert reached - tree == set(), sorted(reached - tree) + assert {"app.js", "styles.css", "views/intent-binding.js", + "views/projection-index.js"} <= reached + + # 10.2a: the one module the bundle reaches for and does not carry is + # the declared one, reached only dynamically, and it is absent. + assert dynamic - tree == {NOT_OWED}, sorted(dynamic - tree) + status, _kind, _body = _get(base, "/" + NOT_OWED) + assert status == 404, status + + bundled_pid = bundle_mod.running_pid(config.DatabaseBundle(state_dir)) + assert bundled_pid is not None, "no bundled server is running" + child.send_signal(signal.SIGTERM) + assert child.wait(timeout=STOP_SECONDS) == 0, said() + finally: + if child.poll() is None: + child.kill() + child.wait(timeout=STOP_SECONDS) + assert bundle_mod.running_pid(config.DatabaseBundle(state_dir)) is None + + +# --------------------------------------------------------------------------- +# 3 — the walk's import reader, over the tree, against an independent reading +# --------------------------------------------------------------------------- + +#: A specifier a statement names at the end of a line of code: `from "./x"`, +#: or a bare `import "./x"`. Read line by line, skipping comment lines, so it +#: shares nothing with `_without_comments` and `_STATIC_IMPORT`. +_LINE_SPECIFIER = re.compile( + r"""(?:\bfrom|^[ \t]*import)[ \t]*(["'])(\.{1,2}/[^"'\n]+)\1""") + +#: The edge Copilot's review of openDox-code#73 found the first reader missed: +#: its import clause spans lines whose comments hold an apostrophe and quotes. +_COMMENTED_CLAUSE_EDGE = ("views/doxbench-editor.js", "views/doxbench-state.js") + + +def _line_specifiers(path: str, text: str) -> set[tuple[str, str]]: + found = set() + for line in text.splitlines(): + if line.strip().startswith(("//", "*", "/*")): + continue + found.update((path, _resolve(path, match.group(2))) + for match in _LINE_SPECIFIER.finditer(line)) + return found + + +def test_the_walk_reads_every_static_import_the_bundle_declares() -> None: + """The browser walk's reader finds every relative static import in the + tree's modules, the same set an independent line-by-line reading finds, + including the one a multi-line clause with commented quotes declares. A + mutant that points only that specifier at a missing module is caught by + the walk, so case 2's closure assertion would refuse it.""" + tree = _tree_files() + read = {path: (WEB / path).read_text(encoding="utf-8") + for path in tree if path.endswith(".js")} + declared = set().union(*(_line_specifiers(p, t) for p, t in read.items())) + found = {(path, target) for path, text in read.items() + for target in _imports(path, text)[0]} + assert found == declared, (sorted(declared - found), sorted(found - declared)) + assert _COMMENTED_CLAUSE_EDGE in found + + def page_or(texts): + # a module the bundle does not carry reads as empty: the walk has + # already counted it as reached, which is what this case asserts + return lambda path: texts[path] if path in texts else ( + (WEB / path).read_text(encoding="utf-8") if path in tree else "") + + reached, _dynamic, edges = _module_graph(page_or(read), tree) + assert _COMMENTED_CLAUSE_EDGE in edges + assert reached - tree == set(), sorted(reached - tree) + importer, imported = _COMMENTED_CLAUSE_EDGE + missing = imported.replace(".js", "-missing.js") + mutated = dict(read) + mutated[importer] = read[importer].replace( + f'}} from "./{Path(imported).name}";', f'}} from "./{Path(missing).name}";') + assert mutated[importer] != read[importer], "the mutant changed nothing" + reached, _dynamic, _edges = _module_graph(page_or(mutated), tree) + assert reached - tree == {missing}, sorted(reached - tree)