From b792768f891ba6d68b69b674759929c1cc8f7904 Mon Sep 17 00:00:00 2001 From: Debasish Bose Date: Sun, 27 Sep 2026 14:11:05 +1000 Subject: [PATCH 1/3] ontology: lazy package exports so init/import run without pyoxigraph On an install WITHOUT the [ontology] extra, `mdl init` and `mdl import erwin` crashed with ModuleNotFoundError: No module named 'pyoxigraph'. Scaffolding calls mdl_ontology.lock.Lock to write .mdl/lock.yaml, and importing that submodule ran mdl_ontology/__init__.py, which eagerly imported the RDF-backed providers/rdf_export chain (pyoxigraph). That broke the core-install promise: the pyoxigraph-free Lock was unreachable. mdl_ontology/__init__.py now exports every public name LAZILY (PEP 562 __getattr__), so importing a pyoxigraph-free member (Lock) no longer pulls the backend; a backend-dependent name imports it only when accessed. _ontology() in the CLI now force-resolves one backend-dependent export (serialize) inside its try/except, so a missing backend still fails there with the 'install modelith-dbt[ontology]' hint (exit 4) instead of leaking a raw ImportError from the command body. Regression tests: init + import erwin under a blocked RDF backend. CLI 0.6.9. --- packages/cli/src/mdl_cli/main.py | 6 + .../cli/tests/test_core_without_ontology.py | 25 +++ .../ontology/src/mdl_ontology/__init__.py | 172 +++++++++++------- pyproject.toml | 2 +- 4 files changed, 138 insertions(+), 67 deletions(-) diff --git a/packages/cli/src/mdl_cli/main.py b/packages/cli/src/mdl_cli/main.py index 0b55efe..54e62b7 100644 --- a/packages/cli/src/mdl_cli/main.py +++ b/packages/cli/src/mdl_cli/main.py @@ -110,6 +110,12 @@ def _ontology(): try: import mdl_ontology + # mdl_ontology exports lazily (PEP 562), so `import mdl_ontology` alone no longer + # pulls the RDF backend — that is deliberate, so pyoxigraph-free members like Lock + # stay reachable on a core install. Force-resolve one backend-dependent export here + # so a missing backend fails INSIDE this guard (→ the install hint), not later as a + # raw ImportError in the command body. + _ = mdl_ontology.serialize # rdf_export → _rdf → pyoxigraph return mdl_ontology except ImportError as e: typer.secho( diff --git a/packages/cli/tests/test_core_without_ontology.py b/packages/cli/tests/test_core_without_ontology.py index ca4c5a1..51cf249 100644 --- a/packages/cli/tests/test_core_without_ontology.py +++ b/packages/cli/tests/test_core_without_ontology.py @@ -142,6 +142,31 @@ def test_ontology_commands_fail_with_install_hint(argv, tmp_path: Path): assert "modelith-dbt[ontology]" in combined, combined +def test_init_scaffolds_without_rdflib(tmp_path: Path): + """`mdl init` writes .mdl/lock.yaml with no RDF backend. Regression: scaffolding + imports mdl_ontology.lock (for Lock), which must NOT drag in the pyoxigraph-backed + providers — a lazy mdl_ontology.__init__ keeps Lock reachable on a core install.""" + with _blocked_backend(): + mod = importlib.reload(importlib.import_module("mdl_cli.main")) + r = runner.invoke(mod.app, ["init", str(tmp_path / "proj")]) + assert r.exit_code == 0, r.output + assert (tmp_path / "proj" / ".mdl" / "lock.yaml").exists() + + +def test_import_erwin_without_rdflib(tmp_path: Path): + """`mdl import erwin` into an empty folder scaffolds a model with no RDF backend.""" + fixture = Path(__file__).resolve().parents[2] / "reverse" / "tests" / "test_erwin.py" + xml = tmp_path / "m.xml" + xml.write_text(fixture.read_text().split('_ERWIN = """')[1].split('"""')[0]) + out = tmp_path / "proj" + with _blocked_backend(): + mod = importlib.reload(importlib.import_module("mdl_cli.main")) + r = runner.invoke(mod.app, ["import", "erwin", str(xml), "-o", str(out)]) + assert r.exit_code == 0, r.output + assert (out / ".mdl" / "lock.yaml").exists() + assert (out / "logical" / "entities" / "counterparty.yaml").exists() + + def test_ontology_still_works_when_backend_present(tmp_path: Path): """Sanity: with the backend installed (the normal test env), an ontology command is NOT gated — it runs. Guards against the helper over-blocking.""" diff --git a/packages/ontology/src/mdl_ontology/__init__.py b/packages/ontology/src/mdl_ontology/__init__.py index ec3c895..242fd52 100644 --- a/packages/ontology/src/mdl_ontology/__init__.py +++ b/packages/ontology/src/mdl_ontology/__init__.py @@ -2,72 +2,112 @@ Vocabulary-agnostic: FIBO is one reference bundle; ACORD/FHIR/ISO 20022/GS1/custom vocabularies plug in by declaration. Depends only on `modelith-core` (§1.3). + +Public names are exported LAZILY (PEP 562). Most of this package's surface pulls in the +RDF backend (pyoxigraph, the optional `[ontology]` extra), but a few members — notably +`Lock` (`.mdl/lock.yaml`, used by `mdl init` / import scaffolding) — do not. Eagerly +importing everything here meant that merely reaching `mdl_ontology.lock` dragged in +pyoxigraph, so `mdl init` / `mdl import erwin` crashed on an install without the extra — +breaking the promise that the core CLI runs without it. Resolving names on first access +instead keeps the pyoxigraph-free members reachable on a core-only install; a name that +does need the backend raises the ModuleNotFoundError only when it is actually used. """ -from mdl_ontology.align import ( - AlignmentProposal, - Candidate, - LexicalMatcher, - Matcher, - align_model, - confidence_band, -) -from mdl_ontology.fetch import ( - FetchError, - FetchResult, - compute_lock, - fetch_all, - fetch_layer, -) -from mdl_ontology.ingest import save_ontology_upload -from mdl_ontology.layers import CoverageReport, check_layers, coverage_report -from mdl_ontology.lock import CACHE_REL, LOCK_MODES, Lock, OntologyLayerLock -from mdl_ontology.providers.cache import cache_from_registry, cache_resolved_term -from mdl_ontology.r2rml_export import ( - R2RMLCoverage, - UnmappedError, - export_r2rml, - r2rml_coverage, -) -from mdl_ontology.rdf_export import export_rdf, export_shacl, serialize -from mdl_ontology.registry import ( - OntologyRegistry, - ResolvedTerm, - VocabularySource, - build_registry, -) +# The TYPE_CHECKING block below re-imports every public name purely so type checkers and +# IDEs resolve them; at runtime __getattr__ does the real (lazy) import. Those imports read +# as unused to the linter, so F401 is suppressed for this file. +# ruff: noqa: F401 + +from __future__ import annotations + +from typing import TYPE_CHECKING + +# public name -> submodule it lives in. Resolved on first attribute access. +_EXPORTS = { + "align_model": "align", + "AlignmentProposal": "align", + "Candidate": "align", + "Matcher": "align", + "LexicalMatcher": "align", + "confidence_band": "align", + "FetchError": "fetch", + "FetchResult": "fetch", + "compute_lock": "fetch", + "fetch_all": "fetch", + "fetch_layer": "fetch", + "save_ontology_upload": "ingest", + "CoverageReport": "layers", + "check_layers": "layers", + "coverage_report": "layers", + "CACHE_REL": "lock", + "LOCK_MODES": "lock", + "Lock": "lock", + "OntologyLayerLock": "lock", + "cache_from_registry": "providers.cache", + "cache_resolved_term": "providers.cache", + "R2RMLCoverage": "r2rml_export", + "UnmappedError": "r2rml_export", + "export_r2rml": "r2rml_export", + "r2rml_coverage": "r2rml_export", + "export_rdf": "rdf_export", + "export_shacl": "rdf_export", + "serialize": "rdf_export", + "OntologyRegistry": "registry", + "ResolvedTerm": "registry", + "VocabularySource": "registry", + "build_registry": "registry", +} + +__all__ = list(_EXPORTS) + + +def __getattr__(name: str): + """PEP 562 lazy export: import the owning submodule only when the name is accessed.""" + module = _EXPORTS.get(name) + if module is None: + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") + import importlib + + mod = importlib.import_module(f"{__name__}.{module}") + return getattr(mod, name) + + +def __dir__() -> list[str]: + return sorted(__all__) + -__all__ = [ - "OntologyRegistry", - "VocabularySource", - "ResolvedTerm", - "build_registry", - "check_layers", - "coverage_report", - "CoverageReport", - "export_rdf", - "export_shacl", - "export_r2rml", - "r2rml_coverage", - "R2RMLCoverage", - "UnmappedError", - "serialize", - "Lock", - "OntologyLayerLock", - "LOCK_MODES", - "CACHE_REL", - "fetch_all", - "fetch_layer", - "compute_lock", - "FetchError", - "FetchResult", - "cache_resolved_term", - "cache_from_registry", - "save_ontology_upload", - "align_model", - "AlignmentProposal", - "Candidate", - "Matcher", - "LexicalMatcher", - "confidence_band", -] +if TYPE_CHECKING: # keep static analysers / IDEs seeing the real symbols + # These are re-exports resolved lazily at runtime via __getattr__; the block exists + # only so type checkers and IDEs see the real symbols (hence the file-level noqa: F401). + from mdl_ontology.align import ( + AlignmentProposal, + Candidate, + LexicalMatcher, + Matcher, + align_model, + confidence_band, + ) + from mdl_ontology.fetch import ( + FetchError, + FetchResult, + compute_lock, + fetch_all, + fetch_layer, + ) + from mdl_ontology.ingest import save_ontology_upload + from mdl_ontology.layers import CoverageReport, check_layers, coverage_report + from mdl_ontology.lock import CACHE_REL, LOCK_MODES, Lock, OntologyLayerLock + from mdl_ontology.providers.cache import cache_from_registry, cache_resolved_term + from mdl_ontology.r2rml_export import ( + R2RMLCoverage, + UnmappedError, + export_r2rml, + r2rml_coverage, + ) + from mdl_ontology.rdf_export import export_rdf, export_shacl, serialize + from mdl_ontology.registry import ( + OntologyRegistry, + ResolvedTerm, + VocabularySource, + build_registry, + ) diff --git a/pyproject.toml b/pyproject.toml index c4ab6ab..c86cb9c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,7 +3,7 @@ # project, so the install name is `modelith-dbt` (install: uv tool install # modelith-dbt). The command stays `mdl`; the import modules stay mdl_*. name = "modelith-dbt" -version = "0.6.8" +version = "0.6.9" description = "Ontology-anchored, git-native data modeling tool for dbt-core teams" readme = "README.md" requires-python = ">=3.11" From 11b6ceef692212720ac4527908263c3f92748987 Mon Sep 17 00:00:00 2001 From: Debasish Bose Date: Sun, 27 Sep 2026 14:22:54 +1000 Subject: [PATCH 2/3] writer: filesystem-safe filenames so imports don't crash on Windows MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit An erwin export carries object names with characters that are illegal in a Windows filename (< > : " / \ | ? *) — e.g. a domain named "". The writer used the name verbatim as a YAML filename, so on Windows the write crashed with OSError [Errno 22] Invalid argument (it silently produced a literally-named file on macOS/Linux, which is why tests missed it). This is the shared serializer every importer (erwin, OSI, reverse) uses, so the crash hit any Windows user with such a name. - _slug() now strips/collapses Windows-illegal characters and trailing dots/spaces to underscores, and never returns empty. It was only lowercasing + replacing spaces before. - Every filename in write_model now routes through _slug (domains, entities, relationships, subject-areas, physical tables previously used the raw or partially-cleaned name). - A collision guard disambiguates names that slug to the same file ("Order Line" vs "Order/Line" -> order_line.yaml + order_line-2.yaml) so lossy slugging never silently overwrites one object with another. The name itself is preserved in the model (a domain stays ""); only the on-disk filename is sanitised. No names are dropped. CLI 0.6.10. --- packages/reverse/src/mdl_reverse/writer.py | 43 ++++++++++++++++--- .../reverse/tests/test_writer_filenames.py | 43 +++++++++++++++++++ pyproject.toml | 2 +- uv.lock | 2 +- 4 files changed, 82 insertions(+), 8 deletions(-) create mode 100644 packages/reverse/tests/test_writer_filenames.py diff --git a/packages/reverse/src/mdl_reverse/writer.py b/packages/reverse/src/mdl_reverse/writer.py index d57f717..3576abf 100644 --- a/packages/reverse/src/mdl_reverse/writer.py +++ b/packages/reverse/src/mdl_reverse/writer.py @@ -8,6 +8,7 @@ from __future__ import annotations +import re from pathlib import Path from mdl_core.ir import Model @@ -65,17 +66,34 @@ def _write_project_config(model: Model, root: Path) -> None: def write_model(model: Model, root: Path) -> list[str]: root = Path(root) written: list[str] = [] + used: set[str] = set() # rel paths already taken this write, for collision-avoidance # Collection fields that default to []: `exclude_none` does not drop an empty # list, so a reversed model would carry a noise `members: []` on every object. _EMPTY_OK = ("members", "synonyms", "subtypes", "ontology_refs", "values") + def _unique(rel: str) -> str: + """Disambiguate a filename collision. Slugging is lossy (two names can map to one + filename, e.g. 'Order Line' and 'Order/Line' -> 'order_line'), so a clash would + silently overwrite one object with another. Append -2, -3… on the stem instead.""" + if rel not in used: + used.add(rel) + return rel + stem, dot, ext = rel.rpartition(".") + n = 2 + while f"{stem}-{n}{dot}{ext}" in used: + n += 1 + cand = f"{stem}-{n}{dot}{ext}" + used.add(cand) + return cand + def dump(rel: str, obj) -> None: data = obj.model_dump(by_alias=True, exclude_none=True, mode="json") for key in _EMPTY_OK: if data.get(key) == []: data.pop(key) # `kind` is an enum -> its value; pydantic mode="json" already handles it. + rel = _unique(rel) path = root / rel path.parent.mkdir(parents=True, exist_ok=True) path.write_text(dump_str(data), encoding="utf-8") @@ -87,25 +105,25 @@ def dump(rel: str, obj) -> None: written.append("mdl-project.yaml") for sa in model.subject_areas.values(): - dump(f"conceptual/subject-areas/{sa.name.lower().replace(' ', '_')}.yaml", sa) + dump(f"conceptual/subject-areas/{_slug(sa.name)}.yaml", sa) for ce in model.conceptual_entities.values(): dump(f"conceptual/entities/{_slug(ce.name)}.yaml", ce) for term in model.terms.values(): dump(f"conceptual/terms/{_slug(term.name)}.yaml", term) for dom in model.domains.values(): - dump(f"logical/domains/{dom.name}.yaml", dom) + dump(f"logical/domains/{_slug(dom.name)}.yaml", dom) for cs in model.code_sets.values(): dump(f"logical/value-sets/{_slug(cs.name)}.yaml", cs) for le in model.logical_entities.values(): - dump(f"logical/entities/{le.name}.yaml", le) + dump(f"logical/entities/{_slug(le.name)}.yaml", le) for rel in model.relationships.values(): - dump(f"logical/relationships/{rel.name}.yaml", rel) + dump(f"logical/relationships/{_slug(rel.name)}.yaml", rel) for kg in model.key_groups.values(): dump(f"logical/key-groups/{_slug(kg.name)}.yaml", kg) for cat in model.categories.values(): dump(f"logical/categories/{_slug(cat.name)}.yaml", cat) for pt in model.physical_tables.values(): - dump(f"physical/{pt.target}/tables/{pt.name.lower()}.yaml", pt) + dump(f"physical/{_slug(pt.target)}/tables/{_slug(pt.name)}.yaml", pt) # Prune stale object files from a PRIOR reverse into this dir: an entity that no # longer exists (e.g. now excluded by an edited reverse.exclude) would otherwise @@ -145,5 +163,18 @@ def _prune_stale(root: Path, written: set[str]) -> None: pass +# Characters illegal in a Windows filename (< > : " / \ | ? *) plus control chars. A +# reversed/imported object name is used verbatim as a filename, and an erwin export can +# carry names like "" or "Order:Line" that crash open() on Windows (OSError 22) and +# make a portable git repo impossible. Map every unsafe run to a single underscore. +_UNSAFE_FS = re.compile(r'[<>:"/\\|?*\x00-\x1f]+') + + def _slug(name: str) -> str: - return name.lower().replace(" ", "_") + """A filesystem-safe, portable slug for a filename: lowercased, spaces and any + Windows-illegal characters collapsed to underscores, and trailing dots/spaces (also + illegal on Windows) stripped. Never returns empty.""" + s = _UNSAFE_FS.sub("_", name).lower().replace(" ", "_") + # Windows also forbids a name ending in a dot or space, and bare dots are confusing. + s = s.strip("._") + return s or "unnamed" diff --git a/packages/reverse/tests/test_writer_filenames.py b/packages/reverse/tests/test_writer_filenames.py new file mode 100644 index 0000000..2f73b6a --- /dev/null +++ b/packages/reverse/tests/test_writer_filenames.py @@ -0,0 +1,43 @@ +"""The writer must produce portable, filesystem-safe filenames. + +A reversed/imported object name is used to derive its YAML filename. An erwin export can +carry names with characters that are illegal in a Windows filename (< > : " / \\ | ? *), +e.g. a domain named "", which crashes open() on Windows (OSError 22). The writer +slugs every name to a safe form, and disambiguates collisions the slugging can create. +""" + +from __future__ import annotations + +from mdl_core.ir import Domain, LogicalEntity, Model, ProjectConfig +from mdl_reverse.writer import _slug, write_model + + +def test_slug_strips_windows_illegal_chars(): + assert _slug("") == "root" + assert _slug("Order:Line") == "order_line" + assert _slug("Foo/Bar") == "foo_bar" + assert _slug('a"b|c?d*e') == "a_b_c_d_e" + assert _slug("trailing.") == "trailing" # Windows forbids a trailing dot + assert _slug(" ") == "unnamed" # never empty + assert _slug("Counterparty") == "counterparty" # a clean name is unchanged + + +def test_writer_handles_illegal_domain_name(tmp_path): + # a domain literally named "" (erwin's structural node) must not crash the write + m = Model(ProjectConfig(name="m", dbt_target="duckdb_dev")) + m.add(Domain(id="01J000000000000000000DOM01", name="", base_type="string")) + written = write_model(m, tmp_path) + # it landed at a safe filename, and no angle brackets reached the filesystem + assert (tmp_path / "logical" / "domains" / "root.yaml").exists() + assert any(w.endswith("root.yaml") for w in written) + + +def test_writer_disambiguates_filename_collisions(tmp_path): + # two entities whose names slug to the SAME filename must both survive, not overwrite + m = Model(ProjectConfig(name="m", dbt_target="duckdb_dev")) + m.add(LogicalEntity(id="01J000000000000000000ENT01", name="Order Line", realises=None)) + m.add(LogicalEntity(id="01J000000000000000000ENT02", name="Order/Line", realises=None)) + write_model(m, tmp_path) + files = sorted(p.name for p in (tmp_path / "logical" / "entities").glob("*.yaml")) + # both were written (one disambiguated with a -2 suffix), neither clobbered + assert files == ["order_line-2.yaml", "order_line.yaml"] diff --git a/pyproject.toml b/pyproject.toml index c86cb9c..1fdf9b2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,7 +3,7 @@ # project, so the install name is `modelith-dbt` (install: uv tool install # modelith-dbt). The command stays `mdl`; the import modules stay mdl_*. name = "modelith-dbt" -version = "0.6.9" +version = "0.6.10" description = "Ontology-anchored, git-native data modeling tool for dbt-core teams" readme = "README.md" requires-python = ">=3.11" diff --git a/uv.lock b/uv.lock index 561f96d..674bb1c 100644 --- a/uv.lock +++ b/uv.lock @@ -939,7 +939,7 @@ requires-dist = [ [[package]] name = "modelith-dbt" -version = "0.6.8" +version = "0.6.9" source = { editable = "." } dependencies = [ { name = "dbt-artifacts-parser" }, From 12de75ff4f5759e0d9eb0424c5cf7ba9dc89d8c1 Mon Sep 17 00:00:00 2001 From: Debasish Bose Date: Sun, 27 Sep 2026 14:30:48 +1000 Subject: [PATCH 3/3] sandbox: core-only clean-install Docker verification for erwin import MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Our dev workspace (uv run) always has every optional dependency, so a command that reaches the optional ontology stack passes locally and crashes on a real `pip install modelith-dbt` without [ontology]. That is how both recent Windows crashes shipped: import erwin pulled pyoxigraph via the scaffold path, and a `` domain produced a filesystem-illegal filename — neither reproducible in the dev env. Adds sandbox/clean-install/: a python:3.12-slim image that installs the locally built wheel CORE-ONLY (no extras) and runs the erwin import flow where a real user runs it — asserting the CLI loads, import erwin into an empty folder scaffolds without the ontology extra, the domain writes to a safe filename, and the model validates. run.sh builds the wheel, builds the image, and runs it in one command; exits non-zero on failure so it is a pre-ship / CI gate. Verified: all checks pass on the current wheel; the env genuinely lacks pyoxigraph (negative control). Scoped to erwin import on a core install for now (the flows that broke); init --workspace / config / generate bars and an [ontology] pass are the natural extensions, plus a GitHub Actions job. --- sandbox/clean-install/Dockerfile | 24 ++++++ sandbox/clean-install/README.md | 49 +++++++++++ sandbox/clean-install/run-clean.sh | 126 +++++++++++++++++++++++++++++ sandbox/clean-install/run.sh | 23 ++++++ 4 files changed, 222 insertions(+) create mode 100644 sandbox/clean-install/Dockerfile create mode 100644 sandbox/clean-install/README.md create mode 100755 sandbox/clean-install/run-clean.sh create mode 100755 sandbox/clean-install/run.sh diff --git a/sandbox/clean-install/Dockerfile b/sandbox/clean-install/Dockerfile new file mode 100644 index 0000000..2e26dfd --- /dev/null +++ b/sandbox/clean-install/Dockerfile @@ -0,0 +1,24 @@ +# Clean-install verification: the CORE-ONLY install a user gets from +# pip install modelith-dbt (NOT 'modelith-dbt[ontology]') +# +# This is the environment our dev workspace hides: `uv run` always has every extra +# (pyoxigraph included), so a command that imports the optional RDF stack passes +# locally and crashes on a real core-only install. That is exactly the class of bug +# that shipped — `mdl import erwin` into an empty folder pulled mdl_ontology → the +# pyoxigraph backend, which a core install does not have, and died with +# ModuleNotFoundError: No module named 'pyoxigraph'. +# +# The image is a faithful clean machine: python:3.12-slim, a non-root user, pip only. +# It installs a LOCALLY BUILT wheel (mounted at /tmp/wheels) so an unreleased branch +# is tested, and it installs NO extras — the point is to prove the core CLI's logical +# flows run without them. run-clean.sh does the asserts and exits non-zero on failure, +# so it doubles as a CI gate. +FROM python:3.12-slim + +RUN useradd -m -s /bin/bash user +USER user +WORKDIR /home/user + +COPY --chown=user:user run-clean.sh /home/user/run-clean.sh + +ENTRYPOINT ["/bin/bash", "/home/user/run-clean.sh"] diff --git a/sandbox/clean-install/README.md b/sandbox/clean-install/README.md new file mode 100644 index 0000000..2f7f982 --- /dev/null +++ b/sandbox/clean-install/README.md @@ -0,0 +1,49 @@ +# Clean-install verification (core-only) + +A Docker harness that runs Modelith's logical flows in the install configuration our +dev workspace hides: a plain `pip install modelith-dbt` with **no extras**. + +## Why this exists + +`uv run` in the workspace always has every optional dependency (pyoxigraph, the RDF +backend, included). So a command that reaches the optional ontology stack passes +locally and crashes on a real core-only install. That is precisely what shipped: +`mdl import erwin` into an empty folder scaffolds `.mdl/lock.yaml` via +`mdl_ontology.lock.Lock`, which used to drag in the pyoxigraph backend — absent on a +core install — and died with `ModuleNotFoundError: No module named 'pyoxigraph'`. A +second bug (an erwin domain named `` producing a Windows-illegal filename) also +only shows on a non-macOS filesystem. + +This harness runs the flows where a real user runs them, so that class of +"green in dev, broken on install" bug is caught before shipping. + +## Run it + +From the repo root, with Docker running: + +```bash +./sandbox/clean-install/run.sh +``` + +It builds the current wheel (`uv build`), builds the clean image, and runs the +core-only verification with the wheel mounted at `/tmp/wheels`. Exits non-zero on any +failure, so it doubles as a pre-ship / CI gate. + +## What it asserts + +| Check | What it proves | +|---|---| +| `pyoxigraph` absent | the env is genuinely core-only (the test is meaningful) | +| `mdl --version` | the CLI imports without the optional stack | +| `mdl import erwin` (empty folder) | the scaffold path runs without the ontology extra | +| `.mdl/lock.yaml` written | `Lock` is reachable without pyoxigraph | +| `` domain → safe filename | filesystem-illegal names don't crash the writer | +| `mdl validate` | the imported model is valid on a core install | + +## Scope + +Today it covers the **erwin import** flow on a core-only install (the flows that +broke). Extend `run-clean.sh` with more bars (init --workspace, config set/get, +generate) and add a second `pip install 'modelith-dbt[ontology]'` pass when broader +coverage is wanted. Not yet wired into CI — run it by hand before shipping; a +GitHub Actions job is the natural follow-up. diff --git a/sandbox/clean-install/run-clean.sh b/sandbox/clean-install/run-clean.sh new file mode 100755 index 0000000..2b3d5c8 --- /dev/null +++ b/sandbox/clean-install/run-clean.sh @@ -0,0 +1,126 @@ +#!/usr/bin/env bash +# Clean-install verification harness (runs INSIDE the core-only container). +# +# Installs the locally built wheel with NO extras, then runs the erwin-import logical +# flow the way a real user on a core install does. Asserts each step and exits +# non-zero on the first hard failure, so this is a CI gate as well as a manual check. +# +# What this specifically guards (the bugs that shipped to the Windows box): +# * `mdl import erwin` into an EMPTY folder must not need the [ontology] extra +# (scaffolding writes .mdl/lock.yaml via mdl_ontology.lock.Lock, which must be +# reachable without pyoxigraph). +# * an erwin name with filesystem-illegal characters (a domain named "") +# must write to a safe filename, not crash open() (a Linux fs rejects the raw +# path differently than macOS, so this runs on a real non-macOS filesystem). +set -uo pipefail + +GREEN=$'\033[32m'; RED=$'\033[31m'; CYAN=$'\033[36m'; RESET=$'\033[0m' +fails=0 +pass() { echo "${GREEN}PASS${RESET} $1"; } +fail() { echo "${RED}FAIL${RESET} $1 ${CYAN}($2)${RESET}"; fails=$((fails+1)); } +hr() { echo "------------------------------------------------------------"; } + +echo "Modelith clean-install (core-only) verification" +export PATH="$HOME/.local/bin:$PATH" +hr + +# --- install the built wheel, CORE ONLY (no extras) ------------------------ +WHL=$(ls /tmp/wheels/modelith_dbt-*.whl 2>/dev/null | head -1) +if [ -z "$WHL" ]; then + fail "wheel present" "no modelith_dbt-*.whl in /tmp/wheels (build it and mount dist/)" + echo "${RED}1 failure${RESET}"; exit 1 +fi +echo "installing (core only, no [ontology]): $(basename "$WHL")" +python3 -m pip install --user --quiet "$WHL" >/tmp/install.log 2>&1 || { + fail "pip install core wheel" "see /tmp/install.log"; cat /tmp/install.log; exit 1; +} + +# prove the optional backend is genuinely ABSENT — otherwise this test is meaningless +if python3 -c "import pyoxigraph" 2>/dev/null; then + fail "core install is truly core" "pyoxigraph is present — not a core-only env" +else + pass "core install has no pyoxigraph (as intended)" +fi +hr + +# --- BAR 1: the CLI loads and --version works (no crash on import) ---------- +if mdl --version >/tmp/version.log 2>&1; then + pass "mdl --version runs on a core install ($(cat /tmp/version.log))" +else + fail "mdl --version" "see /tmp/version.log"; cat /tmp/version.log +fi +hr + +# --- a minimal real-shaped erwin export, incl. a "" domain ----------- +# namespaced, GUID-referenced, props-as-text — the real shape. The domain +# exercises the filesystem-safe-filename path; the whole thing exercises the +# scaffold path that pulled in the optional ontology backend. +cat > /home/user/model.xml <<'XML' + + + + + <root> + id_typeBIGINT + + + PartyPARTY + + party_idD1 + + + pk_partyprimary_key + A1 + + + + + + +XML + +# --- BAR 2: import erwin into an EMPTY folder scaffolds a runnable project -- +PROJ="/home/user/proj" +rm -rf "$PROJ" +if mdl import erwin /home/user/model.xml -o "$PROJ" >/tmp/import.log 2>&1; then + pass "mdl import erwin (empty folder, core install) — no crash" +else + fail "mdl import erwin" "see below"; sed -n '1,40p' /tmp/import.log +fi + +# the skeleton the scaffold path writes (this is where pyoxigraph used to be pulled) +[ -f "$PROJ/.mdl/lock.yaml" ] && pass ".mdl/lock.yaml written (scaffold ran without ontology extra)" \ + || fail ".mdl/lock.yaml missing" "scaffold path failed" + +# the erwin objects landed +[ -f "$PROJ/logical/entities/party.yaml" ] && pass "logical entity written" \ + || fail "logical entity missing" "$(ls -R "$PROJ" 2>/dev/null | head)" + +# the "" domain wrote to a SAFE filename, not a crash / not angle brackets on disk +if ls "$PROJ"/logical/domains/*.yaml >/tmp/doms.log 2>&1; then + if ls "$PROJ"/logical/domains/ | grep -q '[<>]'; then + fail "domain filename has illegal chars" "$(ls "$PROJ"/logical/domains/)" + else + pass "illegal-char domain () wrote to a safe filename ($(ls "$PROJ"/logical/domains/ | tr '\n' ' '))" + fi +else + fail "no domain files written" "see /tmp/doms.log" +fi +hr + +# --- BAR 3: the imported model validates (still core-only) ------------------ +if mdl validate -m "$PROJ" >/tmp/validate.log 2>&1; then + pass "mdl validate passes on the imported model (core install)" +else + fail "mdl validate" "see below"; sed -n '1,30p' /tmp/validate.log +fi +hr + +# --- verdict --------------------------------------------------------------- +if [ "$fails" -eq 0 ]; then + echo "${GREEN}ALL CLEAN-INSTALL CHECKS PASSED${RESET}" + exit 0 +else + echo "${RED}${fails} check(s) FAILED${RESET} (logs under /tmp/*.log in the container)" + exit 1 +fi diff --git a/sandbox/clean-install/run.sh b/sandbox/clean-install/run.sh new file mode 100755 index 0000000..17e3cec --- /dev/null +++ b/sandbox/clean-install/run.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash +# One command: build the current wheel, build the clean-install image, run the +# core-only verification with the wheel mounted in. +# +# ./sandbox/clean-install/run.sh +# +# Exits non-zero if any check fails, so it is usable as a pre-ship gate / CI step. +set -euo pipefail + +HERE="$(cd "$(dirname "$0")" && pwd)" +ROOT="$(cd "$HERE/../.." && pwd)" + +echo ">> building wheel from $ROOT" +rm -rf "$ROOT/dist" +( cd "$ROOT" && uv build --wheel -o dist >/dev/null ) +ls "$ROOT"/dist/modelith_dbt-*.whl >/dev/null || { echo "no wheel built"; exit 1; } +echo " $(basename "$(ls "$ROOT"/dist/modelith_dbt-*.whl | head -1)")" + +echo ">> building clean-install image" +docker build -q -t modelith-clean-install "$HERE" >/dev/null + +echo ">> running core-only verification" +docker run --rm -v "$ROOT/dist:/tmp/wheels:ro" modelith-clean-install