From bd729ff43e73ca2100c01ea9345f1652a03988ad Mon Sep 17 00:00:00 2001 From: Debasish Bose Date: Sun, 27 Sep 2026 09:50:48 +1000 Subject: [PATCH 1/2] import: erwin import into an empty folder bootstraps a full project MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit An erwin export is a whole model, but importing it into an empty folder only wrote object YAMLs plus a bare mdl-project.yaml — no .mdl/lock.yaml, no .mdl/state/, no .gitignore, and in VS Code the command aborted unless a project already existed. Nobody migrates off erwin onto an orphan model. mdl import erwin now scaffolds a runnable project around the import when --out is empty, configurable and structure-respecting: - --scaffold model|workspace|none (default model). workspace also lays down the transform/warehouse dbt project with a zero-setup DuckDB profiles.yml, CODEOWNERS and a .code-workspace; model writes just the skeleton. - --dbt-target names the dbt target/profile written into the config. - The layout is persisted to mdl-project.yaml under a new scaffold: block (ProjectConfig.scaffold) and joins writer._USER_OWNED_CONFIG so a re-import reproduces, never rewrites, it. - An --out that already holds a project (mdl-project.yaml or .mdl/, at the root or a model/ subdir) is respected: no re-scaffold, user config preserved. VS Code: importErwin no longer requires an existing model. In an empty workspace it confirms, then scaffolds using modelith.import.scaffold / modelith.import.dbtTarget and sets modelith.modelDir to the written model so the panels resolve it. bootstrap_project + scaffold_workspace_skeleton reuse the audited init/collab plumbing. CLI 0.6.7, extension 0.3.15. --- README.md | 16 ++++- packages/cli/src/mdl_cli/collab.py | 34 ++++++++++ packages/cli/src/mdl_cli/main.py | 59 +++++++++++++++-- packages/cli/src/mdl_cli/scaffold.py | 70 +++++++++++++++++++-- packages/cli/tests/test_import_erwin_cli.py | 70 +++++++++++++++++++++ packages/core/src/mdl_core/ir.py | 22 +++++++ packages/reverse/src/mdl_reverse/writer.py | 3 + pyproject.toml | 2 +- uv.lock | 2 +- vscode/package.json | 22 ++++++- vscode/src/extension.ts | 54 ++++++++++++++-- 11 files changed, 333 insertions(+), 21 deletions(-) diff --git a/README.md b/README.md index 8a392b5..e0efb99 100644 --- a/README.md +++ b/README.md @@ -467,15 +467,25 @@ foreign keys, relationships (with their verb phrases), subtype/supertype categor subject areas, and domains, keeping the physical names alongside the logical ones: ```bash -mdl import erwin your-model.xml -o model # writes a fresh model -mdl import erwin your-model.xml -o model --apply # or merge into an existing one +mdl import erwin your-model.xml -o proj # empty folder -> a full, runnable project +mdl import erwin your-model.xml -o proj --scaffold workspace # + a transform/warehouse dbt project +mdl import erwin your-model.xml -o model --apply # or merge into a model you already have ``` +Point it at an **empty folder** and it bootstraps a runnable project around the import — the +`.gitignore`, the pinned `.mdl/lock.yaml`, a populated `mdl-project.yaml` — so there is no +separate `mdl init` step. `--scaffold workspace` also lays down the `transform/warehouse` +dbt project (with a zero-setup DuckDB `profiles.yml`), CODEOWNERS and a `.code-workspace`. +The chosen layout is recorded in `mdl-project.yaml`, and if the folder **already** holds a +Modelith project the existing structure and config are respected (no re-scaffold). Both the +layout and the dbt target are configurable from the CLI (`--scaffold`, `--dbt-target`) and +from VS Code settings (`modelith.import.scaffold`, `modelith.import.dbtTarget`). + It parses the export by streaming, so 7-10MB files import without a memory spike, and it degrades gracefully: anything Modelith doesn't model (views, ER-diagram layout, physical transforms) is skipped with a note rather than a failure. In VS Code it's on the palette (**Modelith: Import Erwin XML**), a button in the Reverse Review panel, and a right-click -on any `.xml` file. +on any `.xml` file — and it now works in an empty workspace, offering to scaffold a project. ## Reverse engineering a real warehouse diff --git a/packages/cli/src/mdl_cli/collab.py b/packages/cli/src/mdl_cli/collab.py index 8e01417..d47c373 100644 --- a/packages/cli/src/mdl_cli/collab.py +++ b/packages/cli/src/mdl_cli/collab.py @@ -170,6 +170,23 @@ def _git(*args: str) -> bool: target-path: target """ +# A zero-setup, credential-free local profile (DuckDB, file-based) so the generated +# warehouse can `dbt build` immediately. The profile name matches `profile:` in the +# dbt_project stub. Point it at your real warehouse (and set env vars) when ready — +# Modelith never writes a credential. `{target}` names the output the model config uses. +_PROFILES_STUB = """\ +# Local dbt profile — DuckDB, file-based, zero setup. Swap for your warehouse when ready +# (`mdl reverse --init-profile ` scaffolds a credential-free template). The +# profile name must match `profile:` in dbt_project.yml (warehouse). +warehouse: + target: {target} + outputs: + {target}: + type: duckdb + path: warehouse.duckdb # created on first run, relative to this dir + threads: 4 +""" + def scaffold_workspace(root: Path, project_name: str, scaffold_model) -> list[str]: """One repo, two sibling roots (§2.1): model/ + transform/warehouse/, with @@ -181,12 +198,29 @@ def scaffold_workspace(root: Path, project_name: str, scaffold_model) -> list[st scaffold_model(root / model_root, project_name=project_name) written.append(f"{model_root}/ (model repo)") + written += scaffold_workspace_skeleton(root, project_name) + return written + + +def scaffold_workspace_skeleton( + root: Path, project_name: str, *, dbt_target: str = "duckdb_dev" +) -> list[str]: + """The collaboration topology WITHOUT scaffolding a seed model under model/ (§2.1): + the transform/warehouse dbt project, ontologies/, CODEOWNERS, the .code-workspace and + the git merge drivers. Used by an import that supplies its own model objects, so it + lays down the plumbing and leaves model/ for the importer to write into. Idempotent.""" + written: list[str] = [] + model_root = "model" + tw = root / "transform" / "warehouse" (tw / "models" / "staging").mkdir(parents=True, exist_ok=True) (tw / "macros").mkdir(parents=True, exist_ok=True) if not (tw / "dbt_project.yml").exists(): (tw / "dbt_project.yml").write_text(_DBT_PROJECT_STUB) written.append("transform/warehouse/ (dbt project)") + if not (tw / "profiles.yml").exists(): + (tw / "profiles.yml").write_text(_PROFILES_STUB.format(target=dbt_target)) + written.append("transform/warehouse/profiles.yml") (root / "ontologies" / "industry").mkdir(parents=True, exist_ok=True) diff --git a/packages/cli/src/mdl_cli/main.py b/packages/cli/src/mdl_cli/main.py index 157663b..ce3dcd5 100644 --- a/packages/cli/src/mdl_cli/main.py +++ b/packages/cli/src/mdl_cli/main.py @@ -2730,6 +2730,21 @@ def import_erwin_cmd( file: Path = typer.Argument(..., help="erwin XML export (.xml)"), out: Path = typer.Option(Path("model"), "--out", "-o"), name: str = typer.Option(None, "--name"), + scaffold_layout: str = typer.Option( + "model", + "--scaffold", + help="When --out is EMPTY, bootstrap a full project around the import: " + "`model` (a runnable model repo — .gitignore, .mdl/lock.yaml, .mdl/state/), " + "`workspace` (also the transform/warehouse dbt project, CODEOWNERS, " + ".code-workspace — collab §2.1), or `none` (model objects only, no skeleton). " + "Ignored when --out already holds a project (that layout is respected).", + ), + dbt_target: str = typer.Option( + "duckdb_dev", + "--dbt-target", + help="dbt target/profile name written into the project config (and the workspace " + "profiles.yml). DuckDB by default so the generated warehouse builds with no setup.", + ), apply: bool = typer.Option( False, "--apply", @@ -2744,9 +2759,12 @@ def import_erwin_cmd( """Import a real erwin XML export into a Modelith model (spec §6.4). Default: write a fresh model at --out (like `mdl reverse` — an erwin export is a whole - model). If --out already holds a model it diverts to a `-reversed-v` sibling unless - --force. With --apply, merge the erwin objects into the existing model at --out through - the same validated mutation engine manual edits use.""" + model). Into an EMPTY --out it also scaffolds a runnable project around the objects + (`--scaffold model|workspace|none`, persisted to mdl-project.yaml so a re-import + reproduces the layout). If --out already holds a project the existing structure is + RESPECTED — no re-scaffold, user config preserved — and it diverts to a `-reversed-v` + sibling unless --force. With --apply, merge the erwin objects into the existing model at + --out through the same validated mutation engine manual edits use.""" # streaming reader handles 7-10MB files; pass the PATH so it reads the file directly. result = import_erwin(file, project_name=name) for w in result.warnings: @@ -2771,12 +2789,41 @@ def import_erwin_cmd( ) return + # A project already here (an mdl-project.yaml or a .mdl/ dir, at --out or its model/ + # subdir) is respected: no scaffold, and the writer preserves the user's config. + existing = ( + (out / "mdl-project.yaml").exists() + or (out / ".mdl").exists() + or (out / "model" / "mdl-project.yaml").exists() + ) + + from mdl_cli.scaffold import bootstrap_project + + write_target, scaffolded = bootstrap_project( + out, + project_name=name or _project_name_from_path(out), + layout=scaffold_layout, + dbt_target=dbt_target, + existing=existing, + ) + for line in scaffolded: + typer.secho(f" scaffolded {line}", fg=typer.colors.GREEN) + + # Stamp the config so the layout persists (writer._USER_OWNED_CONFIG keeps it on a + # re-import). Only on a fresh scaffold — an existing project owns its own config. + if not existing and scaffold_layout != "none": + result.model.config.dbt_target = dbt_target + result.model.config.platform_targets = [dbt_target] + result.model.config.scaffold.layout = scaffold_layout + if scaffold_layout == "workspace": + result.model.config.scaffold.dbt_project_dir = "transform/warehouse" + if not force: - out = _nonclobbering_out(out) - write_reversed(result.model, out) + write_target = _nonclobbering_out(write_target) + write_reversed(result.model, write_target) typer.secho( f"imported {len(result.model.logical_entities)} entities, " - f"{len(result.model.relationships)} relationships from erwin to {out}", + f"{len(result.model.relationships)} relationships from erwin to {write_target}", fg=typer.colors.GREEN, ) diff --git a/packages/cli/src/mdl_cli/scaffold.py b/packages/cli/src/mdl_cli/scaffold.py index b9b73fb..0414967 100644 --- a/packages/cli/src/mdl_cli/scaffold.py +++ b/packages/cli/src/mdl_cli/scaffold.py @@ -118,13 +118,75 @@ def scaffold(root: Path, *, project_name: str = "modelith_model") -> dict[str, s p.parent.mkdir(parents=True, exist_ok=True) p.write_text(content, encoding="utf-8") - # Empty state dirs the tool expects. + files.update(_scaffold_skeleton(root)) + return files + + +def _scaffold_skeleton(root: Path) -> dict[str, str]: + """The tool-owned skeleton every project needs, WITHOUT any seed model objects: + the .gitignore, the empty .mdl/state/ dir, and a pinned .mdl/lock.yaml. Shared by + `scaffold` (which adds a seed example) and `bootstrap_project` (which lets an import + supply the objects). Idempotent — safe to call on a dir that already has some of it.""" + written: dict[str, str] = {} + root.mkdir(parents=True, exist_ok=True) + + gi = root / ".gitignore" + if not gi.exists(): + gi.write_text(_GITIGNORE, encoding="utf-8") + written[".gitignore"] = _GITIGNORE + + # Empty state dir the tool expects. (root / ".mdl" / "state").mkdir(parents=True, exist_ok=True) # Pin spec versions (spec §2.2, §13.1). Import lazily so `core`/scaffold stay # free of an ontology dependency at module load. from mdl_ontology.lock import Lock - Lock().save(root) - files[".mdl/lock.yaml"] = (root / ".mdl" / "lock.yaml").read_text(encoding="utf-8") - return files + lock = root / ".mdl" / "lock.yaml" + if not lock.exists(): + Lock().save(root) + written[".mdl/lock.yaml"] = lock.read_text(encoding="utf-8") + return written + + +def bootstrap_project( + root: Path, + *, + project_name: str, + layout: str = "model", + dbt_target: str = "duckdb_dev", + existing: bool = False, +) -> tuple[Path, list[str]]: + """Lay down the project skeleton an IMPORT (erwin, etc.) writes its objects into, + without any seed example objects. Returns (model_root, written_paths): + + - ``model_root`` is where the importer should write its object YAMLs. For a + ``workspace`` layout that is ``root/model`` (the collab §2.1 sibling); otherwise + ``root``. + - ``existing`` True means the target already holds a Modelith project (an + ``mdl-project.yaml`` or a ``.mdl/`` dir): scaffold NOTHING structural and RESPECT + what is there — the caller merges objects and the writer preserves the user's + config. Returns (root, []). + - ``layout == "none"`` writes no skeleton (model objects only, the pre-scaffold + behaviour). Returns (root, []). + - ``layout == "model"`` writes the shared skeleton (.gitignore, .mdl/state, + .mdl/lock.yaml) at ``root``. + - ``layout == "workspace"`` also lays down the transform/warehouse dbt project, + CODEOWNERS, the .code-workspace and git merge drivers (collab §2.1), with the + model under ``root/model``. + """ + if existing or layout == "none": + # An existing WORKSPACE keeps its model under model/; a plain project at root. + nested = existing and (root / "model" / "mdl-project.yaml").exists() + return (root / "model" if nested else root), [] + + if layout == "workspace": + from mdl_cli.collab import scaffold_workspace_skeleton + + model_root = root / "model" + written = scaffold_workspace_skeleton(root, project_name, dbt_target=dbt_target) + written.extend(f"model/{p}" for p in _scaffold_skeleton(model_root)) + return model_root, written + + # layout == "model" (default): a plain model repo skeleton at root. + return root, list(_scaffold_skeleton(root)) diff --git a/packages/cli/tests/test_import_erwin_cli.py b/packages/cli/tests/test_import_erwin_cli.py index e0fee13..f231e90 100644 --- a/packages/cli/tests/test_import_erwin_cli.py +++ b/packages/cli/tests/test_import_erwin_cli.py @@ -61,3 +61,73 @@ def test_import_erwin_diverts_on_existing_model(tmp_path): assert r.exit_code == 0, r.output # the original is untouched; a -reversed-v1 sibling was written assert (tmp_path / "model-reversed-v1" / "mdl-project.yaml").exists() + + +def test_import_erwin_empty_folder_bootstraps_model(tmp_path): + # empty folder + default --scaffold model → a runnable project skeleton + objects + xml = tmp_path / "trading.xml" + xml.write_text(_erwin_xml()) + out = tmp_path / "proj" + r = runner.invoke(app, ["import", "erwin", str(xml), "-o", str(out)]) + assert r.exit_code == 0, r.output + assert (out / ".mdl" / "lock.yaml").exists() + assert (out / ".mdl" / "state").is_dir() + assert (out / ".gitignore").exists() + assert (out / "logical" / "entities" / "counterparty.yaml").exists() + # the layout + dbt target persist in the config + from mdl_core.yaml_io import load_file + + cfg = load_file(out / "mdl-project.yaml") + assert dict(cfg["scaffold"])["layout"] == "model" + assert cfg["dbt_target"] == "duckdb_dev" + assert runner.invoke(app, ["validate", "-m", str(out)]).exit_code == 0 + + +def test_import_erwin_workspace_layout(tmp_path): + # --scaffold workspace → objects under model/, plus the transform/warehouse project + xml = tmp_path / "trading.xml" + xml.write_text(_erwin_xml()) + out = tmp_path / "repo" + r = runner.invoke( + app, ["import", "erwin", str(xml), "-o", str(out), "--scaffold", "workspace"] + ) + assert r.exit_code == 0, r.output + assert (out / "model" / "logical" / "entities" / "counterparty.yaml").exists() + assert (out / "model" / ".mdl" / "lock.yaml").exists() + assert (out / "transform" / "warehouse" / "dbt_project.yml").exists() + assert (out / "transform" / "warehouse" / "profiles.yml").exists() + assert (out / ".github" / "CODEOWNERS").exists() + from mdl_core.yaml_io import load_file + + cfg = load_file(out / "model" / "mdl-project.yaml") + assert dict(cfg["scaffold"])["dbt_project_dir"] == "transform/warehouse" + assert runner.invoke(app, ["validate", "-m", str(out / "model")]).exit_code == 0 + + +def test_import_erwin_none_layout_no_skeleton(tmp_path): + # --scaffold none → model objects only, no .mdl skeleton (pre-scaffold behaviour) + xml = tmp_path / "trading.xml" + xml.write_text(_erwin_xml()) + out = tmp_path / "bare" + r = runner.invoke(app, ["import", "erwin", str(xml), "-o", str(out), "--scaffold", "none"]) + assert r.exit_code == 0, r.output + assert (out / "logical" / "entities" / "counterparty.yaml").exists() + assert not (out / ".mdl" / "lock.yaml").exists() + + +def test_import_erwin_respects_existing_structure(tmp_path): + # a dir that already has a project keeps its config; the import merges, no re-scaffold + xml = tmp_path / "trading.xml" + xml.write_text(_erwin_xml()) + out = tmp_path / "existing" + out.mkdir() + (out / "mdl-project.yaml").write_text( + "name: existing\ndbt_target: snowflake_prod\nplatform_targets: [snowflake_prod]\n" + ) + r = runner.invoke(app, ["import", "erwin", str(xml), "-o", str(out), "--apply"]) + assert r.exit_code == 0, r.output + # the user's target is preserved, not overwritten by the scaffold default + from mdl_core.yaml_io import load_file + + cfg = load_file(out / "mdl-project.yaml") + assert cfg["dbt_target"] == "snowflake_prod" diff --git a/packages/core/src/mdl_core/ir.py b/packages/core/src/mdl_core/ir.py index 21919a5..c8e1e20 100644 --- a/packages/core/src/mdl_core/ir.py +++ b/packages/core/src/mdl_core/ir.py @@ -550,6 +550,25 @@ def classification_is_empty(self) -> bool: ) +class ScaffoldConfig(_Base): + """The `scaffold:` block of mdl-project.yaml — records HOW this project was + bootstrapped so a re-import, or other tooling, reproduces the same layout instead + of guessing. Written once on a fresh import/init; user-owned thereafter (a re-run + must never rewrite it — see writer._USER_OWNED_CONFIG). + + - ``layout`` — ``model`` (a plain model repo), ``workspace`` (the collab topology: + model/ + transform/warehouse siblings, spec §2.1), or ``none`` (model objects only, + no surrounding skeleton). + - ``dbt_project_dir`` — where the generated dbt project lives, relative to the repo + root (e.g. ``transform/warehouse`` for a workspace layout); None for a plain model. + """ + + model_config = ConfigDict(extra="allow") + + layout: str = "model" + dbt_project_dir: str | None = None + + class ProjectConfig(_Base): model_config = ConfigDict(extra="allow") @@ -568,6 +587,9 @@ class ProjectConfig(_Base): # How this warehouse's dbt models map to entities (drift name resolution). Empty by # default -> drift keeps the exact-name behaviour. See ReverseConfig. reverse: ReverseConfig = Field(default_factory=ReverseConfig) + # How this project was bootstrapped (import/init layout). Default `model`; recorded so + # a re-run reproduces the layout. See ScaffoldConfig. + scaffold: ScaffoldConfig = Field(default_factory=ScaffoldConfig) # --- The in-memory graph --------------------------------------------------- diff --git a/packages/reverse/src/mdl_reverse/writer.py b/packages/reverse/src/mdl_reverse/writer.py index fda267d..d57f717 100644 --- a/packages/reverse/src/mdl_reverse/writer.py +++ b/packages/reverse/src/mdl_reverse/writer.py @@ -24,6 +24,9 @@ "ontology_stack", "platform_targets", "kg_base_iri", + # How the project was bootstrapped (import/init layout). Recorded once; a re-import + # must reproduce, not rewrite, it. + "scaffold", ) diff --git a/pyproject.toml b/pyproject.toml index 0635ee5..8a1bb2b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,7 +3,7 @@ # project, so the install name is `modelith-dbt` (install: uv tool install # modelith-dbt). The command stays `mdl`; the import modules stay mdl_*. name = "modelith-dbt" -version = "0.6.6" +version = "0.6.7" description = "Ontology-anchored, git-native data modeling tool for dbt-core teams" readme = "README.md" requires-python = ">=3.11" diff --git a/uv.lock b/uv.lock index 61e513b..994b9fd 100644 --- a/uv.lock +++ b/uv.lock @@ -939,7 +939,7 @@ requires-dist = [ [[package]] name = "modelith-dbt" -version = "0.6.6" +version = "0.6.7" source = { editable = "." } dependencies = [ { name = "dbt-artifacts-parser" }, diff --git a/vscode/package.json b/vscode/package.json index 5c44884..2446e3a 100644 --- a/vscode/package.json +++ b/vscode/package.json @@ -2,7 +2,7 @@ "name": "modelith-vscode", "displayName": "Modelith", "description": "Ontology-anchored, git-native data modeling for dbt — validate, generate, drift, and the Modelith canvas inside VS Code.", - "version": "0.3.14", + "version": "0.3.15", "publisher": "BosonResearch", "license": "Apache-2.0", "icon": "icon.png", @@ -377,6 +377,26 @@ "type": "boolean", "default": true, "description": "Run `mdl validate` when a model YAML file is saved and surface MDL-* diagnostics in Problems." + }, + "modelith.import.scaffold": { + "type": "string", + "enum": [ + "model", + "workspace", + "none" + ], + "default": "model", + "enumDescriptions": [ + "A runnable model repo (.gitignore, .mdl/lock.yaml, .mdl/state/) around the imported objects.", + "Also the transform/warehouse dbt project, CODEOWNERS and .code-workspace (collaboration topology).", + "Model objects only, no surrounding skeleton." + ], + "markdownDescription": "When importing an erwin XML into an EMPTY workspace, what to scaffold around the model. Ignored when a Modelith project already exists (its layout is respected). Persisted to `mdl-project.yaml` so a re-import reproduces it." + }, + "modelith.import.dbtTarget": { + "type": "string", + "default": "duckdb_dev", + "description": "dbt target/profile name written into a freshly scaffolded project's config (and the workspace profiles.yml). DuckDB by default so the generated warehouse builds with no setup." } } }, diff --git a/vscode/src/extension.ts b/vscode/src/extension.ts index c7e2141..dc54df3 100644 --- a/vscode/src/extension.ts +++ b/vscode/src/extension.ts @@ -623,8 +623,21 @@ export async function activate(ctx: vscode.ExtensionContext): Promise { // model (like a reverse), so this WRITES a model dir via the CLI (10MB-safe, server- // free) and opens it — not a canvas paste. Reachable from the palette, the Reverse // Review title bar, and an .xml right-click (which passes the clicked URI). - cmd("modelith.importErwin", (arg: unknown) => - withModelDir(async (dir) => { + cmd("modelith.importErwin", async (arg: unknown) => { + // Unlike the other CLI commands, erwin import does NOT require an existing model: + // importing into an empty workspace is the migration on-ramp. When a model exists + // we use it as the working dir; otherwise we bootstrap a new project at the + // workspace root (the CLI scaffolds the skeleton — see `--scaffold`). + const existingModel = await findModelDir(); + const wsRoot = vscode.workspace.workspaceFolders?.[0]?.uri.fsPath; + const dir = existingModel ?? wsRoot; + if (!dir) { + void vscode.window.showWarningMessage( + "Modelith: open a folder in VS Code first, then import the erwin XML into it.", + ); + return; + } + { let fileUri = arg instanceof vscode.Uri ? arg : undefined; if (!fileUri) { const picked = await vscode.window.showOpenDialog({ @@ -637,12 +650,30 @@ export async function activate(ctx: vscode.ExtensionContext): Promise { fileUri = picked[0]; } const file = fileUri.fsPath; + + // No model here yet → confirm we're bootstrapping a fresh project, and honour the + // configured scaffold layout / dbt target. + const cfg = vscode.workspace.getConfiguration("modelith"); + const scaffold = cfg.get("import.scaffold") || "model"; + const dbtTarget = cfg.get("import.dbtTarget") || "duckdb_dev"; + if (!existingModel) { + const GO = scaffold === "workspace" ? "Scaffold workspace & import" : "Scaffold & import"; + const choice = await vscode.window.showInformationMessage( + "No Modelith project here yet. Import the erwin model and scaffold a new " + + `project (layout: ${scaffold})?`, + { modal: true }, + GO, + ); + if (choice !== GO) return; + } + // Default the target at the workspace's model dir; the picker confirms/redirects // (reusing the reverse no-clobber "update in place vs new folder" flow). const res = await resolveReverseTarget(await bestDefaultTarget(dir), undefined); if (!res) return; const bin = await findMdl(dir); const args = ["import", "erwin", file, "-o", res.target]; + if (!existingModel) args.push("--scaffold", scaffold, "--dbt-target", dbtTarget); if (res.force) args.push("--force"); await vscode.window.withProgress( { location: vscode.ProgressLocation.Notification, title: "Modelith: importing erwin XML…" }, @@ -657,9 +688,22 @@ export async function activate(ctx: vscode.ExtensionContext): Promise { // the CLI diverts on an existing model; read the actual written dir from the output const diverted = /reversing into (.+?) instead/.exec(r.stdout); const writtenRel = diverted ? diverted[1].trim() : res.target; - const writtenDir = path.isAbsolute(writtenRel) + let writtenDir = path.isAbsolute(writtenRel) ? writtenRel : path.join(dir, writtenRel); + // A workspace scaffold puts the model under model/; the mdl-project.yaml lives + // there, so point the panels (and modelith.modelDir) at it, not the repo root. + if (!existingModel && scaffold === "workspace") { + writtenDir = path.join(writtenDir, "model"); + } + // A fresh scaffold with no prior active model: record it so findModelDir and the + // panels resolve it everywhere (mirrors modelith.setActiveModel). + if (!existingModel && wsRoot) { + const rel = path.relative(wsRoot, writtenDir); + await vscode.workspace + .getConfiguration("modelith") + .update("modelDir", rel || ".", vscode.ConfigurationTarget.Workspace); + } const notes = [...r.stdout.matchAll(/note: (.+)/g)].map((m) => m[1]); await Promise.all([reverse.refresh(writtenDir), statusModel.refresh(writtenDir)]); await canvas.open(writtenDir); @@ -669,8 +713,8 @@ export async function activate(ctx: vscode.ExtensionContext): Promise { ); }, ); - }), - ); + } + }); cmd("modelith.stopServer", () => canvas.stop()); From 854369a643800d4fb190693e46256796f1d3b12b Mon Sep 17 00:00:00 2001 From: Debasish Bose Date: Sun, 27 Sep 2026 10:54:52 +1000 Subject: [PATCH 2/2] import/generate: model-only import by default, CLI-editable scaffold config, generate follows it Splits the concerns the erwin-scaffold work had coupled. Creating the logical model and generating the dbt project are now distinct steps, the project's dbt-project location is one config value editable from the CLI, and generate honours it. - mdl import erwin defaults to a MODEL-ONLY project (--scaffold model): the .mdl skeleton + object YAMLs, no dbt shell. --scaffold workspace still lays (or reuses) a dbt project; --scaffold none writes objects only. Producing the dbt project is otherwise a separate mdl generate step, matching mdl init (plain = model-only; --workspace/--demo = model + dbt). - New : the general mdl-project.yaml doorway (the reverse: block keeps its richer reverse-config surface). Comment-preserving, dotted keys, and the whole doc is re-validated through ProjectConfig before writing so a bad set is rejected, not half-applied. e.g. mdl config set scaffold.dbt_project_dir transform/warehouse - mdl generate now defaults its output to config.scaffold.dbt_project_dir (resolved relative to the model dir) when set, else target/dbt; an explicit -o still wins. The key import stamps is no longer dead. - VS Code: importErwin shows a layout QuickPick (model|workspace|none) on a fresh bootstrap, seeded by modelith.import.scaffold, instead of a confirm-only modal. CLI 0.6.8, extension 0.3.16. --- README.md | 23 +-- packages/cli/src/mdl_cli/collab.py | 40 +++-- packages/cli/src/mdl_cli/main.py | 165 ++++++++++++++++-- packages/cli/src/mdl_cli/scaffold.py | 95 +++++++--- packages/cli/tests/test_config_cli.py | 76 ++++++++ .../cli/tests/test_generate_output_dir.py | 45 +++++ packages/cli/tests/test_import_erwin_cli.py | 27 +++ pyproject.toml | 2 +- ...odelith-import-scaffold-generation-spec.md | 132 ++++++++++++++ uv.lock | 2 +- vscode/package.json | 4 +- vscode/src/extension.ts | 39 +++-- 12 files changed, 574 insertions(+), 76 deletions(-) create mode 100644 packages/cli/tests/test_config_cli.py create mode 100644 packages/cli/tests/test_generate_output_dir.py create mode 100644 specs/modelith-import-scaffold-generation-spec.md diff --git a/README.md b/README.md index e0efb99..6ed05ed 100644 --- a/README.md +++ b/README.md @@ -467,25 +467,28 @@ foreign keys, relationships (with their verb phrases), subtype/supertype categor subject areas, and domains, keeping the physical names alongside the logical ones: ```bash -mdl import erwin your-model.xml -o proj # empty folder -> a full, runnable project -mdl import erwin your-model.xml -o proj --scaffold workspace # + a transform/warehouse dbt project +mdl import erwin your-model.xml -o proj # empty folder -> a runnable model-only project +mdl import erwin your-model.xml -o proj --scaffold workspace # also lay a dbt project mdl import erwin your-model.xml -o model --apply # or merge into a model you already have ``` -Point it at an **empty folder** and it bootstraps a runnable project around the import — the +Point it at an **empty folder** and it bootstraps a model project around the import — the `.gitignore`, the pinned `.mdl/lock.yaml`, a populated `mdl-project.yaml` — so there is no -separate `mdl init` step. `--scaffold workspace` also lays down the `transform/warehouse` -dbt project (with a zero-setup DuckDB `profiles.yml`), CODEOWNERS and a `.code-workspace`. -The chosen layout is recorded in `mdl-project.yaml`, and if the folder **already** holds a -Modelith project the existing structure and config are respected (no re-scaffold). Both the -layout and the dbt target are configurable from the CLI (`--scaffold`, `--dbt-target`) and -from VS Code settings (`modelith.import.scaffold`, `modelith.import.dbtTarget`). +separate `mdl init` step. Import owns the **logical model**; producing the dbt project is a +separate step (`mdl generate`), matching the rest of the tool. `--scaffold workspace` lays +the dbt project at import time too — `transform/warehouse` with a zero-setup DuckDB +`profiles.yml`, or, if the folder **already** has a dbt project, it reuses that one instead of +scaffolding a throwaway. The chosen layout and the dbt project location are recorded in +`mdl-project.yaml` (under `scaffold:`); `mdl generate` then defaults its output there, and you +can change it any time with `mdl config set scaffold.dbt_project_dir `. If the folder +already holds a Modelith project the existing structure and config are respected (no +re-scaffold). It parses the export by streaming, so 7-10MB files import without a memory spike, and it degrades gracefully: anything Modelith doesn't model (views, ER-diagram layout, physical transforms) is skipped with a note rather than a failure. In VS Code it's on the palette (**Modelith: Import Erwin XML**), a button in the Reverse Review panel, and a right-click -on any `.xml` file — and it now works in an empty workspace, offering to scaffold a project. +on any `.xml` file — and in an empty workspace it offers a layout picker before scaffolding. ## Reverse engineering a real warehouse diff --git a/packages/cli/src/mdl_cli/collab.py b/packages/cli/src/mdl_cli/collab.py index d47c373..65d001e 100644 --- a/packages/cli/src/mdl_cli/collab.py +++ b/packages/cli/src/mdl_cli/collab.py @@ -151,12 +151,12 @@ def _git(*args: str) -> bool: {{ "folders": [ {{ "name": "model", "path": "{model}" }}, - {{ "name": "transform", "path": "transform/warehouse" }}, + {{ "name": "transform", "path": "{dbt}" }}, {{ "name": "repo", "path": "." }} ], "settings": {{ "modelith.modelDir": "{model}", - "modelith.dbtProjectDir": "transform/warehouse" + "modelith.dbtProjectDir": "{dbt}" }} }} """ @@ -203,24 +203,36 @@ def scaffold_workspace(root: Path, project_name: str, scaffold_model) -> list[st def scaffold_workspace_skeleton( - root: Path, project_name: str, *, dbt_target: str = "duckdb_dev" + root: Path, + project_name: str, + *, + dbt_target: str = "duckdb_dev", + existing_dbt: bool = False, + dbt_project_dir: str = "transform/warehouse", ) -> list[str]: """The collaboration topology WITHOUT scaffolding a seed model under model/ (§2.1): the transform/warehouse dbt project, ontologies/, CODEOWNERS, the .code-workspace and the git merge drivers. Used by an import that supplies its own model objects, so it - lays down the plumbing and leaves model/ for the importer to write into. Idempotent.""" + lays down the plumbing and leaves model/ for the importer to write into. Idempotent. + + ``existing_dbt`` True means the workspace ALREADY has a dbt project (found elsewhere), + so we do NOT scaffold a throwaway transform/warehouse DuckDB project or a profiles.yml + on top of it — the model is wired to the real project instead.""" written: list[str] = [] model_root = "model" - tw = root / "transform" / "warehouse" - (tw / "models" / "staging").mkdir(parents=True, exist_ok=True) - (tw / "macros").mkdir(parents=True, exist_ok=True) - if not (tw / "dbt_project.yml").exists(): - (tw / "dbt_project.yml").write_text(_DBT_PROJECT_STUB) - written.append("transform/warehouse/ (dbt project)") - if not (tw / "profiles.yml").exists(): - (tw / "profiles.yml").write_text(_PROFILES_STUB.format(target=dbt_target)) - written.append("transform/warehouse/profiles.yml") + if not existing_dbt: + tw = root / "transform" / "warehouse" + (tw / "models" / "staging").mkdir(parents=True, exist_ok=True) + (tw / "macros").mkdir(parents=True, exist_ok=True) + if not (tw / "dbt_project.yml").exists(): + (tw / "dbt_project.yml").write_text(_DBT_PROJECT_STUB) + written.append("transform/warehouse/ (dbt project)") + if not (tw / "profiles.yml").exists(): + (tw / "profiles.yml").write_text(_PROFILES_STUB.format(target=dbt_target)) + written.append("transform/warehouse/profiles.yml") + else: + written.append("(reusing the existing dbt project — no DuckDB project scaffolded)") (root / "ontologies" / "industry").mkdir(parents=True, exist_ok=True) @@ -232,7 +244,7 @@ def scaffold_workspace_skeleton( ws = root / f"{project_name}.code-workspace" if not ws.exists(): - ws.write_text(_WORKSPACE.format(model=model_root)) + ws.write_text(_WORKSPACE.format(model=model_root, dbt=dbt_project_dir)) written.append(ws.name) written += ensure_git_hooks(root, model_root) diff --git a/packages/cli/src/mdl_cli/main.py b/packages/cli/src/mdl_cli/main.py index ce3dcd5..0b55efe 100644 --- a/packages/cli/src/mdl_cli/main.py +++ b/packages/cli/src/mdl_cli/main.py @@ -296,7 +296,13 @@ def lint( def generate( model_dir: Path = typer.Option(Path("."), "--model-dir", "-m"), target: str = typer.Option(None, "--target", "-t", help="Physical target"), - out: Path = typer.Option(Path("target/dbt"), "--out", "-o", help="dbt project output dir"), + out: Path = typer.Option( + None, + "--out", + "-o", + help="dbt project output dir. Defaults to the project's scaffold.dbt_project_dir " + "when set (so it follows where the project was scaffolded), else target/dbt.", + ), inline: bool = typer.Option(False, "--inline", help="Inline pattern SQL instead of macros"), emit_contract: bool = typer.Option( False, "--emit-contract", help="Also write an ODCS datacontract.yaml at the model root" @@ -321,6 +327,12 @@ def generate( """ repo = _load(model_dir) tgt = target or repo.model.config.dbt_target or "duckdb_dev" + # Where to write: an explicit -o wins; otherwise follow where the project was + # scaffolded (scaffold.dbt_project_dir, recorded RELATIVE to the model dir), else the + # historical target/dbt default (relative to CWD, as before). + if out is None: + scaffold_dir = repo.model.config.scaffold.dbt_project_dir + out = (model_dir / scaffold_dir) if scaffold_dir else Path("target/dbt") emitter = DbtEmitter(repo.model, tgt, inline=inline) result = emitter.generate(out, write=not dry_run) @@ -2534,6 +2546,111 @@ def reverse_config_import( typer.secho(f"imported reverse: config from {src}", fg=typer.colors.GREEN) +# --- project config: read/write mdl-project.yaml keys from the CLI ----------------- +# The general config doorway (the `reverse:` block has its own richer `reverse-config` +# surface above; this handles the rest — dbt_target, platform_targets, the scaffold +# block, etc.). Comment-preserving, and the whole doc is re-validated through +# ProjectConfig before writing so a bad set is rejected, not half-applied. +config_app = typer.Typer(help="Read and write mdl-project.yaml config keys.") +app.add_typer(config_app, name="config") + +# Known keys and how a CLI string value coerces onto them. Anything not listed is set +# as a plain string (still validated against ProjectConfig, so a wrong type is caught). +_CONFIG_LIST_KEYS = {"platform_targets"} + + +def _coerce_config_value(dotted: str, raw: str): + """Turn a CLI string into the right shape for a known key: a comma-split list for + list-valued keys, an empty string clears (None) for the scaffold's optional dir.""" + if dotted in _CONFIG_LIST_KEYS: + return [v.strip() for v in raw.split(",") if v.strip()] + if dotted == "scaffold.dbt_project_dir" and raw == "": + return None + return raw + + +def _dig(doc, dotted: str): + """Read a dotted key from a loaded doc, or None if any segment is missing.""" + cur = doc + for part in dotted.split("."): + if not hasattr(cur, "get") or part not in cur: + return None + cur = cur[part] + return cur + + +def _set_config_key(doc, dotted: str, value) -> None: + """Set a dotted key on a loaded (comment-preserving) doc, creating intermediate maps.""" + parts = dotted.split(".") + cur = doc + for part in parts[:-1]: + if part not in cur or not hasattr(cur[part], "get"): + cur[part] = {} + cur = cur[part] + cur[parts[-1]] = value + + +@config_app.command("get") +def config_get( + key: str = typer.Argument( + None, help="Dotted key, e.g. scaffold.dbt_project_dir. Omit to print the whole config." + ), + model_dir: Path = typer.Option(Path("."), "--model-dir", "-m"), +) -> None: + """Print the project config, or a single dotted key (scaffold.dbt_project_dir, dbt_target…).""" + from mdl_core.yaml_io import dump_str, load_file + + proj_path = model_dir / "mdl-project.yaml" + if not proj_path.exists(): + typer.secho(f"no mdl-project.yaml in {model_dir}", fg=typer.colors.RED, err=True) + raise typer.Exit(1) + doc = load_file(proj_path) + if key is None: + typer.echo(dump_str(doc).rstrip()) + return + val = _dig(doc, key) + if val is None: + typer.secho(f"{key} is not set", fg=typer.colors.YELLOW) + return + typer.echo(dump_str(val).rstrip() if hasattr(val, "get") or isinstance(val, list) else str(val)) + + +@config_app.command("set") +def config_set( + key: str = typer.Argument(..., help="Dotted key, e.g. scaffold.dbt_project_dir"), + value: str = typer.Argument(..., help="New value. Comma-separated for list keys; '' clears."), + model_dir: Path = typer.Option(Path("."), "--model-dir", "-m"), +) -> None: + """Set an mdl-project.yaml config key (comment-preserving). The whole config is + re-validated through ProjectConfig before writing, so a bad key or wrong type is + rejected rather than half-applied. + + Examples: + mdl config set scaffold.dbt_project_dir transform/warehouse + mdl config set dbt_target snowflake_prod + mdl config set platform_targets duckdb_dev,snowflake_prod + """ + from mdl_core.ir import ProjectConfig + from mdl_core.yaml_io import dump_file, load_file + + proj_path = model_dir / "mdl-project.yaml" + if not proj_path.exists(): + typer.secho(f"no mdl-project.yaml in {model_dir}", fg=typer.colors.RED, err=True) + raise typer.Exit(1) + doc = load_file(proj_path) + _set_config_key(doc, key, _coerce_config_value(key, value)) + # validate the resulting doc before writing — reject a bad set wholesale. + try: + ProjectConfig.model_validate(dict(doc)) + except Exception as e: # noqa: BLE001 - surface the validation error, don't half-write + typer.secho( + f"resulting config is invalid — not written ({e})", fg=typer.colors.RED, err=True + ) + raise typer.Exit(1) from e + dump_file(proj_path, doc) + typer.secho(f"set {key} = {value!r}", fg=typer.colors.GREEN) + + @emit_app.command("semantic") def emit_semantic( fmt: str = typer.Option("metricflow", "--format", help="metricflow|osi"), @@ -2733,10 +2850,12 @@ def import_erwin_cmd( scaffold_layout: str = typer.Option( "model", "--scaffold", - help="When --out is EMPTY, bootstrap a full project around the import: " - "`model` (a runnable model repo — .gitignore, .mdl/lock.yaml, .mdl/state/), " - "`workspace` (also the transform/warehouse dbt project, CODEOWNERS, " - ".code-workspace — collab §2.1), or `none` (model objects only, no skeleton). " + help="When --out is EMPTY, what to bootstrap around the imported model. " + "`model` (DEFAULT) — a runnable model repo only (.gitignore, .mdl/lock.yaml, " + ".mdl/state/); generate the dbt project separately with `mdl generate`. " + "`workspace` — also lay a dbt project (transform/warehouse, or reuse one already " + "in the folder), CODEOWNERS and a .code-workspace (collab §2.1). " + "`none` — model object YAMLs only, no skeleton. " "Ignored when --out already holds a project (that layout is respected).", ), dbt_target: str = typer.Option( @@ -2758,13 +2877,14 @@ def import_erwin_cmd( ) -> None: """Import a real erwin XML export into a Modelith model (spec §6.4). - Default: write a fresh model at --out (like `mdl reverse` — an erwin export is a whole - model). Into an EMPTY --out it also scaffolds a runnable project around the objects - (`--scaffold model|workspace|none`, persisted to mdl-project.yaml so a re-import - reproduces the layout). If --out already holds a project the existing structure is - RESPECTED — no re-scaffold, user config preserved — and it diverts to a `-reversed-v` - sibling unless --force. With --apply, merge the erwin objects into the existing model at - --out through the same validated mutation engine manual edits use.""" + Default: write a fresh, model-only project at --out (like `mdl reverse` — an erwin export + is a whole model). Producing the dbt project is a SEPARATE step (`mdl generate`, or + `--scaffold workspace` to lay the dbt shell here at import time). The chosen layout is + persisted to mdl-project.yaml (scaffold:) so `mdl generate` and a re-import follow it. If + --out already holds a project the existing structure is RESPECTED — no re-scaffold, user + config preserved — and it diverts to a `-reversed-v` sibling unless --force. With + --apply, merge the erwin objects into the existing model at --out through the same + validated mutation engine manual edits use.""" # streaming reader handles 7-10MB files; pass the PATH so it reads the file directly. result = import_erwin(file, project_name=name) for w in result.warnings: @@ -2799,15 +2919,27 @@ def import_erwin_cmd( from mdl_cli.scaffold import bootstrap_project - write_target, scaffolded = bootstrap_project( + boot = bootstrap_project( out, project_name=name or _project_name_from_path(out), layout=scaffold_layout, dbt_target=dbt_target, existing=existing, ) - for line in scaffolded: + write_target = boot.model_root + for line in boot.written: typer.secho(f" scaffolded {line}", fg=typer.colors.GREEN) + reused_dbt = ( + not existing + and scaffold_layout == "workspace" + and boot.dbt_project_dir not in (None, "transform/warehouse") + ) + if reused_dbt: + typer.secho( + f" found an existing dbt project at {boot.dbt_project_dir}; wiring the model " + "to it instead of scaffolding a DuckDB one", + fg=typer.colors.CYAN, + ) # Stamp the config so the layout persists (writer._USER_OWNED_CONFIG keeps it on a # re-import). Only on a fresh scaffold — an existing project owns its own config. @@ -2815,8 +2947,9 @@ def import_erwin_cmd( result.model.config.dbt_target = dbt_target result.model.config.platform_targets = [dbt_target] result.model.config.scaffold.layout = scaffold_layout - if scaffold_layout == "workspace": - result.model.config.scaffold.dbt_project_dir = "transform/warehouse" + # Point the config at the real dbt project (a discovered one, else the scaffolded + # transform/warehouse) so we never wire to a phantom DuckDB. + result.model.config.scaffold.dbt_project_dir = boot.dbt_project_dir if not force: write_target = _nonclobbering_out(write_target) diff --git a/packages/cli/src/mdl_cli/scaffold.py b/packages/cli/src/mdl_cli/scaffold.py index 0414967..110059b 100644 --- a/packages/cli/src/mdl_cli/scaffold.py +++ b/packages/cli/src/mdl_cli/scaffold.py @@ -122,6 +122,20 @@ def scaffold(root: Path, *, project_name: str = "modelith_model") -> dict[str, s return files +def find_dbt_project(root: Path) -> Path | None: + """The directory of an existing dbt project under `root` (the one holding + dbt_project.yml), or None. Skips build/vendor dirs. Shallowest match wins, so a + project at the root or a top-level dir beats one nested in a package. Used so an + import into a workspace that ALREADY has a dbt project never scaffolds a second, + throwaway DuckDB project on top of it.""" + skip = {"node_modules", "dbt_packages", "target", ".venv", ".git", "__pycache__"} + hits = [p for p in root.rglob("dbt_project.yml") if not (set(p.parts) & skip)] + if not hits: + return None + hits.sort(key=lambda p: len(p.parts)) + return hits[0].parent + + def _scaffold_skeleton(root: Path) -> dict[str, str]: """The tool-owned skeleton every project needs, WITHOUT any seed model objects: the .gitignore, the empty .mdl/state/ dir, and a pinned .mdl/lock.yaml. Shared by @@ -149,6 +163,25 @@ def _scaffold_skeleton(root: Path) -> dict[str, str]: return written +class Bootstrap: + """Result of ``bootstrap_project``. + + - ``model_root`` — where the importer writes its object YAMLs. + - ``written`` — human-readable scaffolded paths, for the CLI summary. + - ``dbt_project_dir`` — the dbt project the model config should point at + (relative to ``root``), or None when there is no dbt project. Set to a + DISCOVERED existing project when one is present (so no second, throwaway DuckDB + project is created), else the freshly scaffolded ``transform/warehouse``. + """ + + def __init__( + self, model_root: Path, written: list[str], dbt_project_dir: str | None + ) -> None: + self.model_root = model_root + self.written = written + self.dbt_project_dir = dbt_project_dir + + def bootstrap_project( root: Path, *, @@ -156,37 +189,57 @@ def bootstrap_project( layout: str = "model", dbt_target: str = "duckdb_dev", existing: bool = False, -) -> tuple[Path, list[str]]: +) -> Bootstrap: """Lay down the project skeleton an IMPORT (erwin, etc.) writes its objects into, - without any seed example objects. Returns (model_root, written_paths): + without any seed example objects. - - ``model_root`` is where the importer should write its object YAMLs. For a - ``workspace`` layout that is ``root/model`` (the collab §2.1 sibling); otherwise - ``root``. - ``existing`` True means the target already holds a Modelith project (an ``mdl-project.yaml`` or a ``.mdl/`` dir): scaffold NOTHING structural and RESPECT - what is there — the caller merges objects and the writer preserves the user's - config. Returns (root, []). - - ``layout == "none"`` writes no skeleton (model objects only, the pre-scaffold - behaviour). Returns (root, []). - - ``layout == "model"`` writes the shared skeleton (.gitignore, .mdl/state, - .mdl/lock.yaml) at ``root``. - - ``layout == "workspace"`` also lays down the transform/warehouse dbt project, - CODEOWNERS, the .code-workspace and git merge drivers (collab §2.1), with the - model under ``root/model``. + what is there — the caller merges objects and the writer preserves the user's config. + - ``layout == "none"`` writes no skeleton (model objects only, pre-scaffold behaviour). + - ``layout == "model"`` writes the shared skeleton (.gitignore, .mdl/state, .mdl/lock.yaml). + - ``layout == "workspace"`` also lays down transform/warehouse + CODEOWNERS + the + .code-workspace + merge drivers (collab §2.1), with the model under ``root/model``. + BUT if a dbt project already exists anywhere under ``root``, that project is reused + (config points at it) and NO throwaway DuckDB dbt project / profiles.yml is written. """ if existing or layout == "none": - # An existing WORKSPACE keeps its model under model/; a plain project at root. nested = existing and (root / "model" / "mdl-project.yaml").exists() - return (root / "model" if nested else root), [] + model_root = root / "model" if nested else root + # Respect any dbt project the workspace already has (don't invent a target). + found = find_dbt_project(root) + dbt_dir = _rel(found, root) if found else None + return Bootstrap(model_root, [], dbt_dir) if layout == "workspace": from mdl_cli.collab import scaffold_workspace_skeleton model_root = root / "model" - written = scaffold_workspace_skeleton(root, project_name, dbt_target=dbt_target) + # A dbt project already in the workspace wins — reuse it, don't scaffold a dumb + # DuckDB one on top. Only when there is none do we lay down transform/warehouse. + found = find_dbt_project(root) + dbt_dir = _rel(found, root) if found else "transform/warehouse" + written = scaffold_workspace_skeleton( + root, + project_name, + dbt_target=dbt_target, + existing_dbt=found is not None, + dbt_project_dir=dbt_dir, + ) written.extend(f"model/{p}" for p in _scaffold_skeleton(model_root)) - return model_root, written - - # layout == "model" (default): a plain model repo skeleton at root. - return root, list(_scaffold_skeleton(root)) + return Bootstrap(model_root, written, dbt_dir) + + # layout == "model" (default): a plain model repo skeleton at root. Point at an + # existing dbt project if there is one; a plain model doesn't scaffold its own. + found = find_dbt_project(root) + dbt_dir = _rel(found, root) if found else None + return Bootstrap(root, list(_scaffold_skeleton(root)), dbt_dir) + + +def _rel(target: Path, root: Path) -> str: + """`target` relative to `root` as a forward-slash string, or the path as-is when it + is not under `root` (e.g. a sibling dbt project).""" + try: + return target.resolve().relative_to(root.resolve()).as_posix() + except ValueError: + return target.as_posix() diff --git a/packages/cli/tests/test_config_cli.py b/packages/cli/tests/test_config_cli.py new file mode 100644 index 0000000..0ec60c3 --- /dev/null +++ b/packages/cli/tests/test_config_cli.py @@ -0,0 +1,76 @@ +"""CLI: `mdl config get/set` — the general mdl-project.yaml config doorway.""" + +from __future__ import annotations + +from typer.testing import CliRunner + +from mdl_cli.main import app + +runner = CliRunner() + + +def _init(tmp_path): + d = tmp_path / "proj" + assert runner.invoke(app, ["init", str(d)]).exit_code == 0 + return d + + +def test_config_set_get_scaffold_dir(tmp_path): + d = _init(tmp_path) + r = runner.invoke( + app, ["config", "set", "scaffold.dbt_project_dir", "transform/warehouse", "-m", str(d)] + ) + assert r.exit_code == 0, r.output + g = runner.invoke(app, ["config", "get", "scaffold.dbt_project_dir", "-m", str(d)]) + assert g.exit_code == 0, g.output + assert g.output.strip() == "transform/warehouse" + + +def test_config_set_scalar_and_list(tmp_path): + d = _init(tmp_path) + assert runner.invoke(app, ["config", "set", "dbt_target", "snowflake_prod", "-m", str(d)]).exit_code == 0 + assert ( + runner.invoke( + app, ["config", "set", "platform_targets", "duckdb_dev,snowflake_prod", "-m", str(d)] + ).exit_code + == 0 + ) + from mdl_core.yaml_io import load_file + + cfg = load_file(d / "mdl-project.yaml") + assert cfg["dbt_target"] == "snowflake_prod" + assert list(cfg["platform_targets"]) == ["duckdb_dev", "snowflake_prod"] + + +def test_config_set_rejects_invalid_type(tmp_path): + d = _init(tmp_path) + # platform_targets must be a list; force a scalar via a key the coercion leaves as string + # by targeting a field ProjectConfig types as a list — a bad type is rejected. + r = runner.invoke(app, ["config", "set", "ontology_stack", "not-a-list", "-m", str(d)]) + assert r.exit_code == 1 + assert "invalid" in r.output.lower() + + +def test_config_set_preserves_comments(tmp_path): + d = _init(tmp_path) + before = (d / "mdl-project.yaml").read_text() + assert "#" in before # the scaffolded config is richly commented + runner.invoke(app, ["config", "set", "dbt_target", "bigquery_dev", "-m", str(d)]) + after = (d / "mdl-project.yaml").read_text() + assert "#" in after # comments survive the comment-preserving write + assert "bigquery_dev" in after + + +def test_config_get_whole_doc(tmp_path): + d = _init(tmp_path) + r = runner.invoke(app, ["config", "get", "-m", str(d)]) + assert r.exit_code == 0 + assert "name:" in r.output + + +def test_config_clear_scaffold_dir(tmp_path): + d = _init(tmp_path) + runner.invoke(app, ["config", "set", "scaffold.dbt_project_dir", "x", "-m", str(d)]) + runner.invoke(app, ["config", "set", "scaffold.dbt_project_dir", "", "-m", str(d)]) + g = runner.invoke(app, ["config", "get", "scaffold.dbt_project_dir", "-m", str(d)]) + assert "not set" in g.output diff --git a/packages/cli/tests/test_generate_output_dir.py b/packages/cli/tests/test_generate_output_dir.py new file mode 100644 index 0000000..4807833 --- /dev/null +++ b/packages/cli/tests/test_generate_output_dir.py @@ -0,0 +1,45 @@ +"""CLI: `mdl generate` output-dir resolution — explicit -o wins, else +scaffold.dbt_project_dir, else target/dbt.""" + +from __future__ import annotations + +from typer.testing import CliRunner + +from mdl_cli.main import app + +runner = CliRunner() + + +def _init(tmp_path): + d = tmp_path / "proj" + assert runner.invoke(app, ["init", str(d)]).exit_code == 0 + return d + + +def test_generate_defaults_to_scaffold_dbt_project_dir(tmp_path): + d = _init(tmp_path) + runner.invoke(app, ["config", "set", "scaffold.dbt_project_dir", "transform/warehouse", "-m", str(d)]) + r = runner.invoke(app, ["generate", "-m", str(d)]) + assert r.exit_code == 0, r.output + # models were written under the configured dir, not target/dbt + assert (d / "transform" / "warehouse" / "models").is_dir() + assert not (d / "target" / "dbt").exists() + + +def test_generate_falls_back_to_target_dbt(tmp_path, monkeypatch): + d = _init(tmp_path) # no scaffold.dbt_project_dir set + # the historical fallback is target/dbt relative to CWD; run from the project dir + monkeypatch.chdir(d) + r = runner.invoke(app, ["generate", "-m", str(d)]) + assert r.exit_code == 0, r.output + assert (d / "target" / "dbt" / "models").is_dir() + + +def test_generate_explicit_out_overrides(tmp_path): + d = _init(tmp_path) + runner.invoke(app, ["config", "set", "scaffold.dbt_project_dir", "transform/warehouse", "-m", str(d)]) + out = d / "custom" + r = runner.invoke(app, ["generate", "-m", str(d), "-o", str(out)]) + assert r.exit_code == 0, r.output + assert (out / "models").is_dir() + assert not (d / "transform" / "warehouse").exists() diff --git a/packages/cli/tests/test_import_erwin_cli.py b/packages/cli/tests/test_import_erwin_cli.py index f231e90..f9444de 100644 --- a/packages/cli/tests/test_import_erwin_cli.py +++ b/packages/cli/tests/test_import_erwin_cli.py @@ -104,6 +104,33 @@ def test_import_erwin_workspace_layout(tmp_path): assert runner.invoke(app, ["validate", "-m", str(out / "model")]).exit_code == 0 +def test_import_erwin_workspace_reuses_existing_dbt_project(tmp_path): + # a workspace that already has a dbt project must NOT get a second dumb DuckDB one + xml = tmp_path / "trading.xml" + xml.write_text(_erwin_xml()) + out = tmp_path / "repo" + out.mkdir() + # the user's real dbt project lives at dbt/ with a Snowflake-ish profile + (out / "dbt").mkdir() + (out / "dbt" / "dbt_project.yml").write_text( + "name: analytics\nversion: '1.0.0'\nprofile: analytics\n" + ) + r = runner.invoke( + app, ["import", "erwin", str(xml), "-o", str(out), "--scaffold", "workspace"] + ) + assert r.exit_code == 0, r.output + # NO throwaway transform/warehouse project/profiles was scaffolded + assert not (out / "transform" / "warehouse" / "dbt_project.yml").exists() + assert not (out / "transform" / "warehouse" / "profiles.yml").exists() + # the model is wired to the real dbt project, not a phantom DuckDB one + from mdl_core.yaml_io import load_file + + cfg = load_file(out / "model" / "mdl-project.yaml") + assert dict(cfg["scaffold"])["dbt_project_dir"] == "dbt" + assert "existing dbt project" in r.output + assert runner.invoke(app, ["validate", "-m", str(out / "model")]).exit_code == 0 + + def test_import_erwin_none_layout_no_skeleton(tmp_path): # --scaffold none → model objects only, no .mdl skeleton (pre-scaffold behaviour) xml = tmp_path / "trading.xml" diff --git a/pyproject.toml b/pyproject.toml index 8a1bb2b..c4ab6ab 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,7 +3,7 @@ # project, so the install name is `modelith-dbt` (install: uv tool install # modelith-dbt). The command stays `mdl`; the import modules stay mdl_*. name = "modelith-dbt" -version = "0.6.7" +version = "0.6.8" description = "Ontology-anchored, git-native data modeling tool for dbt-core teams" readme = "README.md" requires-python = ">=3.11" diff --git a/specs/modelith-import-scaffold-generation-spec.md b/specs/modelith-import-scaffold-generation-spec.md new file mode 100644 index 0000000..af1f90d --- /dev/null +++ b/specs/modelith-import-scaffold-generation-spec.md @@ -0,0 +1,132 @@ +# Separate model import from dbt generation; make the scaffold config CLI-editable and generate-aware + +## Context + +The erwin-scaffold work (branch `feat-erwin-scaffold`, already committed) made `mdl import erwin` +into an empty folder bootstrap a full project, defaulting to a `workspace` layout that also lays a +DuckDB `transform/warehouse` dbt shell. Review of the codebase (validated) shows this over-couples +concerns and leaves a dead config key: + +- The tool already has a clean split: model creation (`scaffold`/`bootstrap_project`) vs dbt-project + SHELL (`collab.scaffold_workspace_skeleton`) vs dbt MODELS (`mdl generate`, which writes .sql + + schema.yml into a dbt dir but never creates the shell — [emitter.py:114](packages/emit-dbt/src/mdl_emit_dbt/emitter.py:114)). +- Convention from `mdl init`: plain init = model-only; `--workspace`/`--demo` = model + dbt shell. +- Gap 1: import stamps `config.scaffold.dbt_project_dir`, but `mdl generate` ignores it (reads only + `--out`, default `target/dbt` — [main.py:299](packages/cli/src/mdl_cli/main.py:299)). The key is dead. +- Gap 2: no global config-edit doorway. Only `reverse-config apply` writes config, hard-scoped to the + `reverse:` block ([main.py:2306](packages/cli/src/mdl_cli/main.py:2306)). The `scaffold` block can only + be set at import time, never edited afterward. + +Decisions (from the user): +1. **Default layout = `model`** (model-only, no dbt shell). dbt generation stays a separate, explicit + step. In **VS Code**, surface the layout as a QuickPick at import time (not just a setting). +2. **New general `mdl config` command** (`get`/`set`), reusing the comment-preserving merge machinery, + so `scaffold.dbt_project_dir` / `scaffold.layout` / `dbt_target` are editable globally after import. +3. **`mdl generate` honors `config.scaffold.dbt_project_dir`** as its default `-o` (explicit `-o` wins), + so the stamped value drives generation. + +Intended outcome: importing an erwin export produces a logical model by default; a dbt project is an +opt-in, separate concern; where that dbt project lives is one config value, editable from the CLI and +consumed by both `generate` and the VS Code workspace wiring. + +## Deliverables + +### A. Flip the default layout to `model` — `packages/cli/src/mdl_cli/main.py` +`import_erwin_cmd` `--scaffold` default `"model"` (already `model` in code — CONFIRM and keep). +Ensure the docstring/help states model-only is the default and dbt generation is a separate step +(`mdl generate`). No behavioural change for `workspace`/`none`; this is the documented contract. + +### B. New `mdl config` command group — `packages/cli/src/mdl_cli/main.py` +Add a `config` typer sub-app mirroring `reverse-config`'s structure, backed by a small generic +merge helper (reuse the `load_file`/`dump_file` comment-preserving round-trip and pydantic validation +pattern from `_write_reverse_block`, [main.py:2275](packages/cli/src/mdl_cli/main.py:2275)): +- `mdl config get [KEY] [-m DIR]` — print the whole `mdl-project.yaml` (or a dotted key like + `scaffold.dbt_project_dir`). Read-only. +- `mdl config set KEY VALUE [-m DIR]` — set a dotted key, comment-preserving, then validate the whole + doc through `ProjectConfig.model_validate` before writing (reject a bad set wholesale, like + `_write_reverse_block` does for ReverseConfig). Support at least `dbt_target`, `platform_targets` + (comma-split list), `scaffold.layout`, `scaffold.dbt_project_dir`. +- Keep it general (dotted-path set on the loaded doc) but validate against `ProjectConfig` so a typo'd + key or wrong type is caught. A new helper `_set_config_key(doc, dotted, value)` walks/creates nested + maps; `_coerce_config_value(dotted, raw)` handles list/bool/None for known keys. +Register: `app.add_typer(config_app, name="config")`. + +### C. `mdl generate` honors `scaffold.dbt_project_dir` — `packages/cli/src/mdl_cli/main.py:295` +Change the `--out` option default from `Path("target/dbt")` to `None`; resolve inside the body: +```python +out = out or (Path(repo.model.config.scaffold.dbt_project_dir) + if repo.model.config.scaffold.dbt_project_dir else Path("target/dbt")) +``` +So an explicit `-o` still wins; otherwise the persisted dbt project dir drives it; else the old +default. Update the help text to say so. (This finally consumes the key import stamps.) + +### D. VS Code: layout QuickPick at import time + keep settings as defaults — `vscode/src/extension.ts` + `package.json` +- In `modelith.importErwin`, when bootstrapping a NEW project (no existing model), show a + `vscode.window.showQuickPick` of the three layouts (model | workspace | none) — default-selected + from the `modelith.import.scaffold` setting — INSTEAD of the current confirm-only modal. The setting + becomes the default the picker starts on, not the only control. Pass the chosen layout as + `--scaffold`. +- Keep `modelith.import.dbtTarget` as the `--dbt-target` value (a QuickPick for a free-form target is + overkill; the setting is fine, could add an inputBox later). +- Everything else (empty-folder handling, set `modelith.modelDir`, workspace → model/ subdir, reused + dbt project) stays as built. +- `package.json` settings already added; keep them (they now seed the picker default). Update the + `import.scaffold` description to say it is the DEFAULT selection in the import picker. + +### E. Tests + docs + version bump +- Python: `mdl config get/set` round-trips (set `scaffold.dbt_project_dir`, re-read, validate); + `set` rejects an invalid value; `generate` with no `-o` writes to `scaffold.dbt_project_dir` when + set (assert the output dir), and to `target/dbt` when unset; `import erwin` default is model-only + (no `transform/`, no dbt shell), `--scaffold workspace` still lays the shell, existing-dbt reuse + still holds. +- README: note that import is model-only by default and `mdl generate` (or `--scaffold workspace`) + handles dbt; document `mdl config set/get` and that `generate` follows `scaffold.dbt_project_dir`. +- Bump CLI `pyproject.toml` (0.6.7 -> 0.6.8) and `vscode/package.json` (0.3.15 -> 0.3.16). + +## Critical files +- `packages/cli/src/mdl_cli/main.py` — `config` sub-app (near `reverse-config`, :2271), `generate` + `-o` default (:295), `import_erwin_cmd` default/help (:2729). +- `packages/cli/src/mdl_cli/scaffold.py` — no change needed to `bootstrap_project` (already returns + `dbt_project_dir`); confirm default path stays `model`. +- `vscode/src/extension.ts:626` — QuickPick for layout on a fresh bootstrap. +- `vscode/package.json` — refine the `modelith.import.scaffold` description. + +## Reuse (found in exploration) +- `_write_reverse_block` (main.py:2275): the load_file → merge → `model_validate` → dump_file pattern + to mirror for `mdl config set`. +- `mdl_core.yaml_io.load_file`/`dump_file` (comment-preserving) and `ProjectConfig.model_validate`. +- `bootstrap_project` / `find_dbt_project` (scaffold.py) and `scaffold_workspace_skeleton` (collab.py) + — unchanged; already do model-only vs workspace and reuse an existing dbt project. +- `repo.model.config.scaffold.dbt_project_dir` (ScaffoldConfig, ir.py) — the key generate/config read. + +## Verification +- `uv run pytest && uv run ruff check packages/`: + - `mdl config set scaffold.dbt_project_dir warehouse -m ` then `mdl config get + scaffold.dbt_project_dir` prints `warehouse`; the doc still `ProjectConfig`-valid; comments kept. + - `mdl config set dbt_target snowflake_prod` updates it; a bad key/type is rejected. + - `mdl generate -m ` with `scaffold.dbt_project_dir` set writes into that dir (no `-o`); unset + → `target/dbt`; explicit `-o` overrides both. + - `mdl import erwin fx.xml -o ` (no flag) → model-only: `.mdl/lock.yaml` yes, `transform/` NO, + no dbt shell; `mdl validate` passes. `--scaffold workspace` → shell present. Existing dbt reuse + still asserts no throwaway DuckDB. +- Extension (`cd vscode && npm run typecheck && npm run regression`, Node 20): all gates green; the + import command still registered. +- Manual: empty folder in VS Code → Import erwin XML → layout QuickPick appears (default from setting) + → chosen layout scaffolds; `mdl generate` in that project writes to the recorded dbt dir. + +## Sequencing (branch `feat-erwin-scaffold`; each step green) +1. C: `generate` honors `scaffold.dbt_project_dir` (small, unblocks the config story) + test. +2. B: `mdl config get/set` + tests. +3. A: confirm/adjust import default help to model-only + test. +4. D: VS Code QuickPick + package.json description; regression. +5. E: README + version bumps. + +## Risk posture +- Default flip to `model` is the documented, less-surprising behaviour (matches `mdl init`); workspace + is still one flag away. No data-loss risk — import never had a model to clobber in the empty case. +- `generate` change is guarded: explicit `-o` still wins; only the omitted-`-o` default changes, and + only when `scaffold.dbt_project_dir` is set (else identical to today's `target/dbt`). +- `mdl config set` validates the whole doc through `ProjectConfig` before writing — a bad set is + rejected, not half-applied (same guarantee as `reverse-config apply`). +- No new dbt-shell writer — dbt-project creation stays solely in `collab.scaffold_workspace_skeleton`, + invoked only by `init --workspace` and `import --scaffold workspace`. diff --git a/uv.lock b/uv.lock index 994b9fd..561f96d 100644 --- a/uv.lock +++ b/uv.lock @@ -939,7 +939,7 @@ requires-dist = [ [[package]] name = "modelith-dbt" -version = "0.6.7" +version = "0.6.8" source = { editable = "." } dependencies = [ { name = "dbt-artifacts-parser" }, diff --git a/vscode/package.json b/vscode/package.json index 2446e3a..71321b3 100644 --- a/vscode/package.json +++ b/vscode/package.json @@ -2,7 +2,7 @@ "name": "modelith-vscode", "displayName": "Modelith", "description": "Ontology-anchored, git-native data modeling for dbt — validate, generate, drift, and the Modelith canvas inside VS Code.", - "version": "0.3.15", + "version": "0.3.16", "publisher": "BosonResearch", "license": "Apache-2.0", "icon": "icon.png", @@ -391,7 +391,7 @@ "Also the transform/warehouse dbt project, CODEOWNERS and .code-workspace (collaboration topology).", "Model objects only, no surrounding skeleton." ], - "markdownDescription": "When importing an erwin XML into an EMPTY workspace, what to scaffold around the model. Ignored when a Modelith project already exists (its layout is respected). Persisted to `mdl-project.yaml` so a re-import reproduces it." + "markdownDescription": "Default selection in the layout picker shown when you import an erwin XML into an EMPTY workspace. `model` scaffolds a model-only project (generate dbt separately with `mdl generate`); `workspace` also lays a dbt project (or reuses one already in the folder); `none` writes just the model YAMLs. Ignored when a Modelith project already exists (its layout is respected). Persisted to `mdl-project.yaml` so a re-import reproduces it." }, "modelith.import.dbtTarget": { "type": "string", diff --git a/vscode/src/extension.ts b/vscode/src/extension.ts index dc54df3..74b4c8d 100644 --- a/vscode/src/extension.ts +++ b/vscode/src/extension.ts @@ -651,20 +651,37 @@ export async function activate(ctx: vscode.ExtensionContext): Promise { } const file = fileUri.fsPath; - // No model here yet → confirm we're bootstrapping a fresh project, and honour the - // configured scaffold layout / dbt target. + // No model here yet → let the user pick what to scaffold around the import (the + // configured modelith.import.scaffold seeds the default pick), honouring dbt target. const cfg = vscode.workspace.getConfiguration("modelith"); - const scaffold = cfg.get("import.scaffold") || "model"; const dbtTarget = cfg.get("import.dbtTarget") || "duckdb_dev"; + let scaffold = cfg.get("import.scaffold") || "model"; if (!existingModel) { - const GO = scaffold === "workspace" ? "Scaffold workspace & import" : "Scaffold & import"; - const choice = await vscode.window.showInformationMessage( - "No Modelith project here yet. Import the erwin model and scaffold a new " + - `project (layout: ${scaffold})?`, - { modal: true }, - GO, - ); - if (choice !== GO) return; + const items: (vscode.QuickPickItem & { value: string })[] = [ + { + label: "Model only", + value: "model", + description: "a runnable model repo — generate dbt separately with `mdl generate`", + }, + { + label: "Workspace", + value: "workspace", + description: "also a dbt project (or reuse one here), CODEOWNERS, .code-workspace", + }, + { + label: "Objects only", + value: "none", + description: "just the model YAMLs, no project skeleton", + }, + ]; + // Float the configured default to the top so Enter accepts it. + items.sort((a, b) => (a.value === scaffold ? -1 : b.value === scaffold ? 1 : 0)); + const pick = await vscode.window.showQuickPick(items, { + title: "No Modelith project here — what should the erwin import scaffold?", + placeHolder: `Default: ${scaffold} (from modelith.import.scaffold)`, + }); + if (!pick) return; + scaffold = pick.value; } // Default the target at the workspace's model dir; the picker confirms/redirects