Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
19 changes: 16 additions & 3 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -467,15 +467,28 @@ foreign keys, relationships (with their verb phrases), subtype/supertype categor
subject areas, and domains, keeping the physical names alongside the logical ones:

```bash
mdl import erwin your-model.xml -o model # writes a fresh model
mdl import erwin your-model.xml -o model --apply # or merge into an existing one
mdl import erwin your-model.xml -o proj # empty folder -> a runnable model-only project
mdl import erwin your-model.xml -o proj --scaffold workspace # also lay a dbt project
mdl import erwin your-model.xml -o model --apply # or merge into a model you already have
```

Point it at an **empty folder** and it bootstraps a model project around the import — the
`.gitignore`, the pinned `.mdl/lock.yaml`, a populated `mdl-project.yaml` — so there is no
separate `mdl init` step. Import owns the **logical model**; producing the dbt project is a
separate step (`mdl generate`), matching the rest of the tool. `--scaffold workspace` lays
the dbt project at import time too — `transform/warehouse` with a zero-setup DuckDB
`profiles.yml`, or, if the folder **already** has a dbt project, it reuses that one instead of
scaffolding a throwaway. The chosen layout and the dbt project location are recorded in
`mdl-project.yaml` (under `scaffold:`); `mdl generate` then defaults its output there, and you
can change it any time with `mdl config set scaffold.dbt_project_dir <dir>`. If the folder
already holds a Modelith project the existing structure and config are respected (no
re-scaffold).

It parses the export by streaming, so 7-10MB files import without a memory spike, and it
degrades gracefully: anything Modelith doesn't model (views, ER-diagram layout, physical
transforms) is skipped with a note rather than a failure. In VS Code it's on the palette
(**Modelith: Import Erwin XML**), a button in the Reverse Review panel, and a right-click
on any `.xml` file.
on any `.xml` file — and in an empty workspace it offers a layout picker before scaffolding.

## Reverse engineering a real warehouse

Expand Down
64 changes: 55 additions & 9 deletions packages/cli/src/mdl_cli/collab.py
Original file line number Diff line number Diff line change
Expand Up @@ -151,12 +151,12 @@ def _git(*args: str) -> bool:
{{
"folders": [
{{ "name": "model", "path": "{model}" }},
{{ "name": "transform", "path": "transform/warehouse" }},
{{ "name": "transform", "path": "{dbt}" }},
{{ "name": "repo", "path": "." }}
],
"settings": {{
"modelith.modelDir": "{model}",
"modelith.dbtProjectDir": "transform/warehouse"
"modelith.dbtProjectDir": "{dbt}"
}}
}}
"""
Expand All @@ -170,6 +170,23 @@ def _git(*args: str) -> bool:
target-path: target
"""

# A zero-setup, credential-free local profile (DuckDB, file-based) so the generated
# warehouse can `dbt build` immediately. The profile name matches `profile:` in the
# dbt_project stub. Point it at your real warehouse (and set env vars) when ready —
# Modelith never writes a credential. `{target}` names the output the model config uses.
_PROFILES_STUB = """\
# Local dbt profile — DuckDB, file-based, zero setup. Swap for your warehouse when ready
# (`mdl reverse --init-profile <adapter>` scaffolds a credential-free template). The
# profile name must match `profile:` in dbt_project.yml (warehouse).
warehouse:
target: {target}
outputs:
{target}:
type: duckdb
path: warehouse.duckdb # created on first run, relative to this dir
threads: 4
"""


def scaffold_workspace(root: Path, project_name: str, scaffold_model) -> list[str]:
"""One repo, two sibling roots (§2.1): model/ + transform/warehouse/, with
Expand All @@ -181,12 +198,41 @@ def scaffold_workspace(root: Path, project_name: str, scaffold_model) -> list[st
scaffold_model(root / model_root, project_name=project_name)
written.append(f"{model_root}/ (model repo)")

tw = root / "transform" / "warehouse"
(tw / "models" / "staging").mkdir(parents=True, exist_ok=True)
(tw / "macros").mkdir(parents=True, exist_ok=True)
if not (tw / "dbt_project.yml").exists():
(tw / "dbt_project.yml").write_text(_DBT_PROJECT_STUB)
written.append("transform/warehouse/ (dbt project)")
written += scaffold_workspace_skeleton(root, project_name)
return written


def scaffold_workspace_skeleton(
root: Path,
project_name: str,
*,
dbt_target: str = "duckdb_dev",
existing_dbt: bool = False,
dbt_project_dir: str = "transform/warehouse",
) -> list[str]:
"""The collaboration topology WITHOUT scaffolding a seed model under model/ (§2.1):
the transform/warehouse dbt project, ontologies/, CODEOWNERS, the .code-workspace and
the git merge drivers. Used by an import that supplies its own model objects, so it
lays down the plumbing and leaves model/ for the importer to write into. Idempotent.

``existing_dbt`` True means the workspace ALREADY has a dbt project (found elsewhere),
so we do NOT scaffold a throwaway transform/warehouse DuckDB project or a profiles.yml
on top of it — the model is wired to the real project instead."""
written: list[str] = []
model_root = "model"

if not existing_dbt:
tw = root / "transform" / "warehouse"
(tw / "models" / "staging").mkdir(parents=True, exist_ok=True)
(tw / "macros").mkdir(parents=True, exist_ok=True)
if not (tw / "dbt_project.yml").exists():
(tw / "dbt_project.yml").write_text(_DBT_PROJECT_STUB)
written.append("transform/warehouse/ (dbt project)")
if not (tw / "profiles.yml").exists():
(tw / "profiles.yml").write_text(_PROFILES_STUB.format(target=dbt_target))
written.append("transform/warehouse/profiles.yml")
else:
written.append("(reusing the existing dbt project — no DuckDB project scaffolded)")

(root / "ontologies" / "industry").mkdir(parents=True, exist_ok=True)

Expand All @@ -198,7 +244,7 @@ def scaffold_workspace(root: Path, project_name: str, scaffold_model) -> list[st

ws = root / f"{project_name}.code-workspace"
if not ws.exists():
ws.write_text(_WORKSPACE.format(model=model_root))
ws.write_text(_WORKSPACE.format(model=model_root, dbt=dbt_project_dir))
written.append(ws.name)

written += ensure_git_hooks(root, model_root)
Expand Down
196 changes: 188 additions & 8 deletions packages/cli/src/mdl_cli/main.py
Original file line number Diff line number Diff line change
Expand Up @@ -296,7 +296,13 @@ def lint(
def generate(
model_dir: Path = typer.Option(Path("."), "--model-dir", "-m"),
target: str = typer.Option(None, "--target", "-t", help="Physical target"),
out: Path = typer.Option(Path("target/dbt"), "--out", "-o", help="dbt project output dir"),
out: Path = typer.Option(
None,
"--out",
"-o",
help="dbt project output dir. Defaults to the project's scaffold.dbt_project_dir "
"when set (so it follows where the project was scaffolded), else target/dbt.",
),
inline: bool = typer.Option(False, "--inline", help="Inline pattern SQL instead of macros"),
emit_contract: bool = typer.Option(
False, "--emit-contract", help="Also write an ODCS datacontract.yaml at the model root"
Expand All @@ -321,6 +327,12 @@ def generate(
"""
repo = _load(model_dir)
tgt = target or repo.model.config.dbt_target or "duckdb_dev"
# Where to write: an explicit -o wins; otherwise follow where the project was
# scaffolded (scaffold.dbt_project_dir, recorded RELATIVE to the model dir), else the
# historical target/dbt default (relative to CWD, as before).
if out is None:
scaffold_dir = repo.model.config.scaffold.dbt_project_dir
out = (model_dir / scaffold_dir) if scaffold_dir else Path("target/dbt")
emitter = DbtEmitter(repo.model, tgt, inline=inline)
result = emitter.generate(out, write=not dry_run)

Expand Down Expand Up @@ -2534,6 +2546,111 @@ def reverse_config_import(
typer.secho(f"imported reverse: config from {src}", fg=typer.colors.GREEN)


# --- project config: read/write mdl-project.yaml keys from the CLI -----------------
# The general config doorway (the `reverse:` block has its own richer `reverse-config`
# surface above; this handles the rest — dbt_target, platform_targets, the scaffold
# block, etc.). Comment-preserving, and the whole doc is re-validated through
# ProjectConfig before writing so a bad set is rejected, not half-applied.
config_app = typer.Typer(help="Read and write mdl-project.yaml config keys.")
app.add_typer(config_app, name="config")

# Known keys and how a CLI string value coerces onto them. Anything not listed is set
# as a plain string (still validated against ProjectConfig, so a wrong type is caught).
_CONFIG_LIST_KEYS = {"platform_targets"}


def _coerce_config_value(dotted: str, raw: str):
"""Turn a CLI string into the right shape for a known key: a comma-split list for
list-valued keys, an empty string clears (None) for the scaffold's optional dir."""
if dotted in _CONFIG_LIST_KEYS:
return [v.strip() for v in raw.split(",") if v.strip()]
if dotted == "scaffold.dbt_project_dir" and raw == "":
return None
return raw


def _dig(doc, dotted: str):
"""Read a dotted key from a loaded doc, or None if any segment is missing."""
cur = doc
for part in dotted.split("."):
if not hasattr(cur, "get") or part not in cur:
return None
cur = cur[part]
return cur


def _set_config_key(doc, dotted: str, value) -> None:
"""Set a dotted key on a loaded (comment-preserving) doc, creating intermediate maps."""
parts = dotted.split(".")
cur = doc
for part in parts[:-1]:
if part not in cur or not hasattr(cur[part], "get"):
cur[part] = {}
cur = cur[part]
cur[parts[-1]] = value


@config_app.command("get")
def config_get(
key: str = typer.Argument(
None, help="Dotted key, e.g. scaffold.dbt_project_dir. Omit to print the whole config."
),
model_dir: Path = typer.Option(Path("."), "--model-dir", "-m"),
) -> None:
"""Print the project config, or a single dotted key (scaffold.dbt_project_dir, dbt_target…)."""
from mdl_core.yaml_io import dump_str, load_file

proj_path = model_dir / "mdl-project.yaml"
if not proj_path.exists():
typer.secho(f"no mdl-project.yaml in {model_dir}", fg=typer.colors.RED, err=True)
raise typer.Exit(1)
doc = load_file(proj_path)
if key is None:
typer.echo(dump_str(doc).rstrip())
return
val = _dig(doc, key)
if val is None:
typer.secho(f"{key} is not set", fg=typer.colors.YELLOW)
return
typer.echo(dump_str(val).rstrip() if hasattr(val, "get") or isinstance(val, list) else str(val))


@config_app.command("set")
def config_set(
key: str = typer.Argument(..., help="Dotted key, e.g. scaffold.dbt_project_dir"),
value: str = typer.Argument(..., help="New value. Comma-separated for list keys; '' clears."),
model_dir: Path = typer.Option(Path("."), "--model-dir", "-m"),
) -> None:
"""Set an mdl-project.yaml config key (comment-preserving). The whole config is
re-validated through ProjectConfig before writing, so a bad key or wrong type is
rejected rather than half-applied.

Examples:
mdl config set scaffold.dbt_project_dir transform/warehouse
mdl config set dbt_target snowflake_prod
mdl config set platform_targets duckdb_dev,snowflake_prod
"""
from mdl_core.ir import ProjectConfig
from mdl_core.yaml_io import dump_file, load_file

proj_path = model_dir / "mdl-project.yaml"
if not proj_path.exists():
typer.secho(f"no mdl-project.yaml in {model_dir}", fg=typer.colors.RED, err=True)
raise typer.Exit(1)
doc = load_file(proj_path)
_set_config_key(doc, key, _coerce_config_value(key, value))
# validate the resulting doc before writing — reject a bad set wholesale.
try:
ProjectConfig.model_validate(dict(doc))
except Exception as e: # noqa: BLE001 - surface the validation error, don't half-write
typer.secho(
f"resulting config is invalid — not written ({e})", fg=typer.colors.RED, err=True
)
raise typer.Exit(1) from e
dump_file(proj_path, doc)
typer.secho(f"set {key} = {value!r}", fg=typer.colors.GREEN)


@emit_app.command("semantic")
def emit_semantic(
fmt: str = typer.Option("metricflow", "--format", help="metricflow|osi"),
Expand Down Expand Up @@ -2730,6 +2847,23 @@ def import_erwin_cmd(
file: Path = typer.Argument(..., help="erwin XML export (.xml)"),
out: Path = typer.Option(Path("model"), "--out", "-o"),
name: str = typer.Option(None, "--name"),
scaffold_layout: str = typer.Option(
"model",
"--scaffold",
help="When --out is EMPTY, what to bootstrap around the imported model. "
"`model` (DEFAULT) — a runnable model repo only (.gitignore, .mdl/lock.yaml, "
".mdl/state/); generate the dbt project separately with `mdl generate`. "
"`workspace` — also lay a dbt project (transform/warehouse, or reuse one already "
"in the folder), CODEOWNERS and a .code-workspace (collab §2.1). "
"`none` — model object YAMLs only, no skeleton. "
"Ignored when --out already holds a project (that layout is respected).",
),
dbt_target: str = typer.Option(
"duckdb_dev",
"--dbt-target",
help="dbt target/profile name written into the project config (and the workspace "
"profiles.yml). DuckDB by default so the generated warehouse builds with no setup.",
),
apply: bool = typer.Option(
False,
"--apply",
Expand All @@ -2743,10 +2877,14 @@ def import_erwin_cmd(
) -> None:
"""Import a real erwin XML export into a Modelith model (spec §6.4).

Default: write a fresh model at --out (like `mdl reverse` — an erwin export is a whole
model). If --out already holds a model it diverts to a `-reversed-v<N>` sibling unless
--force. With --apply, merge the erwin objects into the existing model at --out through
the same validated mutation engine manual edits use."""
Default: write a fresh, model-only project at --out (like `mdl reverse` — an erwin export
is a whole model). Producing the dbt project is a SEPARATE step (`mdl generate`, or
`--scaffold workspace` to lay the dbt shell here at import time). The chosen layout is
persisted to mdl-project.yaml (scaffold:) so `mdl generate` and a re-import follow it. If
--out already holds a project the existing structure is RESPECTED — no re-scaffold, user
config preserved — and it diverts to a `-reversed-v<N>` sibling unless --force. With
--apply, merge the erwin objects into the existing model at --out through the same
validated mutation engine manual edits use."""
# streaming reader handles 7-10MB files; pass the PATH so it reads the file directly.
result = import_erwin(file, project_name=name)
for w in result.warnings:
Expand All @@ -2771,12 +2909,54 @@ def import_erwin_cmd(
)
return

# A project already here (an mdl-project.yaml or a .mdl/ dir, at --out or its model/
# subdir) is respected: no scaffold, and the writer preserves the user's config.
existing = (
(out / "mdl-project.yaml").exists()
or (out / ".mdl").exists()
or (out / "model" / "mdl-project.yaml").exists()
)

from mdl_cli.scaffold import bootstrap_project

boot = bootstrap_project(
out,
project_name=name or _project_name_from_path(out),
layout=scaffold_layout,
dbt_target=dbt_target,
existing=existing,
)
write_target = boot.model_root
for line in boot.written:
typer.secho(f" scaffolded {line}", fg=typer.colors.GREEN)
reused_dbt = (
not existing
and scaffold_layout == "workspace"
and boot.dbt_project_dir not in (None, "transform/warehouse")
)
if reused_dbt:
typer.secho(
f" found an existing dbt project at {boot.dbt_project_dir}; wiring the model "
"to it instead of scaffolding a DuckDB one",
fg=typer.colors.CYAN,
)

# Stamp the config so the layout persists (writer._USER_OWNED_CONFIG keeps it on a
# re-import). Only on a fresh scaffold — an existing project owns its own config.
if not existing and scaffold_layout != "none":
result.model.config.dbt_target = dbt_target
result.model.config.platform_targets = [dbt_target]
result.model.config.scaffold.layout = scaffold_layout
# Point the config at the real dbt project (a discovered one, else the scaffolded
# transform/warehouse) so we never wire to a phantom DuckDB.
result.model.config.scaffold.dbt_project_dir = boot.dbt_project_dir

if not force:
out = _nonclobbering_out(out)
write_reversed(result.model, out)
write_target = _nonclobbering_out(write_target)
write_reversed(result.model, write_target)
typer.secho(
f"imported {len(result.model.logical_entities)} entities, "
f"{len(result.model.relationships)} relationships from erwin to {out}",
f"{len(result.model.relationships)} relationships from erwin to {write_target}",
fg=typer.colors.GREEN,
)

Expand Down
Loading
Loading