From e21f8699e10e648d39b5acf6bc1884e8e0e9956c Mon Sep 17 00:00:00 2001 From: Debasish Bose Date: Sun, 27 Sep 2026 15:39:16 +1000 Subject: [PATCH] import: map erwin UDPs and naming conventions; gitignore the schemas MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The erwin importer skipped ~17 object types. Validated the requested ones against the real XSD schemas and mapped the two with clean IR homes: - UDP_Definition / UDP_Instance -> the IR udp dict (already on entity, attribute, relationship, key_group, domain), which flows to dbt meta:. _read_props now captures a UDP_Instance_Groups nested in an object's ; a def-id index resolves each instance's name (by the name attr, else the UDP_Definition Name, else the raw GUID so nothing is lost). - Naming_Options -> ProjectConfig.naming: Case_Conversion_Type UPPER/LOWER map to physical/logical case, honouring Is_Physical_Only/Is_Logical_Only. Both are additive optional IR fields, so objects without them are unchanged (udp stays None), and the three types no longer emit skip warnings. Deferred (need a real export XML / no IR home this round): Data_Vault_Object and _Properties (not even in these schemas — a Data Vault edition export; target is LogicalEntity.pattern), and Name_Mapping (source lineage). Also gitignore erwin/ — the vendored XSD schemas (4.5MB) were untracked but committable; they must not ship in a public repo. CLI 0.6.11. --- .gitignore | 3 + README.md | 5 +- packages/reverse/src/mdl_reverse/erwin.py | 115 +++++++++++++++++++++- packages/reverse/tests/test_erwin.py | 53 ++++++++++ pyproject.toml | 2 +- uv.lock | 2 +- 6 files changed, 172 insertions(+), 8 deletions(-) diff --git a/.gitignore b/.gitignore index 7ff610d..666715b 100644 --- a/.gitignore +++ b/.gitignore @@ -36,3 +36,6 @@ demo/**/dbt_packages/ # Container-generated YAML-schema settings (paths are machine-specific). .vscode/settings.json .claude/worktrees/ + +# Vendored erwin XSD schemas — reference only, must NOT ship in a public repo. +erwin/ diff --git a/README.md b/README.md index 6ed05ed..6f88dc1 100644 --- a/README.md +++ b/README.md @@ -464,7 +464,10 @@ touches disk or git until you submit the proposal. Nobody adopts a modeling tool greenfield, so `mdl import erwin` reads a real erwin Data Modeler XML export and turns it into a committable Modelith model — entities, primary and foreign keys, relationships (with their verb phrases), subtype/supertype categories, -subject areas, and domains, keeping the physical names alongside the logical ones: +subject areas, and domains, keeping the physical names alongside the logical ones. It also +carries your **user-defined properties** (erwin UDPs become `udp:` on each object, which flow +to dbt `meta:`) and your **naming conventions** (erwin's case options become the project's +`naming:` standards): ```bash mdl import erwin your-model.xml -o proj # empty folder -> a runnable model-only project diff --git a/packages/reverse/src/mdl_reverse/erwin.py b/packages/reverse/src/mdl_reverse/erwin.py index 4c8bf84..77467bb 100644 --- a/packages/reverse/src/mdl_reverse/erwin.py +++ b/packages/reverse/src/mdl_reverse/erwin.py @@ -113,6 +113,9 @@ class _Obj: refs: dict[str, str] # single refs: prop-name -> target GUID ref_arrays: dict[str, list[str]] # array refs: prop-name -> ordered target GUIDs children: list[_Obj] = field(default_factory=list) # nested objects (attrs, key groups) + # erwin UDP instances carried inside this object's : (name attr, def GUID, value). + # The name is resolved to the UDP_Definition's Name in a later pass (see _resolve_udp). + udp_raw: list[tuple[str | None, str, str]] = field(default_factory=list) @dataclass @@ -148,10 +151,21 @@ def _extract(el: ET.Element) -> _Obj: def _read_props(props_el: ET.Element, obj: _Obj) -> None: - """Read a `` element's children into the object's props/refs/ref_arrays.""" + """Read a `` element's children into the object's props/refs/ref_arrays. + + UDP instances are carried as a `` child of the Props element (each + `value`); they are collected into + `obj.udp_raw` and resolved to `{udp name: value}` later (see _resolve_udp).""" for p in props_el: name = _local(p.tag) - if name.endswith("_Ref_Array"): + if name == "UDP_Instance_Groups": + for inst in p: + if _local(inst.tag) != "UDP_Instance": + continue + value = (inst.text or "").strip() + if value: # an empty UDP instance carries nothing to keep + obj.udp_raw.append((inst.get("name"), inst.get("id") or "", value)) + elif name.endswith("_Ref_Array"): key = name[: -len("_Array")] # Foo_Ref_Array -> Foo_Ref items: list[tuple[int, str]] = [] for ref in p: @@ -176,7 +190,18 @@ def _read_props(props_el: ET.Element, obj: _Obj) -> None: # Top-level object element names we index (inside their *_Groups). Nested objects # (Attribute, Key_Group, Key_Group_Member) are captured via _extract recursion. _TOP_OBJECTS = frozenset( - {"Entity", "Relationship", "Subject_Area", "Domain", "View", "ER_Diagram"} + { + "Entity", + "Relationship", + "Subject_Area", + "Domain", + "View", + "ER_Diagram", + # Model-level: UDP definitions (id -> name, for resolving instances) and the + # naming-convention options that map onto the project's NamingStandards. + "UDP_Definition", + "Naming_Options", + } ) # Object names that appear NESTED inside a mapped object (consumed by _extract, not at the @@ -195,6 +220,9 @@ def _read_props(props_el: ET.Element, obj: _Obj) -> None: "Extended_Notes", "History_Information", "Display_Style_Sheet", + # UDP instances live inside an object's (read by _read_props); listed + # here so a UDP_Instance_Groups that ever surfaces standalone isn't flagged skipped. + "UDP_Instance", } ) @@ -244,6 +272,26 @@ def _name_of(obj: _Obj, fallback: str | None = None) -> str | None: return obj.props.get("Name") or fallback +def _resolve_udp(obj: _Obj, udp_name_by_def_id: dict[str, str]) -> dict[str, str] | None: + """Resolve an object's raw UDP instances to `{udp name: value}`. Each instance carries a + `name` attribute (the UDP definition's Name) and a def GUID; prefer the name, fall back to + the definition index, then to the raw GUID so a value is never lost. Last write wins on a + duplicate key. Returns None when the object has no UDPs (so `udp` stays unset).""" + if not obj.udp_raw: + return None + out: dict[str, str] = {} + for name_attr, def_id, value in obj.udp_raw: + key = name_attr or udp_name_by_def_id.get(def_id) or def_id + if key: + out[key] = value + return out or None + + +def _truthy(v: str | None) -> bool: + """erwin boolean-ish props are text; treat 1/true/yes/y as True, else False.""" + return (v or "").strip().lower() in {"1", "true", "yes", "y"} + + def _is_key_group_pk(kg: _Obj) -> bool: t = (kg.props.get("Key_Group_Type") or "").strip().lower() return t in {"primary_key", "pk", "0", "primary"} @@ -301,11 +349,16 @@ def import_erwin(source: str | Path, *, project_name: str | None = None) -> Erwi # pass 1: entities (+ conceptual), attributes, key groups, domains, subject areas. domain_name_by_id: dict[str, str] = {} + udp_name_by_def_id: dict[str, str] = {} # UDP_Definition GUID -> its Name (for instances) for obj in ordered: if obj.kind == "Domain": dname = _name_of(obj) if dname: domain_name_by_id[obj.id] = dname + elif obj.kind == "UDP_Definition": + un = _name_of(obj) + if un: + udp_name_by_def_id[obj.id] = un ce_id_by_entity: dict[str, str] = {} # erwin entity id -> Modelith conceptual id le_by_erwin_id: dict[str, LogicalEntity] = {} # erwin entity id -> LogicalEntity @@ -327,6 +380,7 @@ def import_erwin(source: str | Path, *, project_name: str | None = None) -> Erwi obj.props.get("Logical_Data_Type") or obj.props.get("Physical_Data_Type") ), definition=obj.props.get("Definition"), + udp=_resolve_udp(obj, udp_name_by_def_id), ) ) @@ -366,6 +420,7 @@ def import_erwin(source: str | Path, *, project_name: str | None = None) -> Erwi domain=domain, role="business_key" if is_key else "attribute", nullable=_nullable(a.props.get("Null_Option_Type"), is_key), + udp=_resolve_udp(a, udp_name_by_def_id), ) attrs.append(attr) attr_le_by_erwin_id[a.id] = attr.id @@ -385,6 +440,7 @@ def import_erwin(source: str | Path, *, project_name: str | None = None) -> Erwi realises=ce.id, definition=obj.props.get("Definition"), attributes=attrs, + udp=_resolve_udp(obj, udp_name_by_def_id), ) model.add(le) le_by_erwin_id[obj.id] = le @@ -407,6 +463,7 @@ def import_erwin(source: str | Path, *, project_name: str | None = None) -> Erwi name=_name_of(kg) or f"{_key_group_ir_type(kg)}_{ename}", type=_key_group_ir_type(kg), members=members, + udp=_resolve_udp(kg, udp_name_by_def_id), ) ) @@ -423,10 +480,20 @@ def import_erwin(source: str | Path, *, project_name: str | None = None) -> Erwi # pass 2: relationships (+ detect subtype -> Category) and subject-area membership. _build_relationships( - ordered, index, model, le_by_erwin_id, attr_le_by_erwin_id, ce_id_by_entity, warnings + ordered, + index, + model, + le_by_erwin_id, + attr_le_by_erwin_id, + ce_id_by_entity, + udp_name_by_def_id, + warnings, ) _build_subject_areas(ordered, model, ce_id_by_entity, warnings) + # naming conventions -> the project's NamingStandards (physical/logical case). + _apply_naming_options(ordered, model) + # notes for what erwin had that we dropped if any(o.kind == "ER_Diagram" for o in ordered): warnings.append("ER diagram layout (positions) was not imported; Modelith auto-layouts") @@ -443,7 +510,14 @@ def import_erwin(source: str | Path, *, project_name: str | None = None) -> Erwi def _build_relationships( - ordered, index, model, le_by_erwin_id, attr_le_by_erwin_id, ce_id_by_entity, warnings + ordered, + index, + model, + le_by_erwin_id, + attr_le_by_erwin_id, + ce_id_by_entity, + udp_name_by_def_id, + warnings, ) -> None: subtype_children_by_parent: dict[str, list[str]] = {} subtype_discriminator_by_parent: dict[str, str] = {} # parent erwin id -> attr erwin id @@ -488,6 +562,7 @@ def _build_relationships( identifying="identifying" in rtype, verb_phrase=obj.props.get("Parent_To_Child_Verb_Phrase"), inverse_verb_phrase=obj.props.get("Child_To_Parent_Verb_Phrase"), + udp=_resolve_udp(obj, udp_name_by_def_id), ) ) @@ -554,6 +629,36 @@ def _build_subject_areas(ordered, model, ce_id_by_entity, warnings) -> None: ) +# erwin Case_Conversion_Type (a free string) -> Modelith NamingStandards case values. Only +# the shapes NamingStandards allows are produced; anything else degrades to "any". +def _physical_case(cc: str) -> str: + return {"UPPER": "upper_snake", "LOWER": "snake"}.get(cc, "any") + + +def _logical_case(cc: str) -> str: + return {"LOWER": "snake"}.get(cc, "any") + + +def _apply_naming_options(ordered, model) -> None: + """Map erwin Naming_Options onto the project's NamingStandards. An options object flagged + Is_Physical_Only sets physical_case; Is_Logical_Only sets logical_case; an unflagged one + sets both. erwin often ships one of each — iterating applies them independently. Missing or + unknown values degrade to "any" and never raise.""" + for obj in ordered: + if obj.kind != "Naming_Options": + continue + cc = (obj.props.get("Case_Conversion_Type") or "").strip().upper() + physical_only = _truthy(obj.props.get("Is_Physical_Only")) + logical_only = _truthy(obj.props.get("Is_Logical_Only")) + if physical_only and not logical_only: + model.config.naming.physical_case = _physical_case(cc) + elif logical_only and not physical_only: + model.config.naming.logical_case = _logical_case(cc) + else: # an unscoped options object governs both + model.config.naming.physical_case = _physical_case(cc) + model.config.naming.logical_case = _logical_case(cc) + + def _open(source: str | Path): """Return (stream, size_hint_bytes|None) for iterparse. Accepts a path, an XML string, or an open stream.""" diff --git a/packages/reverse/tests/test_erwin.py b/packages/reverse/tests/test_erwin.py index 77ab210..e3223d8 100644 --- a/packages/reverse/tests/test_erwin.py +++ b/packages/reverse/tests/test_erwin.py @@ -22,10 +22,28 @@ id_typeBIGINT + + + StewardText + + + + + PhysicalUPPER + 1 + + + LogicalLOWER + 1 + + CounterpartyCOUNTERPARTY A trading partner. + + Jane Doe + @@ -34,6 +52,9 @@ legal_nameVARCHAR(255) + + Bob + @@ -170,6 +191,35 @@ def test_domain_imported(): assert any(d.name == "id_type" for d in m.domains.values()) +def test_udp_on_entity_by_name(): + # a UDP_Instance carrying the definition's Name attribute -> {name: value} on the entity + m = _import().model + cp = next(le for le in m.logical_entities.values() if le.name == "counterparty") + assert cp.udp == {"Steward": "Jane Doe"} + + +def test_udp_on_attribute_resolved_by_def_id(): + # a UDP_Instance with only the def GUID (no name attr) resolves via the UDP_Definition index + m = _import().model + cp = next(le for le in m.logical_entities.values() if le.name == "counterparty") + ln = next(a for a in cp.attributes if a.name == "legal_name") + assert ln.udp == {"Steward": "Bob"} + + +def test_entities_without_udps_have_none(): + # an object with no UDP instances keeps udp unset (no noise `udp: {}` on every object) + m = _import().model + trade = next(le for le in m.logical_entities.values() if le.name == "trade") + assert trade.udp is None + + +def test_naming_options_applied(): + # erwin Naming_Options -> the project's NamingStandards (physical UPPER, logical LOWER) + m = _import().model + assert m.config.naming.physical_case == "upper_snake" + assert m.config.naming.logical_case == "snake" + + def test_unknown_objects_warn_not_fail(): r = _import() joined = " ".join(r.warnings).lower() @@ -178,6 +228,9 @@ def test_unknown_objects_warn_not_fail(): # nested Attribute/Key_Group are NOT reported as skipped Model-level objects assert "attribute" not in joined assert "key_group" not in joined + # the newly-mapped types no longer warn + assert "udp" not in joined + assert "naming_options" not in joined def test_imported_model_validates(): diff --git a/pyproject.toml b/pyproject.toml index 1fdf9b2..1e7a460 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,7 +3,7 @@ # project, so the install name is `modelith-dbt` (install: uv tool install # modelith-dbt). The command stays `mdl`; the import modules stay mdl_*. name = "modelith-dbt" -version = "0.6.10" +version = "0.6.11" description = "Ontology-anchored, git-native data modeling tool for dbt-core teams" readme = "README.md" requires-python = ">=3.11" diff --git a/uv.lock b/uv.lock index 674bb1c..20370f9 100644 --- a/uv.lock +++ b/uv.lock @@ -939,7 +939,7 @@ requires-dist = [ [[package]] name = "modelith-dbt" -version = "0.6.9" +version = "0.6.10" source = { editable = "." } dependencies = [ { name = "dbt-artifacts-parser" },