diff --git a/README.md b/README.md index ce3955e..006e65a 100644 --- a/README.md +++ b/README.md @@ -54,6 +54,24 @@ The expected output is checked in at `examples/expected-selection.avsc`. Its `data` union contains only `ListEventsResponse`; the `POST /events` JSON response and `GET /events/{id}/export` `text/csv` response are skipped. +To also emit Confluent Schema Registry referenced schemas while preserving the +bundled `.avsc` output: + +```bash +uv run openapi-get-avro generate \ + --input examples/minimal.openapi.json \ + --namespace com.example.sports \ + --rootname SportsEnvelope \ + --output build/sports-envelope.avsc \ + --references-output-dir build/schema-references \ + --root-subject sts.abc.def.avro-value \ + --reference-subject-template "{fullname}" +``` + +The references directory receives one `.avsc` file per generated Avro named +type plus `manifest.json`. The manifest is ordered for registration: dependency +subjects first, then schemas that reference them, with the root envelope last. + Useful policy and naming options: ```bash @@ -81,6 +99,10 @@ Accepted CLI values: - `--unknown-object-policy`: `fail`, `map`, `string`, or `empty-record` - `--include-status-codes`: comma-separated response codes, evaluated in the order provided - `--remove-name-suffixes`: comma-separated, case-sensitive suffixes removed from generated Avro named types, for example `Dto` +- `--references-output-dir`: writes Confluent Schema Registry referenced schemas in addition to the bundled output +- `--references-manifest-output`: overrides the referenced-schema manifest path +- `--reference-subject-template`: formats registry subjects with `{fullname}`, `{namespace}`, `{name}`, and `{rootname}` +- `--root-subject`: overrides the root envelope subject; for Confluent's default topic value subject, use `-value` The implemented strict behavior is the default: invalid enum values and ambiguous free-form objects fail. `--any-of-policy union` is implemented when every branch diff --git a/docs/TECHNICAL_SPEC.md b/docs/TECHNICAL_SPEC.md index 090d439..8b1c092 100644 --- a/docs/TECHNICAL_SPEC.md +++ b/docs/TECHNICAL_SPEC.md @@ -23,6 +23,13 @@ Optional options: --enum-policy fail, string, or sanitize. Default: fail. --unknown-object-policy fail, map, string, or empty-record. Default: fail. --remove-name-suffixes Comma-separated generated named-type suffixes to remove. Default: none. +--references-output-dir Also write Confluent Schema Registry referenced schemas to this directory. +--references-manifest-output + Manifest path for referenced schemas. Default: /manifest.json. +--reference-subject-template + Subject template. Supports {fullname}, {namespace}, {name}, and {rootname}. Default: {fullname}. +--root-subject Schema Registry subject for the root envelope. For Confluent TopicNameStrategy + topic values, use -value. ``` ## OpenAPI selection rules @@ -154,3 +161,41 @@ root envelope fields remain unchanged. After generating the schema, validate it with `fastavro.parse_schema`. Validation failures should include enough context to identify the generated type that failed. + +## Confluent Schema Registry references + +When `--references-output-dir` is set, the CLI must still write the bundled +self-contained schema to `--output` or stdout. It also writes one standalone +schema file per generated Avro named type and a manifest describing Confluent +Schema Registry registration metadata. + +Referenced-schema files must use fully qualified Avro names for dependencies. +Reference subjects are formatted with `--reference-subject-template`. The root +envelope subject may be overridden independently with `--root-subject`, which is +the expected option for Confluent's default topic value subject, `-value`. +The manifest must be deterministic and ordered so dependency subjects appear +before schemas that reference them: + +```text +shared leaf named types +-> named types that depend on them +-> GET response records +-> root envelope schema +``` + +Manifest entries include: + +```json +{ + "fullname": "com.example.GetMatchResponse", + "subject": "com.example.GetMatchResponse", + "file": "com.example.GetMatchResponse.avsc", + "references": [ + { + "name": "com.example.Venue", + "subject": "com.example.Venue", + "version": "latest" + } + ] +} +``` diff --git a/src/openapi_get_avro/cli.py b/src/openapi_get_avro/cli.py index fb78946..bc2461e 100644 --- a/src/openapi_get_avro/cli.py +++ b/src/openapi_get_avro/cli.py @@ -10,7 +10,7 @@ import typer from ruamel.yaml import YAML -from .converter import convert_openapi_to_avro +from .converter import convert_openapi_to_avro, convert_openapi_to_referenced_avro from .exceptions import OpenApiAvroError from .models import ( AnyOfPolicy, @@ -18,6 +18,7 @@ FieldNameCase, GenerationOptions, NameStrategy, + ReferencedSchemaSet, UnknownObjectPolicy, ) @@ -91,6 +92,41 @@ def _parse_name_suffixes(value: str) -> tuple[str, ...]: return suffixes +def _render_json(value: object) -> str: + return json.dumps(value, indent=2, ensure_ascii=False) + "\n" + + +def _write_referenced_schemas( + schema_set: ReferencedSchemaSet, output_dir: Path, manifest_output: Path | None +) -> None: + output_dir.mkdir(parents=True, exist_ok=True) + for artifact in schema_set.artifacts: + (output_dir / artifact.filename).write_text( + _render_json(artifact.schema), + encoding="utf-8", + ) + + manifest_path = manifest_output or output_dir / "manifest.json" + manifest_path.parent.mkdir(parents=True, exist_ok=True) + manifest = [ + { + "fullname": artifact.fullname, + "subject": artifact.subject, + "file": artifact.filename, + "references": [ + { + "name": reference.name, + "subject": reference.subject, + "version": reference.version, + } + for reference in artifact.references + ], + } + for artifact in schema_set.artifacts + ] + manifest_path.write_text(_render_json(manifest), encoding="utf-8") + + @app.command() def generate( input: Annotated[ @@ -149,6 +185,43 @@ def generate( help="Comma-separated generated Avro named-type suffixes to remove", ), ] = "", + references_output_dir: Annotated[ + Path | None, + typer.Option( + "--references-output-dir", + help="Also write Confluent Schema Registry referenced schemas to this directory", + ), + ] = None, + references_manifest_output: Annotated[ + Path | None, + typer.Option( + "--references-manifest-output", + help=( + "Manifest path for referenced schemas; defaults to " + "/manifest.json" + ), + ), + ] = None, + reference_subject_template: Annotated[ + str, + typer.Option( + "--reference-subject-template", + help=( + "Subject template for referenced schemas. " + "Supports {fullname}, {namespace}, {name}, and {rootname}" + ), + ), + ] = "{fullname}", + root_subject: Annotated[ + str | None, + typer.Option( + "--root-subject", + help=( + "Schema Registry subject for the root envelope; for default " + "Confluent topic values use -value" + ), + ), + ] = None, ) -> None: """Generate an Avro envelope schema from GET responses in an OpenAPI document.""" try: @@ -174,8 +247,22 @@ def generate( ), remove_name_suffixes=_parse_name_suffixes(remove_name_suffixes), ) - avro_schema = convert_openapi_to_avro(openapi_doc, options) - rendered = json.dumps(avro_schema, indent=2, ensure_ascii=False) + "\n" + if references_output_dir is None: + avro_schema = convert_openapi_to_avro(openapi_doc, options) + else: + schema_set = convert_openapi_to_referenced_avro( + openapi_doc, + options, + subject_template=reference_subject_template, + root_subject=root_subject, + ) + avro_schema = schema_set.bundled_schema + _write_referenced_schemas( + schema_set, + references_output_dir, + references_manifest_output, + ) + rendered = _render_json(avro_schema) if output is None: typer.echo(rendered, nl=False) else: diff --git a/src/openapi_get_avro/converter.py b/src/openapi_get_avro/converter.py index f3b7e01..3f8f6a3 100644 --- a/src/openapi_get_avro/converter.py +++ b/src/openapi_get_avro/converter.py @@ -3,6 +3,7 @@ from __future__ import annotations import re +from collections import OrderedDict from collections.abc import Hashable from typing import Any @@ -14,7 +15,13 @@ OpenApiAvroError, UnsupportedSchemaError, ) -from .models import GenerationOptions, SelectedOperation +from .models import ( + GenerationOptions, + ReferencedSchemaArtifact, + ReferencedSchemaSet, + SchemaRegistryReference, + SelectedOperation, +) AVRO_NAME_RE = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$") LOCAL_COMPONENT_REF_PREFIX = "#/components/schemas/" @@ -22,6 +29,16 @@ JsonDict = dict[str, Any] NameIdentity = tuple[Hashable, ...] +PRIMITIVE_AVRO_TYPES = { + "null", + "boolean", + "int", + "long", + "float", + "double", + "bytes", + "string", +} class _Converter: @@ -861,3 +878,200 @@ def convert_openapi_to_avro( OpenApiAvroError: If the document cannot be converted under the selected policies. """ return _Converter(openapi_doc, options).convert() + + +class _ReferencedSchemaRenderer: + def __init__( + self, + bundled_schema: JsonDict, + *, + namespace: str, + root_name: str, + subject_template: str, + root_subject: str | None = None, + ) -> None: + self.bundled_schema = bundled_schema + self.namespace = namespace + self.root_name = root_name + self.subject_template = subject_template + self.root_subject = root_subject + self.definitions: OrderedDict[str, JsonDict] = OrderedDict() + self.name_to_fullname: dict[str, str] = {} + + def render(self) -> ReferencedSchemaSet: + self._collect_named_definitions(self.bundled_schema, default_namespace=self.namespace) + dependencies: dict[str, tuple[str, ...]] = {} + schemas: dict[str, JsonDict] = {} + for fullname, schema in self.definitions.items(): + collected_dependencies: list[str] = [] + schemas[fullname] = self._standalone_schema(schema, fullname, collected_dependencies) + dependencies[fullname] = tuple(dict.fromkeys(collected_dependencies)) + + ordered_fullnames = self._registration_order(dependencies) + root_fullname = f"{self.namespace}.{self.root_name}" + artifacts = tuple( + ReferencedSchemaArtifact( + fullname=fullname, + subject=self._subject(fullname, root_fullname=root_fullname), + filename=f"{fullname}.avsc", + schema=schemas[fullname], + references=tuple( + SchemaRegistryReference( + name=dependency, + subject=self._subject(dependency, root_fullname=root_fullname), + ) + for dependency in dependencies[fullname] + ), + ) + for fullname in ordered_fullnames + ) + return ReferencedSchemaSet(bundled_schema=self.bundled_schema, artifacts=artifacts) + + def _collect_named_definitions(self, value: Any, *, default_namespace: str) -> None: + if isinstance(value, dict): + schema_type = value.get("type") + nested_namespace = default_namespace + if ( + isinstance(schema_type, str) + and schema_type in {"record", "enum", "fixed"} + and isinstance(value.get("name"), str) + ): + fullname = self._fullname(value, default_namespace) + self.definitions.setdefault(fullname, value) + simple_name = fullname.rsplit(".", 1)[-1] + self.name_to_fullname[simple_name] = fullname + self.name_to_fullname[fullname] = fullname + nested_namespace = fullname.rsplit(".", 1)[0] + for child in value.values(): + self._collect_named_definitions(child, default_namespace=nested_namespace) + elif isinstance(value, list): + for child in value: + self._collect_named_definitions(child, default_namespace=default_namespace) + + def _standalone_schema( + self, schema: JsonDict, owner_fullname: str, dependencies: list[str] + ) -> JsonDict: + rendered: JsonDict = {} + for key, value in schema.items(): + if key == "namespace": + continue + if key == "name": + rendered[key] = owner_fullname.rsplit(".", 1)[-1] + continue + rendered[key] = self._render_type(value, owner_fullname, dependencies) + rendered["namespace"] = owner_fullname.rsplit(".", 1)[0] + return self._order_named_schema(rendered) + + def _render_type(self, value: Any, owner_fullname: str, dependencies: list[str]) -> Any: + if isinstance(value, dict): + return self._render_dict_type(value, owner_fullname, dependencies) + if isinstance(value, list): + return [self._render_type(child, owner_fullname, dependencies) for child in value] + if isinstance(value, str) and value not in PRIMITIVE_AVRO_TYPES: + return self._render_string_reference(value, owner_fullname, dependencies) + return value + + def _render_dict_type( + self, value: JsonDict, owner_fullname: str, dependencies: list[str] + ) -> Any: + schema_type = value.get("type") + if ( + isinstance(schema_type, str) + and schema_type in {"record", "enum", "fixed"} + and isinstance(value.get("name"), str) + ): + fullname = self._fullname(value, owner_fullname.rsplit(".", 1)[0]) + if fullname != owner_fullname: + dependencies.append(fullname) + return fullname + return { + key: self._render_type(child, owner_fullname, dependencies) + for key, child in value.items() + } + + def _render_string_reference( + self, value: str, owner_fullname: str, dependencies: list[str] + ) -> str: + fullname = self.name_to_fullname.get(value) + if fullname is None: + return value + if fullname != owner_fullname: + dependencies.append(fullname) + return fullname + + def _order_named_schema(self, schema: JsonDict) -> JsonDict: + ordered: JsonDict = {} + for key in ("type", "namespace", "name", "doc", "symbols", "fields", "size"): + if key in schema: + ordered[key] = schema[key] + for key, value in schema.items(): + if key not in ordered: + ordered[key] = value + return ordered + + def _registration_order(self, dependencies: dict[str, tuple[str, ...]]) -> tuple[str, ...]: + ordered: list[str] = [] + visiting: set[str] = set() + visited: set[str] = set() + + def visit(fullname: str) -> None: + if fullname in visited: + return + if fullname in visiting: + return + visiting.add(fullname) + for dependency in dependencies[fullname]: + if dependency in dependencies: + visit(dependency) + visiting.remove(fullname) + visited.add(fullname) + ordered.append(fullname) + + for fullname in self.definitions: + visit(fullname) + return tuple(ordered) + + def _fullname(self, schema: JsonDict, default_namespace: str) -> str: + name = schema.get("name") + if not isinstance(name, str): + raise OpenApiAvroError("Named Avro schema is missing a string name") + if "." in name: + return name + namespace = schema.get("namespace") + if isinstance(namespace, str) and namespace: + return f"{namespace}.{name}" + return f"{default_namespace}.{name}" + + def _subject(self, fullname: str, *, root_fullname: str) -> str: + if fullname == root_fullname and self.root_subject is not None: + return self.root_subject + namespace, name = fullname.rsplit(".", 1) + try: + return self.subject_template.format( + fullname=fullname, + namespace=namespace, + name=name, + rootname=self.root_name, + ) + except KeyError as exc: + raise OpenApiAvroError( + f"Unknown placeholder {{{exc.args[0]}}} in reference subject template" + ) from exc + + +def convert_openapi_to_referenced_avro( + openapi_doc: dict[str, Any], + options: GenerationOptions, + *, + subject_template: str = "{fullname}", + root_subject: str | None = None, +) -> ReferencedSchemaSet: + """Convert OpenAPI to bundled Avro plus Confluent-friendly schema references.""" + bundled_schema = _Converter(openapi_doc, options).convert() + return _ReferencedSchemaRenderer( + bundled_schema, + namespace=options.namespace, + root_name=options.root_name, + subject_template=subject_template, + root_subject=root_subject, + ).render() diff --git a/src/openapi_get_avro/models.py b/src/openapi_get_avro/models.py index 44f3ed7..75aeec4 100644 --- a/src/openapi_get_avro/models.py +++ b/src/openapi_get_avro/models.py @@ -30,6 +30,34 @@ class GenerationOptions: remove_name_suffixes: tuple[str, ...] = () +@dataclass(frozen=True) +class SchemaRegistryReference: + """Confluent Schema Registry reference metadata for one dependency.""" + + name: str + subject: str + version: str = "latest" + + +@dataclass(frozen=True) +class ReferencedSchemaArtifact: + """A standalone Avro schema and its registry metadata.""" + + fullname: str + subject: str + filename: str + schema: dict[str, object] + references: tuple[SchemaRegistryReference, ...] = () + + +@dataclass(frozen=True) +class ReferencedSchemaSet: + """Bundled schema plus deterministic referenced-schema artifacts.""" + + bundled_schema: dict[str, object] + artifacts: tuple[ReferencedSchemaArtifact, ...] + + @dataclass(frozen=True) class SelectedOperation: """A GET operation selected for Avro generation.""" diff --git a/tests/test_cli_contract.py b/tests/test_cli_contract.py index 6ffcb46..5dcd363 100644 --- a/tests/test_cli_contract.py +++ b/tests/test_cli_contract.py @@ -59,6 +59,10 @@ def test_cli_help_exposes_generation_policy_options() -> None: assert "--unknown-object-policy" in help_output assert "--remove-name-suffixes" in help_output assert "--field-name-case" in help_output + assert "--references-output-dir" in help_output + assert "--references-manifest-output" in help_output + assert "--reference-subject-template" in help_output + assert "--root-subject" in help_output def test_cli_include_status_codes_preserves_requested_order(tmp_path: Path) -> None: diff --git a/tests/test_referenced_schemas.py b/tests/test_referenced_schemas.py new file mode 100644 index 0000000..73dca7a --- /dev/null +++ b/tests/test_referenced_schemas.py @@ -0,0 +1,112 @@ +from __future__ import annotations + +import json +from pathlib import Path + +from typer.testing import CliRunner + +from openapi_get_avro.cli import app +from openapi_get_avro.converter import convert_openapi_to_referenced_avro +from openapi_get_avro.models import GenerationOptions + +FIXTURES = Path(__file__).parent / "fixtures" + + +def load_json(name: str) -> dict[str, object]: + return json.loads((FIXTURES / name).read_text(encoding="utf-8")) + + +def test_referenced_schema_set_uses_dependency_registration_order() -> None: + schema_set = convert_openapi_to_referenced_avro( + load_json("minimal.openapi.json"), + GenerationOptions(namespace="com.example.sports", root_name="SportsEnvelope"), + ) + + assert schema_set.bundled_schema == load_json("expected-minimal.avsc") + assert [artifact.fullname for artifact in schema_set.artifacts] == [ + "com.example.sports.Operation", + "com.example.sports.EntityType", + "com.example.sports.Venue", + "com.example.sports.GetMatchResponse", + "com.example.sports.SportsEnvelope", + ] + + artifacts = {artifact.fullname: artifact for artifact in schema_set.artifacts} + response = artifacts["com.example.sports.GetMatchResponse"] + assert response.schema["namespace"] == "com.example.sports" + assert response.schema["fields"][-1]["type"] == [ + "null", + "com.example.sports.Venue", + ] + assert [reference.name for reference in response.references] == ["com.example.sports.Venue"] + + root = artifacts["com.example.sports.SportsEnvelope"] + assert [reference.name for reference in root.references] == [ + "com.example.sports.Operation", + "com.example.sports.EntityType", + "com.example.sports.GetMatchResponse", + ] + assert root.schema["fields"][-1]["type"] == ["com.example.sports.GetMatchResponse"] + + +def test_cli_writes_bundled_schema_and_confluent_references(tmp_path: Path) -> None: + runner = CliRunner() + bundled_output = tmp_path / "schema.avsc" + references_dir = tmp_path / "references" + + result = runner.invoke( + app, + [ + "generate", + "--input", + str(FIXTURES / "minimal.openapi.json"), + "--namespace", + "com.example.sports", + "--rootname", + "SportsEnvelope", + "--output", + str(bundled_output), + "--references-output-dir", + str(references_dir), + "--reference-subject-template", + "{fullname}", + "--root-subject", + "sts.abc.def.avro-value", + ], + ) + + assert result.exit_code == 0, result.output + assert json.loads(bundled_output.read_text(encoding="utf-8")) == load_json( + "expected-minimal.avsc" + ) + + manifest = json.loads((references_dir / "manifest.json").read_text(encoding="utf-8")) + assert [entry["subject"] for entry in manifest] == [ + "com.example.sports.Operation", + "com.example.sports.EntityType", + "com.example.sports.Venue", + "com.example.sports.GetMatchResponse", + "sts.abc.def.avro-value", + ] + assert manifest[-1]["references"] == [ + { + "name": "com.example.sports.Operation", + "subject": "com.example.sports.Operation", + "version": "latest", + }, + { + "name": "com.example.sports.EntityType", + "subject": "com.example.sports.EntityType", + "version": "latest", + }, + { + "name": "com.example.sports.GetMatchResponse", + "subject": "com.example.sports.GetMatchResponse", + "version": "latest", + }, + ] + + response = json.loads( + (references_dir / "com.example.sports.GetMatchResponse.avsc").read_text(encoding="utf-8") + ) + assert response["fields"][-1]["type"] == ["null", "com.example.sports.Venue"]